Chris@87: from __future__ import division, absolute_import, print_function
Chris@87: 
Chris@87: import sys
Chris@87: import gzip
Chris@87: import os
Chris@87: import threading
Chris@87: from tempfile import mkstemp, NamedTemporaryFile
Chris@87: import time
Chris@87: import warnings
Chris@87: import gc
Chris@87: from io import BytesIO
Chris@87: from datetime import datetime
Chris@87: 
Chris@87: import numpy as np
Chris@87: import numpy.ma as ma
Chris@87: from numpy.lib._iotools import (ConverterError, ConverterLockError,
Chris@87:                                 ConversionWarning)
Chris@87: from numpy.compat import asbytes, asbytes_nested, bytes, asstr
Chris@87: from nose import SkipTest
Chris@87: from numpy.ma.testutils import (
Chris@87:     TestCase, assert_equal, assert_array_equal,
Chris@87:     assert_raises, assert_raises_regex, run_module_suite
Chris@87: )
Chris@87: from numpy.testing import assert_warns, assert_, build_err_msg
Chris@87: from numpy.testing.utils import tempdir
Chris@87: 
Chris@87: 
Chris@87: class TextIO(BytesIO):
Chris@87:     """Helper IO class.
Chris@87: 
Chris@87:     Writes encode strings to bytes if needed, reads return bytes.
Chris@87:     This makes it easier to emulate files opened in binary mode
Chris@87:     without needing to explicitly convert strings to bytes in
Chris@87:     setting up the test data.
Chris@87: 
Chris@87:     """
Chris@87:     def __init__(self, s=""):
Chris@87:         BytesIO.__init__(self, asbytes(s))
Chris@87: 
Chris@87:     def write(self, s):
Chris@87:         BytesIO.write(self, asbytes(s))
Chris@87: 
Chris@87:     def writelines(self, lines):
Chris@87:         BytesIO.writelines(self, [asbytes(s) for s in lines])
Chris@87: 
Chris@87: 
Chris@87: MAJVER, MINVER = sys.version_info[:2]
Chris@87: IS_64BIT = sys.maxsize > 2**32
Chris@87: 
Chris@87: 
Chris@87: def strptime(s, fmt=None):
Chris@87:     """This function is available in the datetime module only
Chris@87:     from Python >= 2.5.
Chris@87: 
Chris@87:     """
Chris@87:     if sys.version_info[0] >= 3:
Chris@87:         return datetime(*time.strptime(s.decode('latin1'), fmt)[:3])
Chris@87:     else:
Chris@87:         return datetime(*time.strptime(s, fmt)[:3])
Chris@87: 
Chris@87: 
Chris@87: class RoundtripTest(object):
Chris@87:     def roundtrip(self, save_func, *args, **kwargs):
Chris@87:         """
Chris@87:         save_func : callable
Chris@87:             Function used to save arrays to file.
Chris@87:         file_on_disk : bool
Chris@87:             If true, store the file on disk, instead of in a
Chris@87:             string buffer.
Chris@87:         save_kwds : dict
Chris@87:             Parameters passed to `save_func`.
Chris@87:         load_kwds : dict
Chris@87:             Parameters passed to `numpy.load`.
Chris@87:         args : tuple of arrays
Chris@87:             Arrays stored to file.
Chris@87: 
Chris@87:         """
Chris@87:         save_kwds = kwargs.get('save_kwds', {})
Chris@87:         load_kwds = kwargs.get('load_kwds', {})
Chris@87:         file_on_disk = kwargs.get('file_on_disk', False)
Chris@87: 
Chris@87:         if file_on_disk:
Chris@87:             target_file = NamedTemporaryFile(delete=False)
Chris@87:             load_file = target_file.name
Chris@87:         else:
Chris@87:             target_file = BytesIO()
Chris@87:             load_file = target_file
Chris@87: 
Chris@87:         try:
Chris@87:             arr = args
Chris@87: 
Chris@87:             save_func(target_file, *arr, **save_kwds)
Chris@87:             target_file.flush()
Chris@87:             target_file.seek(0)
Chris@87: 
Chris@87:             if sys.platform == 'win32' and not isinstance(target_file, BytesIO):
Chris@87:                 target_file.close()
Chris@87: 
Chris@87:             arr_reloaded = np.load(load_file, **load_kwds)
Chris@87: 
Chris@87:             self.arr = arr
Chris@87:             self.arr_reloaded = arr_reloaded
Chris@87:         finally:
Chris@87:             if not isinstance(target_file, BytesIO):
Chris@87:                 target_file.close()
Chris@87:                 # holds an open file descriptor so it can't be deleted on win
Chris@87:                 if not isinstance(arr_reloaded, np.lib.npyio.NpzFile):
Chris@87:                     os.remove(target_file.name)
Chris@87: 
Chris@87:     def check_roundtrips(self, a):
Chris@87:         self.roundtrip(a)
Chris@87:         self.roundtrip(a, file_on_disk=True)
Chris@87:         self.roundtrip(np.asfortranarray(a))
Chris@87:         self.roundtrip(np.asfortranarray(a), file_on_disk=True)
Chris@87:         if a.shape[0] > 1:
Chris@87:             # neither C nor Fortran contiguous for 2D arrays or more
Chris@87:             self.roundtrip(np.asfortranarray(a)[1:])
Chris@87:             self.roundtrip(np.asfortranarray(a)[1:], file_on_disk=True)
Chris@87: 
Chris@87:     def test_array(self):
Chris@87:         a = np.array([], float)
Chris@87:         self.check_roundtrips(a)
Chris@87: 
Chris@87:         a = np.array([[1, 2], [3, 4]], float)
Chris@87:         self.check_roundtrips(a)
Chris@87: 
Chris@87:         a = np.array([[1, 2], [3, 4]], int)
Chris@87:         self.check_roundtrips(a)
Chris@87: 
Chris@87:         a = np.array([[1 + 5j, 2 + 6j], [3 + 7j, 4 + 8j]], dtype=np.csingle)
Chris@87:         self.check_roundtrips(a)
Chris@87: 
Chris@87:         a = np.array([[1 + 5j, 2 + 6j], [3 + 7j, 4 + 8j]], dtype=np.cdouble)
Chris@87:         self.check_roundtrips(a)
Chris@87: 
Chris@87:     def test_array_object(self):
Chris@87:         if sys.version_info[:2] >= (2, 7):
Chris@87:             a = np.array([], object)
Chris@87:             self.check_roundtrips(a)
Chris@87: 
Chris@87:             a = np.array([[1, 2], [3, 4]], object)
Chris@87:             self.check_roundtrips(a)
Chris@87:         # Fails with UnpicklingError: could not find MARK on Python 2.6
Chris@87: 
Chris@87:     def test_1D(self):
Chris@87:         a = np.array([1, 2, 3, 4], int)
Chris@87:         self.roundtrip(a)
Chris@87: 
Chris@87:     @np.testing.dec.knownfailureif(sys.platform == 'win32', "Fail on Win32")
Chris@87:     def test_mmap(self):
Chris@87:         a = np.array([[1, 2.5], [4, 7.3]])
Chris@87:         self.roundtrip(a, file_on_disk=True, load_kwds={'mmap_mode': 'r'})
Chris@87: 
Chris@87:         a = np.asfortranarray([[1, 2.5], [4, 7.3]])
Chris@87:         self.roundtrip(a, file_on_disk=True, load_kwds={'mmap_mode': 'r'})
Chris@87: 
Chris@87:     def test_record(self):
Chris@87:         a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
Chris@87:         self.check_roundtrips(a)
Chris@87: 
Chris@87:     def test_format_2_0(self):
Chris@87:         dt = [(("%d" % i) * 100, float) for i in range(500)]
Chris@87:         a = np.ones(1000, dtype=dt)
Chris@87:         with warnings.catch_warnings(record=True):
Chris@87:             warnings.filterwarnings('always', '', UserWarning)
Chris@87:             self.check_roundtrips(a)
Chris@87: 
Chris@87: 
Chris@87: class TestSaveLoad(RoundtripTest, TestCase):
Chris@87:     def roundtrip(self, *args, **kwargs):
Chris@87:         RoundtripTest.roundtrip(self, np.save, *args, **kwargs)
Chris@87:         assert_equal(self.arr[0], self.arr_reloaded)
Chris@87:         assert_equal(self.arr[0].dtype, self.arr_reloaded.dtype)
Chris@87:         assert_equal(self.arr[0].flags.fnc, self.arr_reloaded.flags.fnc)
Chris@87: 
Chris@87: 
Chris@87: class TestSavezLoad(RoundtripTest, TestCase):
Chris@87:     def roundtrip(self, *args, **kwargs):
Chris@87:         RoundtripTest.roundtrip(self, np.savez, *args, **kwargs)
Chris@87:         try:
Chris@87:             for n, arr in enumerate(self.arr):
Chris@87:                 reloaded = self.arr_reloaded['arr_%d' % n]
Chris@87:                 assert_equal(arr, reloaded)
Chris@87:                 assert_equal(arr.dtype, reloaded.dtype)
Chris@87:                 assert_equal(arr.flags.fnc, reloaded.flags.fnc)
Chris@87:         finally:
Chris@87:             # delete tempfile, must be done here on windows
Chris@87:             if self.arr_reloaded.fid:
Chris@87:                 self.arr_reloaded.fid.close()
Chris@87:                 os.remove(self.arr_reloaded.fid.name)
Chris@87: 
Chris@87:     @np.testing.dec.skipif(not IS_64BIT, "Works only with 64bit systems")
Chris@87:     @np.testing.dec.slow
Chris@87:     def test_big_arrays(self):
Chris@87:         L = (1 << 31) + 100000
Chris@87:         a = np.empty(L, dtype=np.uint8)
Chris@87:         with tempdir(prefix="numpy_test_big_arrays_") as tmpdir:
Chris@87:             tmp = os.path.join(tmpdir, "file.npz")
Chris@87:             np.savez(tmp, a=a)
Chris@87:             del a
Chris@87:             npfile = np.load(tmp)
Chris@87:             a = npfile['a']
Chris@87:             npfile.close()
Chris@87: 
Chris@87:     def test_multiple_arrays(self):
Chris@87:         a = np.array([[1, 2], [3, 4]], float)
Chris@87:         b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex)
Chris@87:         self.roundtrip(a, b)
Chris@87: 
Chris@87:     def test_named_arrays(self):
Chris@87:         a = np.array([[1, 2], [3, 4]], float)
Chris@87:         b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex)
Chris@87:         c = BytesIO()
Chris@87:         np.savez(c, file_a=a, file_b=b)
Chris@87:         c.seek(0)
Chris@87:         l = np.load(c)
Chris@87:         assert_equal(a, l['file_a'])
Chris@87:         assert_equal(b, l['file_b'])
Chris@87: 
Chris@87:     def test_savez_filename_clashes(self):
Chris@87:         # Test that issue #852 is fixed
Chris@87:         # and savez functions in multithreaded environment
Chris@87: 
Chris@87:         def writer(error_list):
Chris@87:             fd, tmp = mkstemp(suffix='.npz')
Chris@87:             os.close(fd)
Chris@87:             try:
Chris@87:                 arr = np.random.randn(500, 500)
Chris@87:                 try:
Chris@87:                     np.savez(tmp, arr=arr)
Chris@87:                 except OSError as err:
Chris@87:                     error_list.append(err)
Chris@87:             finally:
Chris@87:                 os.remove(tmp)
Chris@87: 
Chris@87:         errors = []
Chris@87:         threads = [threading.Thread(target=writer, args=(errors,))
Chris@87:                    for j in range(3)]
Chris@87:         for t in threads:
Chris@87:             t.start()
Chris@87:         for t in threads:
Chris@87:             t.join()
Chris@87: 
Chris@87:         if errors:
Chris@87:             raise AssertionError(errors)
Chris@87: 
Chris@87:     def test_not_closing_opened_fid(self):
Chris@87:         # Test that issue #2178 is fixed:
Chris@87:         # verify could seek on 'loaded' file
Chris@87: 
Chris@87:         fd, tmp = mkstemp(suffix='.npz')
Chris@87:         os.close(fd)
Chris@87:         try:
Chris@87:             fp = open(tmp, 'wb')
Chris@87:             np.savez(fp, data='LOVELY LOAD')
Chris@87:             fp.close()
Chris@87: 
Chris@87:             fp = open(tmp, 'rb', 10000)
Chris@87:             fp.seek(0)
Chris@87:             assert_(not fp.closed)
Chris@87:             _ = np.load(fp)['data']
Chris@87:             assert_(not fp.closed)
Chris@87:                     # must not get closed by .load(opened fp)
Chris@87:             fp.seek(0)
Chris@87:             assert_(not fp.closed)
Chris@87: 
Chris@87:         finally:
Chris@87:             fp.close()
Chris@87:             os.remove(tmp)
Chris@87: 
Chris@87:     def test_closing_fid(self):
Chris@87:         # Test that issue #1517 (too many opened files) remains closed
Chris@87:         # It might be a "weak" test since failed to get triggered on
Chris@87:         # e.g. Debian sid of 2012 Jul 05 but was reported to
Chris@87:         # trigger the failure on Ubuntu 10.04:
Chris@87:         # http://projects.scipy.org/numpy/ticket/1517#comment:2
Chris@87:         fd, tmp = mkstemp(suffix='.npz')
Chris@87:         os.close(fd)
Chris@87: 
Chris@87:         try:
Chris@87:             fp = open(tmp, 'wb')
Chris@87:             np.savez(fp, data='LOVELY LOAD')
Chris@87:             fp.close()
Chris@87:             # We need to check if the garbage collector can properly close
Chris@87:             # numpy npz file returned by np.load when their reference count
Chris@87:             # goes to zero.  Python 3 running in debug mode raises a
Chris@87:             # ResourceWarning when file closing is left to the garbage
Chris@87:             # collector, so we catch the warnings.  Because ResourceWarning
Chris@87:             # is unknown in Python < 3.x, we take the easy way out and
Chris@87:             # catch all warnings.
Chris@87:             with warnings.catch_warnings():
Chris@87:                 warnings.simplefilter("ignore")
Chris@87:                 for i in range(1, 1025):
Chris@87:                     try:
Chris@87:                         np.load(tmp)["data"]
Chris@87:                     except Exception as e:
Chris@87:                         msg = "Failed to load data from a file: %s" % e
Chris@87:                         raise AssertionError(msg)
Chris@87:         finally:
Chris@87:             os.remove(tmp)
Chris@87: 
Chris@87:     def test_closing_zipfile_after_load(self):
Chris@87:         # Check that zipfile owns file and can close it.
Chris@87:         # This needs to pass a file name to load for the
Chris@87:         # test.
Chris@87:         with tempdir(prefix="numpy_test_closing_zipfile_after_load_") as tmpdir:
Chris@87:             fd, tmp = mkstemp(suffix='.npz', dir=tmpdir)
Chris@87:             os.close(fd)
Chris@87:             np.savez(tmp, lab='place holder')
Chris@87:             data = np.load(tmp)
Chris@87:             fp = data.zip.fp
Chris@87:             data.close()
Chris@87:             assert_(fp.closed)
Chris@87: 
Chris@87: 
Chris@87: class TestSaveTxt(TestCase):
Chris@87:     def test_array(self):
Chris@87:         a = np.array([[1, 2], [3, 4]], float)
Chris@87:         fmt = "%.18e"
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt=fmt)
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.readlines(),
Chris@87:                      [asbytes((fmt + ' ' + fmt + '\n') % (1, 2)),
Chris@87:                       asbytes((fmt + ' ' + fmt + '\n') % (3, 4))])
Chris@87: 
Chris@87:         a = np.array([[1, 2], [3, 4]], int)
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='%d')
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.readlines(), [b'1 2\n', b'3 4\n'])
Chris@87: 
Chris@87:     def test_1D(self):
Chris@87:         a = np.array([1, 2, 3, 4], int)
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='%d')
Chris@87:         c.seek(0)
Chris@87:         lines = c.readlines()
Chris@87:         assert_equal(lines, [b'1\n', b'2\n', b'3\n', b'4\n'])
Chris@87: 
Chris@87:     def test_record(self):
Chris@87:         a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='%d')
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.readlines(), [b'1 2\n', b'3 4\n'])
Chris@87: 
Chris@87:     def test_delimiter(self):
Chris@87:         a = np.array([[1., 2.], [3., 4.]])
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, delimiter=',', fmt='%d')
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.readlines(), [b'1,2\n', b'3,4\n'])
Chris@87: 
Chris@87:     def test_format(self):
Chris@87:         a = np.array([(1, 2), (3, 4)])
Chris@87:         c = BytesIO()
Chris@87:         # Sequence of formats
Chris@87:         np.savetxt(c, a, fmt=['%02d', '%3.1f'])
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.readlines(), [b'01 2.0\n', b'03 4.0\n'])
Chris@87: 
Chris@87:         # A single multiformat string
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='%02d : %3.1f')
Chris@87:         c.seek(0)
Chris@87:         lines = c.readlines()
Chris@87:         assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n'])
Chris@87: 
Chris@87:         # Specify delimiter, should be overiden
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='%02d : %3.1f', delimiter=',')
Chris@87:         c.seek(0)
Chris@87:         lines = c.readlines()
Chris@87:         assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n'])
Chris@87: 
Chris@87:         # Bad fmt, should raise a ValueError
Chris@87:         c = BytesIO()
Chris@87:         assert_raises(ValueError, np.savetxt, c, a, fmt=99)
Chris@87: 
Chris@87:     def test_header_footer(self):
Chris@87:         """
Chris@87:         Test the functionality of the header and footer keyword argument.
Chris@87:         """
Chris@87:         c = BytesIO()
Chris@87:         a = np.array([(1, 2), (3, 4)], dtype=np.int)
Chris@87:         test_header_footer = 'Test header / footer'
Chris@87:         # Test the header keyword argument
Chris@87:         np.savetxt(c, a, fmt='%1d', header=test_header_footer)
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.read(),
Chris@87:                      asbytes('# ' + test_header_footer + '\n1 2\n3 4\n'))
Chris@87:         # Test the footer keyword argument
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='%1d', footer=test_header_footer)
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.read(),
Chris@87:                      asbytes('1 2\n3 4\n# ' + test_header_footer + '\n'))
Chris@87:         # Test the commentstr keyword argument used on the header
Chris@87:         c = BytesIO()
Chris@87:         commentstr = '% '
Chris@87:         np.savetxt(c, a, fmt='%1d',
Chris@87:                    header=test_header_footer, comments=commentstr)
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.read(),
Chris@87:                      asbytes(commentstr + test_header_footer + '\n' + '1 2\n3 4\n'))
Chris@87:         # Test the commentstr keyword argument used on the footer
Chris@87:         c = BytesIO()
Chris@87:         commentstr = '% '
Chris@87:         np.savetxt(c, a, fmt='%1d',
Chris@87:                    footer=test_header_footer, comments=commentstr)
Chris@87:         c.seek(0)
Chris@87:         assert_equal(c.read(),
Chris@87:                      asbytes('1 2\n3 4\n' + commentstr + test_header_footer + '\n'))
Chris@87: 
Chris@87:     def test_file_roundtrip(self):
Chris@87:         f, name = mkstemp()
Chris@87:         os.close(f)
Chris@87:         try:
Chris@87:             a = np.array([(1, 2), (3, 4)])
Chris@87:             np.savetxt(name, a)
Chris@87:             b = np.loadtxt(name)
Chris@87:             assert_array_equal(a, b)
Chris@87:         finally:
Chris@87:             os.unlink(name)
Chris@87: 
Chris@87:     def test_complex_arrays(self):
Chris@87:         ncols = 2
Chris@87:         nrows = 2
Chris@87:         a = np.zeros((ncols, nrows), dtype=np.complex128)
Chris@87:         re = np.pi
Chris@87:         im = np.e
Chris@87:         a[:] = re + 1.0j * im
Chris@87: 
Chris@87:         # One format only
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt=' %+.3e')
Chris@87:         c.seek(0)
Chris@87:         lines = c.readlines()
Chris@87:         assert_equal(
Chris@87:             lines,
Chris@87:             [b' ( +3.142e+00+ +2.718e+00j)  ( +3.142e+00+ +2.718e+00j)\n',
Chris@87:              b' ( +3.142e+00+ +2.718e+00j)  ( +3.142e+00+ +2.718e+00j)\n'])
Chris@87: 
Chris@87:         # One format for each real and imaginary part
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt='  %+.3e' * 2 * ncols)
Chris@87:         c.seek(0)
Chris@87:         lines = c.readlines()
Chris@87:         assert_equal(
Chris@87:             lines,
Chris@87:             [b'  +3.142e+00  +2.718e+00  +3.142e+00  +2.718e+00\n',
Chris@87:              b'  +3.142e+00  +2.718e+00  +3.142e+00  +2.718e+00\n'])
Chris@87: 
Chris@87:         # One format for each complex number
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a, fmt=['(%.3e%+.3ej)'] * ncols)
Chris@87:         c.seek(0)
Chris@87:         lines = c.readlines()
Chris@87:         assert_equal(
Chris@87:             lines,
Chris@87:             [b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n',
Chris@87:              b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n'])
Chris@87: 
Chris@87:     def test_custom_writer(self):
Chris@87: 
Chris@87:         class CustomWriter(list):
Chris@87:             def write(self, text):
Chris@87:                 self.extend(text.split(b'\n'))
Chris@87: 
Chris@87:         w = CustomWriter()
Chris@87:         a = np.array([(1, 2), (3, 4)])
Chris@87:         np.savetxt(w, a)
Chris@87:         b = np.loadtxt(w)
Chris@87:         assert_array_equal(a, b)
Chris@87: 
Chris@87: 
Chris@87: class TestLoadTxt(TestCase):
Chris@87:     def test_record(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1 2\n3 4')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=[('x', np.int32), ('y', np.int32)])
Chris@87:         a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:         d = TextIO()
Chris@87:         d.write('M 64.0 75.0\nF 25.0 60.0')
Chris@87:         d.seek(0)
Chris@87:         mydescriptor = {'names': ('gender', 'age', 'weight'),
Chris@87:                         'formats': ('S1', 'i4', 'f4')}
Chris@87:         b = np.array([('M', 64.0, 75.0),
Chris@87:                       ('F', 25.0, 60.0)], dtype=mydescriptor)
Chris@87:         y = np.loadtxt(d, dtype=mydescriptor)
Chris@87:         assert_array_equal(y, b)
Chris@87: 
Chris@87:     def test_array(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1 2\n3 4')
Chris@87: 
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=np.int)
Chris@87:         a = np.array([[1, 2], [3, 4]], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=float)
Chris@87:         a = np.array([[1, 2], [3, 4]], float)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_1D(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1\n2\n3\n4\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int)
Chris@87:         a = np.array([1, 2, 3, 4], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:         c = TextIO()
Chris@87:         c.write('1,2,3,4\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',')
Chris@87:         a = np.array([1, 2, 3, 4], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_missing(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1,2,3,,5\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',',
Chris@87:                        converters={3: lambda s: int(s or - 999)})
Chris@87:         a = np.array([1, 2, 3, -999, 5], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_converters_with_usecols(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1,2,3,,5\n6,7,8,9,10\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',',
Chris@87:                        converters={3: lambda s: int(s or - 999)},
Chris@87:                        usecols=(1, 3,))
Chris@87:         a = np.array([[2, -999], [7, 9]], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_comments(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('# comment\n1,2,3,5\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',',
Chris@87:                        comments='#')
Chris@87:         a = np.array([1, 2, 3, 5], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_skiprows(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('comment\n1,2,3,5\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',',
Chris@87:                        skiprows=1)
Chris@87:         a = np.array([1, 2, 3, 5], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:         c = TextIO()
Chris@87:         c.write('# comment\n1,2,3,5\n')
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',',
Chris@87:                        skiprows=1)
Chris@87:         a = np.array([1, 2, 3, 5], int)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_usecols(self):
Chris@87:         a = np.array([[1, 2], [3, 4]], float)
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a)
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=float, usecols=(1,))
Chris@87:         assert_array_equal(x, a[:, 1])
Chris@87: 
Chris@87:         a = np.array([[1, 2, 3], [3, 4, 5]], float)
Chris@87:         c = BytesIO()
Chris@87:         np.savetxt(c, a)
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=float, usecols=(1, 2))
Chris@87:         assert_array_equal(x, a[:, 1:])
Chris@87: 
Chris@87:         # Testing with arrays instead of tuples.
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=float, usecols=np.array([1, 2]))
Chris@87:         assert_array_equal(x, a[:, 1:])
Chris@87: 
Chris@87:         # Checking with dtypes defined converters.
Chris@87:         data = '''JOE 70.1 25.3
Chris@87:                 BOB 60.5 27.9
Chris@87:                 '''
Chris@87:         c = TextIO(data)
Chris@87:         names = ['stid', 'temp']
Chris@87:         dtypes = ['S4', 'f8']
Chris@87:         arr = np.loadtxt(c, usecols=(0, 2), dtype=list(zip(names, dtypes)))
Chris@87:         assert_equal(arr['stid'], [b"JOE", b"BOB"])
Chris@87:         assert_equal(arr['temp'], [25.3, 27.9])
Chris@87: 
Chris@87:     def test_fancy_dtype(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1,2,3.0\n4,5,6.0\n')
Chris@87:         c.seek(0)
Chris@87:         dt = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
Chris@87:         x = np.loadtxt(c, dtype=dt, delimiter=',')
Chris@87:         a = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_shaped_dtype(self):
Chris@87:         c = TextIO("aaaa  1.0  8.0  1 2 3 4 5 6")
Chris@87:         dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
Chris@87:                        ('block', int, (2, 3))])
Chris@87:         x = np.loadtxt(c, dtype=dt)
Chris@87:         a = np.array([('aaaa', 1.0, 8.0, [[1, 2, 3], [4, 5, 6]])],
Chris@87:                      dtype=dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_3d_shaped_dtype(self):
Chris@87:         c = TextIO("aaaa  1.0  8.0  1 2 3 4 5 6 7 8 9 10 11 12")
Chris@87:         dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
Chris@87:                        ('block', int, (2, 2, 3))])
Chris@87:         x = np.loadtxt(c, dtype=dt)
Chris@87:         a = np.array([('aaaa', 1.0, 8.0,
Chris@87:                        [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])],
Chris@87:                      dtype=dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_empty_file(self):
Chris@87:         with warnings.catch_warnings():
Chris@87:             warnings.filterwarnings("ignore",
Chris@87:                                     message="loadtxt: Empty input file:")
Chris@87:             c = TextIO()
Chris@87:             x = np.loadtxt(c)
Chris@87:             assert_equal(x.shape, (0,))
Chris@87:             x = np.loadtxt(c, dtype=np.int64)
Chris@87:             assert_equal(x.shape, (0,))
Chris@87:             assert_(x.dtype == np.int64)
Chris@87: 
Chris@87:     def test_unused_converter(self):
Chris@87:         c = TextIO()
Chris@87:         c.writelines(['1 21\n', '3 42\n'])
Chris@87:         c.seek(0)
Chris@87:         data = np.loadtxt(c, usecols=(1,),
Chris@87:                           converters={0: lambda s: int(s, 16)})
Chris@87:         assert_array_equal(data, [21, 42])
Chris@87: 
Chris@87:         c.seek(0)
Chris@87:         data = np.loadtxt(c, usecols=(1,),
Chris@87:                           converters={1: lambda s: int(s, 16)})
Chris@87:         assert_array_equal(data, [33, 66])
Chris@87: 
Chris@87:     def test_dtype_with_object(self):
Chris@87:         "Test using an explicit dtype with an object"
Chris@87:         from datetime import date
Chris@87:         import time
Chris@87:         data = """ 1; 2001-01-01
Chris@87:                    2; 2002-01-31 """
Chris@87:         ndtype = [('idx', int), ('code', np.object)]
Chris@87:         func = lambda s: strptime(s.strip(), "%Y-%m-%d")
Chris@87:         converters = {1: func}
Chris@87:         test = np.loadtxt(TextIO(data), delimiter=";", dtype=ndtype,
Chris@87:                           converters=converters)
Chris@87:         control = np.array(
Chris@87:             [(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
Chris@87:             dtype=ndtype)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_uint64_type(self):
Chris@87:         tgt = (9223372043271415339, 9223372043271415853)
Chris@87:         c = TextIO()
Chris@87:         c.write("%s %s" % tgt)
Chris@87:         c.seek(0)
Chris@87:         res = np.loadtxt(c, dtype=np.uint64)
Chris@87:         assert_equal(res, tgt)
Chris@87: 
Chris@87:     def test_int64_type(self):
Chris@87:         tgt = (-9223372036854775807, 9223372036854775807)
Chris@87:         c = TextIO()
Chris@87:         c.write("%s %s" % tgt)
Chris@87:         c.seek(0)
Chris@87:         res = np.loadtxt(c, dtype=np.int64)
Chris@87:         assert_equal(res, tgt)
Chris@87: 
Chris@87:     def test_universal_newline(self):
Chris@87:         f, name = mkstemp()
Chris@87:         os.write(f, b'1 21\r3 42\r')
Chris@87:         os.close(f)
Chris@87: 
Chris@87:         try:
Chris@87:             data = np.loadtxt(name)
Chris@87:             assert_array_equal(data, [[1, 21], [3, 42]])
Chris@87:         finally:
Chris@87:             os.unlink(name)
Chris@87: 
Chris@87:     def test_empty_field_after_tab(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1 \t2 \t3\tstart \n4\t5\t6\t  \n7\t8\t9.5\t')
Chris@87:         c.seek(0)
Chris@87:         dt = {'names': ('x', 'y', 'z', 'comment'),
Chris@87:               'formats': ('<i4', '<i4', '<f4', '|S8')}
Chris@87:         x = np.loadtxt(c, dtype=dt, delimiter='\t')
Chris@87:         a = np.array([b'start ', b'  ', b''])
Chris@87:         assert_array_equal(x['comment'], a)
Chris@87: 
Chris@87:     def test_structure_unpack(self):
Chris@87:         txt = TextIO("M 21 72\nF 35 58")
Chris@87:         dt = {'names': ('a', 'b', 'c'), 'formats': ('|S1', '<i4', '<f4')}
Chris@87:         a, b, c = np.loadtxt(txt, dtype=dt, unpack=True)
Chris@87:         assert_(a.dtype.str == '|S1')
Chris@87:         assert_(b.dtype.str == '<i4')
Chris@87:         assert_(c.dtype.str == '<f4')
Chris@87:         assert_array_equal(a, np.array([b'M', b'F']))
Chris@87:         assert_array_equal(b, np.array([21, 35]))
Chris@87:         assert_array_equal(c, np.array([72.,  58.]))
Chris@87: 
Chris@87:     def test_ndmin_keyword(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1,2,3\n4,5,6')
Chris@87:         c.seek(0)
Chris@87:         assert_raises(ValueError, np.loadtxt, c, ndmin=3)
Chris@87:         c.seek(0)
Chris@87:         assert_raises(ValueError, np.loadtxt, c, ndmin=1.5)
Chris@87:         c.seek(0)
Chris@87:         x = np.loadtxt(c, dtype=int, delimiter=',', ndmin=1)
Chris@87:         a = np.array([[1, 2, 3], [4, 5, 6]])
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:         d = TextIO()
Chris@87:         d.write('0,1,2')
Chris@87:         d.seek(0)
Chris@87:         x = np.loadtxt(d, dtype=int, delimiter=',', ndmin=2)
Chris@87:         assert_(x.shape == (1, 3))
Chris@87:         d.seek(0)
Chris@87:         x = np.loadtxt(d, dtype=int, delimiter=',', ndmin=1)
Chris@87:         assert_(x.shape == (3,))
Chris@87:         d.seek(0)
Chris@87:         x = np.loadtxt(d, dtype=int, delimiter=',', ndmin=0)
Chris@87:         assert_(x.shape == (3,))
Chris@87: 
Chris@87:         e = TextIO()
Chris@87:         e.write('0\n1\n2')
Chris@87:         e.seek(0)
Chris@87:         x = np.loadtxt(e, dtype=int, delimiter=',', ndmin=2)
Chris@87:         assert_(x.shape == (3, 1))
Chris@87:         e.seek(0)
Chris@87:         x = np.loadtxt(e, dtype=int, delimiter=',', ndmin=1)
Chris@87:         assert_(x.shape == (3,))
Chris@87:         e.seek(0)
Chris@87:         x = np.loadtxt(e, dtype=int, delimiter=',', ndmin=0)
Chris@87:         assert_(x.shape == (3,))
Chris@87: 
Chris@87:         # Test ndmin kw with empty file.
Chris@87:         with warnings.catch_warnings():
Chris@87:             warnings.filterwarnings("ignore",
Chris@87:                                     message="loadtxt: Empty input file:")
Chris@87:             f = TextIO()
Chris@87:             assert_(np.loadtxt(f, ndmin=2).shape == (0, 1,))
Chris@87:             assert_(np.loadtxt(f, ndmin=1).shape == (0,))
Chris@87: 
Chris@87:     def test_generator_source(self):
Chris@87:         def count():
Chris@87:             for i in range(10):
Chris@87:                 yield "%d" % i
Chris@87: 
Chris@87:         res = np.loadtxt(count())
Chris@87:         assert_array_equal(res, np.arange(10))
Chris@87: 
Chris@87:     def test_bad_line(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1 2 3\n4 5 6\n2 3')
Chris@87:         c.seek(0)
Chris@87: 
Chris@87:         # Check for exception and that exception contains line number
Chris@87:         assert_raises_regex(ValueError, "3", np.loadtxt, c)
Chris@87: 
Chris@87: 
Chris@87: class Testfromregex(TestCase):
Chris@87:     # np.fromregex expects files opened in binary mode.
Chris@87:     def test_record(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1.312 foo\n1.534 bar\n4.444 qux')
Chris@87:         c.seek(0)
Chris@87: 
Chris@87:         dt = [('num', np.float64), ('val', 'S3')]
Chris@87:         x = np.fromregex(c, r"([0-9.]+)\s+(...)", dt)
Chris@87:         a = np.array([(1.312, 'foo'), (1.534, 'bar'), (4.444, 'qux')],
Chris@87:                      dtype=dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_record_2(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1312 foo\n1534 bar\n4444 qux')
Chris@87:         c.seek(0)
Chris@87: 
Chris@87:         dt = [('num', np.int32), ('val', 'S3')]
Chris@87:         x = np.fromregex(c, r"(\d+)\s+(...)", dt)
Chris@87:         a = np.array([(1312, 'foo'), (1534, 'bar'), (4444, 'qux')],
Chris@87:                      dtype=dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_record_3(self):
Chris@87:         c = TextIO()
Chris@87:         c.write('1312 foo\n1534 bar\n4444 qux')
Chris@87:         c.seek(0)
Chris@87: 
Chris@87:         dt = [('num', np.float64)]
Chris@87:         x = np.fromregex(c, r"(\d+)\s+...", dt)
Chris@87:         a = np.array([(1312,), (1534,), (4444,)], dtype=dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87: 
Chris@87: #####--------------------------------------------------------------------------
Chris@87: 
Chris@87: 
Chris@87: class TestFromTxt(TestCase):
Chris@87:     #
Chris@87:     def test_record(self):
Chris@87:         "Test w/ explicit dtype"
Chris@87:         data = TextIO('1 2\n3 4')
Chris@87: #        data.seek(0)
Chris@87:         test = np.ndfromtxt(data, dtype=[('x', np.int32), ('y', np.int32)])
Chris@87:         control = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         data = TextIO('M 64.0 75.0\nF 25.0 60.0')
Chris@87: #        data.seek(0)
Chris@87:         descriptor = {'names': ('gender', 'age', 'weight'),
Chris@87:                       'formats': ('S1', 'i4', 'f4')}
Chris@87:         control = np.array([('M', 64.0, 75.0), ('F', 25.0, 60.0)],
Chris@87:                            dtype=descriptor)
Chris@87:         test = np.ndfromtxt(data, dtype=descriptor)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_array(self):
Chris@87:         "Test outputing a standard ndarray"
Chris@87:         data = TextIO('1 2\n3 4')
Chris@87:         control = np.array([[1, 2], [3, 4]], dtype=int)
Chris@87:         test = np.ndfromtxt(data, dtype=int)
Chris@87:         assert_array_equal(test, control)
Chris@87:         #
Chris@87:         data.seek(0)
Chris@87:         control = np.array([[1, 2], [3, 4]], dtype=float)
Chris@87:         test = np.loadtxt(data, dtype=float)
Chris@87:         assert_array_equal(test, control)
Chris@87: 
Chris@87:     def test_1D(self):
Chris@87:         "Test squeezing to 1D"
Chris@87:         control = np.array([1, 2, 3, 4], int)
Chris@87:         #
Chris@87:         data = TextIO('1\n2\n3\n4\n')
Chris@87:         test = np.ndfromtxt(data, dtype=int)
Chris@87:         assert_array_equal(test, control)
Chris@87:         #
Chris@87:         data = TextIO('1,2,3,4\n')
Chris@87:         test = np.ndfromtxt(data, dtype=int, delimiter=',')
Chris@87:         assert_array_equal(test, control)
Chris@87: 
Chris@87:     def test_comments(self):
Chris@87:         "Test the stripping of comments"
Chris@87:         control = np.array([1, 2, 3, 5], int)
Chris@87:         # Comment on its own line
Chris@87:         data = TextIO('# comment\n1,2,3,5\n')
Chris@87:         test = np.ndfromtxt(data, dtype=int, delimiter=',', comments='#')
Chris@87:         assert_equal(test, control)
Chris@87:         # Comment at the end of a line
Chris@87:         data = TextIO('1,2,3,5# comment\n')
Chris@87:         test = np.ndfromtxt(data, dtype=int, delimiter=',', comments='#')
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_skiprows(self):
Chris@87:         "Test row skipping"
Chris@87:         control = np.array([1, 2, 3, 5], int)
Chris@87:         kwargs = dict(dtype=int, delimiter=',')
Chris@87:         #
Chris@87:         data = TextIO('comment\n1,2,3,5\n')
Chris@87:         test = np.ndfromtxt(data, skip_header=1, **kwargs)
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         data = TextIO('# comment\n1,2,3,5\n')
Chris@87:         test = np.loadtxt(data, skiprows=1, **kwargs)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_skip_footer(self):
Chris@87:         data = ["# %i" % i for i in range(1, 6)]
Chris@87:         data.append("A, B, C")
Chris@87:         data.extend(["%i,%3.1f,%03s" % (i, i, i) for i in range(51)])
Chris@87:         data[-1] = "99,99"
Chris@87:         kwargs = dict(delimiter=",", names=True, skip_header=5, skip_footer=10)
Chris@87:         test = np.genfromtxt(TextIO("\n".join(data)), **kwargs)
Chris@87:         ctrl = np.array([("%f" % i, "%f" % i, "%f" % i) for i in range(41)],
Chris@87:                         dtype=[(_, float) for _ in "ABC"])
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_skip_footer_with_invalid(self):
Chris@87:         with warnings.catch_warnings():
Chris@87:             warnings.filterwarnings("ignore")
Chris@87:             basestr = '1 1\n2 2\n3 3\n4 4\n5  \n6  \n7  \n'
Chris@87:             # Footer too small to get rid of all invalid values
Chris@87:             assert_raises(ValueError, np.genfromtxt,
Chris@87:                           TextIO(basestr), skip_footer=1)
Chris@87:     #        except ValueError:
Chris@87:     #            pass
Chris@87:             a = np.genfromtxt(
Chris@87:                 TextIO(basestr), skip_footer=1, invalid_raise=False)
Chris@87:             assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
Chris@87:             #
Chris@87:             a = np.genfromtxt(TextIO(basestr), skip_footer=3)
Chris@87:             assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
Chris@87:             #
Chris@87:             basestr = '1 1\n2  \n3 3\n4 4\n5  \n6 6\n7 7\n'
Chris@87:             a = np.genfromtxt(
Chris@87:                 TextIO(basestr), skip_footer=1, invalid_raise=False)
Chris@87:             assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.], [6., 6.]]))
Chris@87:             a = np.genfromtxt(
Chris@87:                 TextIO(basestr), skip_footer=3, invalid_raise=False)
Chris@87:             assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.]]))
Chris@87: 
Chris@87:     def test_header(self):
Chris@87:         "Test retrieving a header"
Chris@87:         data = TextIO('gender age weight\nM 64.0 75.0\nF 25.0 60.0')
Chris@87:         test = np.ndfromtxt(data, dtype=None, names=True)
Chris@87:         control = {'gender': np.array([b'M', b'F']),
Chris@87:                    'age': np.array([64.0, 25.0]),
Chris@87:                    'weight': np.array([75.0, 60.0])}
Chris@87:         assert_equal(test['gender'], control['gender'])
Chris@87:         assert_equal(test['age'], control['age'])
Chris@87:         assert_equal(test['weight'], control['weight'])
Chris@87: 
Chris@87:     def test_auto_dtype(self):
Chris@87:         "Test the automatic definition of the output dtype"
Chris@87:         data = TextIO('A 64 75.0 3+4j True\nBCD 25 60.0 5+6j False')
Chris@87:         test = np.ndfromtxt(data, dtype=None)
Chris@87:         control = [np.array([b'A', b'BCD']),
Chris@87:                    np.array([64, 25]),
Chris@87:                    np.array([75.0, 60.0]),
Chris@87:                    np.array([3 + 4j, 5 + 6j]),
Chris@87:                    np.array([True, False]), ]
Chris@87:         assert_equal(test.dtype.names, ['f0', 'f1', 'f2', 'f3', 'f4'])
Chris@87:         for (i, ctrl) in enumerate(control):
Chris@87:             assert_equal(test['f%i' % i], ctrl)
Chris@87: 
Chris@87:     def test_auto_dtype_uniform(self):
Chris@87:         "Tests whether the output dtype can be uniformized"
Chris@87:         data = TextIO('1 2 3 4\n5 6 7 8\n')
Chris@87:         test = np.ndfromtxt(data, dtype=None)
Chris@87:         control = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_fancy_dtype(self):
Chris@87:         "Check that a nested dtype isn't MIA"
Chris@87:         data = TextIO('1,2,3.0\n4,5,6.0\n')
Chris@87:         fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
Chris@87:         test = np.ndfromtxt(data, dtype=fancydtype, delimiter=',')
Chris@87:         control = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dtype=fancydtype)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_names_overwrite(self):
Chris@87:         "Test overwriting the names of the dtype"
Chris@87:         descriptor = {'names': ('g', 'a', 'w'),
Chris@87:                       'formats': ('S1', 'i4', 'f4')}
Chris@87:         data = TextIO(b'M 64.0 75.0\nF 25.0 60.0')
Chris@87:         names = ('gender', 'age', 'weight')
Chris@87:         test = np.ndfromtxt(data, dtype=descriptor, names=names)
Chris@87:         descriptor['names'] = names
Chris@87:         control = np.array([('M', 64.0, 75.0),
Chris@87:                             ('F', 25.0, 60.0)], dtype=descriptor)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_commented_header(self):
Chris@87:         "Check that names can be retrieved even if the line is commented out."
Chris@87:         data = TextIO("""
Chris@87: #gender age weight
Chris@87: M   21  72.100000
Chris@87: F   35  58.330000
Chris@87: M   33  21.99
Chris@87:         """)
Chris@87:         # The # is part of the first name and should be deleted automatically.
Chris@87:         test = np.genfromtxt(data, names=True, dtype=None)
Chris@87:         ctrl = np.array([('M', 21, 72.1), ('F', 35, 58.33), ('M', 33, 21.99)],
Chris@87:                         dtype=[('gender', '|S1'), ('age', int), ('weight', float)])
Chris@87:         assert_equal(test, ctrl)
Chris@87:         # Ditto, but we should get rid of the first element
Chris@87:         data = TextIO(b"""
Chris@87: # gender age weight
Chris@87: M   21  72.100000
Chris@87: F   35  58.330000
Chris@87: M   33  21.99
Chris@87:         """)
Chris@87:         test = np.genfromtxt(data, names=True, dtype=None)
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_autonames_and_usecols(self):
Chris@87:         "Tests names and usecols"
Chris@87:         data = TextIO('A B C D\n aaaa 121 45 9.1')
Chris@87:         test = np.ndfromtxt(data, usecols=('A', 'C', 'D'),
Chris@87:                             names=True, dtype=None)
Chris@87:         control = np.array(('aaaa', 45, 9.1),
Chris@87:                            dtype=[('A', '|S4'), ('C', int), ('D', float)])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_converters_with_usecols(self):
Chris@87:         "Test the combination user-defined converters and usecol"
Chris@87:         data = TextIO('1,2,3,,5\n6,7,8,9,10\n')
Chris@87:         test = np.ndfromtxt(data, dtype=int, delimiter=',',
Chris@87:                             converters={3: lambda s: int(s or - 999)},
Chris@87:                             usecols=(1, 3,))
Chris@87:         control = np.array([[2, -999], [7, 9]], int)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_converters_with_usecols_and_names(self):
Chris@87:         "Tests names and usecols"
Chris@87:         data = TextIO('A B C D\n aaaa 121 45 9.1')
Chris@87:         test = np.ndfromtxt(data, usecols=('A', 'C', 'D'), names=True,
Chris@87:                             dtype=None, converters={'C': lambda s: 2 * int(s)})
Chris@87:         control = np.array(('aaaa', 90, 9.1),
Chris@87:                            dtype=[('A', '|S4'), ('C', int), ('D', float)])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_converters_cornercases(self):
Chris@87:         "Test the conversion to datetime."
Chris@87:         converter = {
Chris@87:             'date': lambda s: strptime(s, '%Y-%m-%d %H:%M:%SZ')}
Chris@87:         data = TextIO('2009-02-03 12:00:00Z, 72214.0')
Chris@87:         test = np.ndfromtxt(data, delimiter=',', dtype=None,
Chris@87:                             names=['date', 'stid'], converters=converter)
Chris@87:         control = np.array((datetime(2009, 2, 3), 72214.),
Chris@87:                            dtype=[('date', np.object_), ('stid', float)])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_converters_cornercases2(self):
Chris@87:         "Test the conversion to datetime64."
Chris@87:         converter = {
Chris@87:             'date': lambda s: np.datetime64(strptime(s, '%Y-%m-%d %H:%M:%SZ'))}
Chris@87:         data = TextIO('2009-02-03 12:00:00Z, 72214.0')
Chris@87:         test = np.ndfromtxt(data, delimiter=',', dtype=None,
Chris@87:                             names=['date', 'stid'], converters=converter)
Chris@87:         control = np.array((datetime(2009, 2, 3), 72214.),
Chris@87:                            dtype=[('date', 'datetime64[us]'), ('stid', float)])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_unused_converter(self):
Chris@87:         "Test whether unused converters are forgotten"
Chris@87:         data = TextIO("1 21\n  3 42\n")
Chris@87:         test = np.ndfromtxt(data, usecols=(1,),
Chris@87:                             converters={0: lambda s: int(s, 16)})
Chris@87:         assert_equal(test, [21, 42])
Chris@87:         #
Chris@87:         data.seek(0)
Chris@87:         test = np.ndfromtxt(data, usecols=(1,),
Chris@87:                             converters={1: lambda s: int(s, 16)})
Chris@87:         assert_equal(test, [33, 66])
Chris@87: 
Chris@87:     def test_invalid_converter(self):
Chris@87:         strip_rand = lambda x: float((b'r' in x.lower() and x.split()[-1]) or
Chris@87:                                      (b'r' not in x.lower() and x.strip() or 0.0))
Chris@87:         strip_per = lambda x: float((b'%' in x.lower() and x.split()[0]) or
Chris@87:                                     (b'%' not in x.lower() and x.strip() or 0.0))
Chris@87:         s = TextIO("D01N01,10/1/2003 ,1 %,R 75,400,600\r\n"
Chris@87:                    "L24U05,12/5/2003, 2 %,1,300, 150.5\r\n"
Chris@87:                    "D02N03,10/10/2004,R 1,,7,145.55")
Chris@87:         kwargs = dict(
Chris@87:             converters={2: strip_per, 3: strip_rand}, delimiter=",",
Chris@87:             dtype=None)
Chris@87:         assert_raises(ConverterError, np.genfromtxt, s, **kwargs)
Chris@87: 
Chris@87:     def test_tricky_converter_bug1666(self):
Chris@87:         "Test some corner case"
Chris@87:         s = TextIO('q1,2\nq3,4')
Chris@87:         cnv = lambda s: float(s[1:])
Chris@87:         test = np.genfromtxt(s, delimiter=',', converters={0: cnv})
Chris@87:         control = np.array([[1., 2.], [3., 4.]])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_dtype_with_converters(self):
Chris@87:         dstr = "2009; 23; 46"
Chris@87:         test = np.ndfromtxt(TextIO(dstr,),
Chris@87:                             delimiter=";", dtype=float, converters={0: bytes})
Chris@87:         control = np.array([('2009', 23., 46)],
Chris@87:                            dtype=[('f0', '|S4'), ('f1', float), ('f2', float)])
Chris@87:         assert_equal(test, control)
Chris@87:         test = np.ndfromtxt(TextIO(dstr,),
Chris@87:                             delimiter=";", dtype=float, converters={0: float})
Chris@87:         control = np.array([2009., 23., 46],)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_dtype_with_converters_and_usecols(self):
Chris@87:         dstr = "1,5,-1,1:1\n2,8,-1,1:n\n3,3,-2,m:n\n"
Chris@87:         dmap = {'1:1':0, '1:n':1, 'm:1':2, 'm:n':3}
Chris@87:         dtyp = [('E1','i4'),('E2','i4'),('E3','i2'),('N', 'i1')]
Chris@87:         conv = {0: int, 1: int, 2: int, 3: lambda r: dmap[r.decode()]}
Chris@87:         test = np.recfromcsv(TextIO(dstr,), dtype=dtyp, delimiter=',',
Chris@87:                              names=None, converters=conv)
Chris@87:         control = np.rec.array([[1,5,-1,0], [2,8,-1,1], [3,3,-2,3]], dtype=dtyp)
Chris@87:         assert_equal(test, control)
Chris@87:         dtyp = [('E1','i4'),('E2','i4'),('N', 'i1')]
Chris@87:         test = np.recfromcsv(TextIO(dstr,), dtype=dtyp, delimiter=',',
Chris@87:                              usecols=(0,1,3), names=None, converters=conv)
Chris@87:         control = np.rec.array([[1,5,0], [2,8,1], [3,3,3]], dtype=dtyp)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_dtype_with_object(self):
Chris@87:         "Test using an explicit dtype with an object"
Chris@87:         from datetime import date
Chris@87:         import time
Chris@87:         data = """ 1; 2001-01-01
Chris@87:                    2; 2002-01-31 """
Chris@87:         ndtype = [('idx', int), ('code', np.object)]
Chris@87:         func = lambda s: strptime(s.strip(), "%Y-%m-%d")
Chris@87:         converters = {1: func}
Chris@87:         test = np.genfromtxt(TextIO(data), delimiter=";", dtype=ndtype,
Chris@87:                              converters=converters)
Chris@87:         control = np.array(
Chris@87:             [(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
Chris@87:             dtype=ndtype)
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         ndtype = [('nest', [('idx', int), ('code', np.object)])]
Chris@87:         try:
Chris@87:             test = np.genfromtxt(TextIO(data), delimiter=";",
Chris@87:                                  dtype=ndtype, converters=converters)
Chris@87:         except NotImplementedError:
Chris@87:             pass
Chris@87:         else:
Chris@87:             errmsg = "Nested dtype involving objects should be supported."
Chris@87:             raise AssertionError(errmsg)
Chris@87: 
Chris@87:     def test_userconverters_with_explicit_dtype(self):
Chris@87:         "Test user_converters w/ explicit (standard) dtype"
Chris@87:         data = TextIO('skip,skip,2001-01-01,1.0,skip')
Chris@87:         test = np.genfromtxt(data, delimiter=",", names=None, dtype=float,
Chris@87:                              usecols=(2, 3), converters={2: bytes})
Chris@87:         control = np.array([('2001-01-01', 1.)],
Chris@87:                            dtype=[('', '|S10'), ('', float)])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_spacedelimiter(self):
Chris@87:         "Test space delimiter"
Chris@87:         data = TextIO("1  2  3  4   5\n6  7  8  9  10")
Chris@87:         test = np.ndfromtxt(data)
Chris@87:         control = np.array([[1., 2., 3., 4., 5.],
Chris@87:                             [6., 7., 8., 9., 10.]])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_integer_delimiter(self):
Chris@87:         "Test using an integer for delimiter"
Chris@87:         data = "  1  2  3\n  4  5 67\n890123  4"
Chris@87:         test = np.genfromtxt(TextIO(data), delimiter=3)
Chris@87:         control = np.array([[1, 2, 3], [4, 5, 67], [890, 123, 4]])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_missing(self):
Chris@87:         data = TextIO('1,2,3,,5\n')
Chris@87:         test = np.ndfromtxt(data, dtype=int, delimiter=',',
Chris@87:                             converters={3: lambda s: int(s or - 999)})
Chris@87:         control = np.array([1, 2, 3, -999, 5], int)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_missing_with_tabs(self):
Chris@87:         "Test w/ a delimiter tab"
Chris@87:         txt = "1\t2\t3\n\t2\t\n1\t\t3"
Chris@87:         test = np.genfromtxt(TextIO(txt), delimiter="\t",
Chris@87:                              usemask=True,)
Chris@87:         ctrl_d = np.array([(1, 2, 3), (np.nan, 2, np.nan), (1, np.nan, 3)],)
Chris@87:         ctrl_m = np.array([(0, 0, 0), (1, 0, 1), (0, 1, 0)], dtype=bool)
Chris@87:         assert_equal(test.data, ctrl_d)
Chris@87:         assert_equal(test.mask, ctrl_m)
Chris@87: 
Chris@87:     def test_usecols(self):
Chris@87:         "Test the selection of columns"
Chris@87:         # Select 1 column
Chris@87:         control = np.array([[1, 2], [3, 4]], float)
Chris@87:         data = TextIO()
Chris@87:         np.savetxt(data, control)
Chris@87:         data.seek(0)
Chris@87:         test = np.ndfromtxt(data, dtype=float, usecols=(1,))
Chris@87:         assert_equal(test, control[:, 1])
Chris@87:         #
Chris@87:         control = np.array([[1, 2, 3], [3, 4, 5]], float)
Chris@87:         data = TextIO()
Chris@87:         np.savetxt(data, control)
Chris@87:         data.seek(0)
Chris@87:         test = np.ndfromtxt(data, dtype=float, usecols=(1, 2))
Chris@87:         assert_equal(test, control[:, 1:])
Chris@87:         # Testing with arrays instead of tuples.
Chris@87:         data.seek(0)
Chris@87:         test = np.ndfromtxt(data, dtype=float, usecols=np.array([1, 2]))
Chris@87:         assert_equal(test, control[:, 1:])
Chris@87: 
Chris@87:     def test_usecols_as_css(self):
Chris@87:         "Test giving usecols with a comma-separated string"
Chris@87:         data = "1 2 3\n4 5 6"
Chris@87:         test = np.genfromtxt(TextIO(data),
Chris@87:                              names="a, b, c", usecols="a, c")
Chris@87:         ctrl = np.array([(1, 3), (4, 6)], dtype=[(_, float) for _ in "ac"])
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_usecols_with_structured_dtype(self):
Chris@87:         "Test usecols with an explicit structured dtype"
Chris@87:         data = TextIO("JOE 70.1 25.3\nBOB 60.5 27.9")
Chris@87:         names = ['stid', 'temp']
Chris@87:         dtypes = ['S4', 'f8']
Chris@87:         test = np.ndfromtxt(
Chris@87:             data, usecols=(0, 2), dtype=list(zip(names, dtypes)))
Chris@87:         assert_equal(test['stid'], [b"JOE", b"BOB"])
Chris@87:         assert_equal(test['temp'], [25.3, 27.9])
Chris@87: 
Chris@87:     def test_usecols_with_integer(self):
Chris@87:         "Test usecols with an integer"
Chris@87:         test = np.genfromtxt(TextIO(b"1 2 3\n4 5 6"), usecols=0)
Chris@87:         assert_equal(test, np.array([1., 4.]))
Chris@87: 
Chris@87:     def test_usecols_with_named_columns(self):
Chris@87:         "Test usecols with named columns"
Chris@87:         ctrl = np.array([(1, 3), (4, 6)], dtype=[('a', float), ('c', float)])
Chris@87:         data = "1 2 3\n4 5 6"
Chris@87:         kwargs = dict(names="a, b, c")
Chris@87:         test = np.genfromtxt(TextIO(data), usecols=(0, -1), **kwargs)
Chris@87:         assert_equal(test, ctrl)
Chris@87:         test = np.genfromtxt(TextIO(data),
Chris@87:                              usecols=('a', 'c'), **kwargs)
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_empty_file(self):
Chris@87:         "Test that an empty file raises the proper warning."
Chris@87:         with warnings.catch_warnings():
Chris@87:             warnings.filterwarnings("ignore",
Chris@87:                                     message="genfromtxt: Empty input file:")
Chris@87:             data = TextIO()
Chris@87:             test = np.genfromtxt(data)
Chris@87:             assert_equal(test, np.array([]))
Chris@87: 
Chris@87:     def test_fancy_dtype_alt(self):
Chris@87:         "Check that a nested dtype isn't MIA"
Chris@87:         data = TextIO('1,2,3.0\n4,5,6.0\n')
Chris@87:         fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
Chris@87:         test = np.mafromtxt(data, dtype=fancydtype, delimiter=',')
Chris@87:         control = ma.array([(1, (2, 3.0)), (4, (5, 6.0))], dtype=fancydtype)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_shaped_dtype(self):
Chris@87:         c = TextIO("aaaa  1.0  8.0  1 2 3 4 5 6")
Chris@87:         dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
Chris@87:                        ('block', int, (2, 3))])
Chris@87:         x = np.ndfromtxt(c, dtype=dt)
Chris@87:         a = np.array([('aaaa', 1.0, 8.0, [[1, 2, 3], [4, 5, 6]])],
Chris@87:                      dtype=dt)
Chris@87:         assert_array_equal(x, a)
Chris@87: 
Chris@87:     def test_withmissing(self):
Chris@87:         data = TextIO('A,B\n0,1\n2,N/A')
Chris@87:         kwargs = dict(delimiter=",", missing_values="N/A", names=True)
Chris@87:         test = np.mafromtxt(data, dtype=None, **kwargs)
Chris@87:         control = ma.array([(0, 1), (2, -1)],
Chris@87:                            mask=[(False, False), (False, True)],
Chris@87:                            dtype=[('A', np.int), ('B', np.int)])
Chris@87:         assert_equal(test, control)
Chris@87:         assert_equal(test.mask, control.mask)
Chris@87:         #
Chris@87:         data.seek(0)
Chris@87:         test = np.mafromtxt(data, **kwargs)
Chris@87:         control = ma.array([(0, 1), (2, -1)],
Chris@87:                            mask=[(False, False), (False, True)],
Chris@87:                            dtype=[('A', np.float), ('B', np.float)])
Chris@87:         assert_equal(test, control)
Chris@87:         assert_equal(test.mask, control.mask)
Chris@87: 
Chris@87:     def test_user_missing_values(self):
Chris@87:         data = "A, B, C\n0, 0., 0j\n1, N/A, 1j\n-9, 2.2, N/A\n3, -99, 3j"
Chris@87:         basekwargs = dict(dtype=None, delimiter=",", names=True,)
Chris@87:         mdtype = [('A', int), ('B', float), ('C', complex)]
Chris@87:         #
Chris@87:         test = np.mafromtxt(TextIO(data), missing_values="N/A",
Chris@87:                             **basekwargs)
Chris@87:         control = ma.array([(0, 0.0, 0j), (1, -999, 1j),
Chris@87:                             (-9, 2.2, -999j), (3, -99, 3j)],
Chris@87:                            mask=[(0, 0, 0), (0, 1, 0), (0, 0, 1), (0, 0, 0)],
Chris@87:                            dtype=mdtype)
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         basekwargs['dtype'] = mdtype
Chris@87:         test = np.mafromtxt(TextIO(data),
Chris@87:                             missing_values={0: -9, 1: -99, 2: -999j}, **basekwargs)
Chris@87:         control = ma.array([(0, 0.0, 0j), (1, -999, 1j),
Chris@87:                             (-9, 2.2, -999j), (3, -99, 3j)],
Chris@87:                            mask=[(0, 0, 0), (0, 1, 0), (1, 0, 1), (0, 1, 0)],
Chris@87:                            dtype=mdtype)
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         test = np.mafromtxt(TextIO(data),
Chris@87:                             missing_values={0: -9, 'B': -99, 'C': -999j},
Chris@87:                             **basekwargs)
Chris@87:         control = ma.array([(0, 0.0, 0j), (1, -999, 1j),
Chris@87:                             (-9, 2.2, -999j), (3, -99, 3j)],
Chris@87:                            mask=[(0, 0, 0), (0, 1, 0), (1, 0, 1), (0, 1, 0)],
Chris@87:                            dtype=mdtype)
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_user_filling_values(self):
Chris@87:         "Test with missing and filling values"
Chris@87:         ctrl = np.array([(0, 3), (4, -999)], dtype=[('a', int), ('b', int)])
Chris@87:         data = "N/A, 2, 3\n4, ,???"
Chris@87:         kwargs = dict(delimiter=",",
Chris@87:                       dtype=int,
Chris@87:                       names="a,b,c",
Chris@87:                       missing_values={0: "N/A", 'b': " ", 2: "???"},
Chris@87:                       filling_values={0: 0, 'b': 0, 2: -999})
Chris@87:         test = np.genfromtxt(TextIO(data), **kwargs)
Chris@87:         ctrl = np.array([(0, 2, 3), (4, 0, -999)],
Chris@87:                         dtype=[(_, int) for _ in "abc"])
Chris@87:         assert_equal(test, ctrl)
Chris@87:         #
Chris@87:         test = np.genfromtxt(TextIO(data), usecols=(0, -1), **kwargs)
Chris@87:         ctrl = np.array([(0, 3), (4, -999)], dtype=[(_, int) for _ in "ac"])
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:         data2 = "1,2,*,4\n5,*,7,8\n"
Chris@87:         test = np.genfromtxt(TextIO(data2), delimiter=',', dtype=int,
Chris@87:                              missing_values="*", filling_values=0)
Chris@87:         ctrl = np.array([[1, 2, 0, 4], [5, 0, 7, 8]])
Chris@87:         assert_equal(test, ctrl)
Chris@87:         test = np.genfromtxt(TextIO(data2), delimiter=',', dtype=int,
Chris@87:                              missing_values="*", filling_values=-1)
Chris@87:         ctrl = np.array([[1, 2, -1, 4], [5, -1, 7, 8]])
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_withmissing_float(self):
Chris@87:         data = TextIO('A,B\n0,1.5\n2,-999.00')
Chris@87:         test = np.mafromtxt(data, dtype=None, delimiter=',',
Chris@87:                             missing_values='-999.0', names=True,)
Chris@87:         control = ma.array([(0, 1.5), (2, -1.)],
Chris@87:                            mask=[(False, False), (False, True)],
Chris@87:                            dtype=[('A', np.int), ('B', np.float)])
Chris@87:         assert_equal(test, control)
Chris@87:         assert_equal(test.mask, control.mask)
Chris@87: 
Chris@87:     def test_with_masked_column_uniform(self):
Chris@87:         "Test masked column"
Chris@87:         data = TextIO('1 2 3\n4 5 6\n')
Chris@87:         test = np.genfromtxt(data, dtype=None,
Chris@87:                              missing_values='2,5', usemask=True)
Chris@87:         control = ma.array([[1, 2, 3], [4, 5, 6]], mask=[[0, 1, 0], [0, 1, 0]])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_with_masked_column_various(self):
Chris@87:         "Test masked column"
Chris@87:         data = TextIO('True 2 3\nFalse 5 6\n')
Chris@87:         test = np.genfromtxt(data, dtype=None,
Chris@87:                              missing_values='2,5', usemask=True)
Chris@87:         control = ma.array([(1, 2, 3), (0, 5, 6)],
Chris@87:                            mask=[(0, 1, 0), (0, 1, 0)],
Chris@87:                            dtype=[('f0', bool), ('f1', bool), ('f2', int)])
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_invalid_raise(self):
Chris@87:         "Test invalid raise"
Chris@87:         data = ["1, 1, 1, 1, 1"] * 50
Chris@87:         for i in range(5):
Chris@87:             data[10 * i] = "2, 2, 2, 2 2"
Chris@87:         data.insert(0, "a, b, c, d, e")
Chris@87:         mdata = TextIO("\n".join(data))
Chris@87:         #
Chris@87:         kwargs = dict(delimiter=",", dtype=None, names=True)
Chris@87:         # XXX: is there a better way to get the return value of the callable in
Chris@87:         # assert_warns ?
Chris@87:         ret = {}
Chris@87: 
Chris@87:         def f(_ret={}):
Chris@87:             _ret['mtest'] = np.ndfromtxt(mdata, invalid_raise=False, **kwargs)
Chris@87:         assert_warns(ConversionWarning, f, _ret=ret)
Chris@87:         mtest = ret['mtest']
Chris@87:         assert_equal(len(mtest), 45)
Chris@87:         assert_equal(mtest, np.ones(45, dtype=[(_, int) for _ in 'abcde']))
Chris@87:         #
Chris@87:         mdata.seek(0)
Chris@87:         assert_raises(ValueError, np.ndfromtxt, mdata,
Chris@87:                       delimiter=",", names=True)
Chris@87: 
Chris@87:     def test_invalid_raise_with_usecols(self):
Chris@87:         "Test invalid_raise with usecols"
Chris@87:         data = ["1, 1, 1, 1, 1"] * 50
Chris@87:         for i in range(5):
Chris@87:             data[10 * i] = "2, 2, 2, 2 2"
Chris@87:         data.insert(0, "a, b, c, d, e")
Chris@87:         mdata = TextIO("\n".join(data))
Chris@87:         kwargs = dict(delimiter=",", dtype=None, names=True,
Chris@87:                       invalid_raise=False)
Chris@87:         # XXX: is there a better way to get the return value of the callable in
Chris@87:         # assert_warns ?
Chris@87:         ret = {}
Chris@87: 
Chris@87:         def f(_ret={}):
Chris@87:             _ret['mtest'] = np.ndfromtxt(mdata, usecols=(0, 4), **kwargs)
Chris@87:         assert_warns(ConversionWarning, f, _ret=ret)
Chris@87:         mtest = ret['mtest']
Chris@87:         assert_equal(len(mtest), 45)
Chris@87:         assert_equal(mtest, np.ones(45, dtype=[(_, int) for _ in 'ae']))
Chris@87:         #
Chris@87:         mdata.seek(0)
Chris@87:         mtest = np.ndfromtxt(mdata, usecols=(0, 1), **kwargs)
Chris@87:         assert_equal(len(mtest), 50)
Chris@87:         control = np.ones(50, dtype=[(_, int) for _ in 'ab'])
Chris@87:         control[[10 * _ for _ in range(5)]] = (2, 2)
Chris@87:         assert_equal(mtest, control)
Chris@87: 
Chris@87:     def test_inconsistent_dtype(self):
Chris@87:         "Test inconsistent dtype"
Chris@87:         data = ["1, 1, 1, 1, -1.1"] * 50
Chris@87:         mdata = TextIO("\n".join(data))
Chris@87: 
Chris@87:         converters = {4: lambda x: "(%s)" % x}
Chris@87:         kwargs = dict(delimiter=",", converters=converters,
Chris@87:                       dtype=[(_, int) for _ in 'abcde'],)
Chris@87:         assert_raises(ValueError, np.genfromtxt, mdata, **kwargs)
Chris@87: 
Chris@87:     def test_default_field_format(self):
Chris@87:         "Test default format"
Chris@87:         data = "0, 1, 2.3\n4, 5, 6.7"
Chris@87:         mtest = np.ndfromtxt(TextIO(data),
Chris@87:                              delimiter=",", dtype=None, defaultfmt="f%02i")
Chris@87:         ctrl = np.array([(0, 1, 2.3), (4, 5, 6.7)],
Chris@87:                         dtype=[("f00", int), ("f01", int), ("f02", float)])
Chris@87:         assert_equal(mtest, ctrl)
Chris@87: 
Chris@87:     def test_single_dtype_wo_names(self):
Chris@87:         "Test single dtype w/o names"
Chris@87:         data = "0, 1, 2.3\n4, 5, 6.7"
Chris@87:         mtest = np.ndfromtxt(TextIO(data),
Chris@87:                              delimiter=",", dtype=float, defaultfmt="f%02i")
Chris@87:         ctrl = np.array([[0., 1., 2.3], [4., 5., 6.7]], dtype=float)
Chris@87:         assert_equal(mtest, ctrl)
Chris@87: 
Chris@87:     def test_single_dtype_w_explicit_names(self):
Chris@87:         "Test single dtype w explicit names"
Chris@87:         data = "0, 1, 2.3\n4, 5, 6.7"
Chris@87:         mtest = np.ndfromtxt(TextIO(data),
Chris@87:                              delimiter=",", dtype=float, names="a, b, c")
Chris@87:         ctrl = np.array([(0., 1., 2.3), (4., 5., 6.7)],
Chris@87:                         dtype=[(_, float) for _ in "abc"])
Chris@87:         assert_equal(mtest, ctrl)
Chris@87: 
Chris@87:     def test_single_dtype_w_implicit_names(self):
Chris@87:         "Test single dtype w implicit names"
Chris@87:         data = "a, b, c\n0, 1, 2.3\n4, 5, 6.7"
Chris@87:         mtest = np.ndfromtxt(TextIO(data),
Chris@87:                              delimiter=",", dtype=float, names=True)
Chris@87:         ctrl = np.array([(0., 1., 2.3), (4., 5., 6.7)],
Chris@87:                         dtype=[(_, float) for _ in "abc"])
Chris@87:         assert_equal(mtest, ctrl)
Chris@87: 
Chris@87:     def test_easy_structured_dtype(self):
Chris@87:         "Test easy structured dtype"
Chris@87:         data = "0, 1, 2.3\n4, 5, 6.7"
Chris@87:         mtest = np.ndfromtxt(TextIO(data), delimiter=",",
Chris@87:                              dtype=(int, float, float), defaultfmt="f_%02i")
Chris@87:         ctrl = np.array([(0, 1., 2.3), (4, 5., 6.7)],
Chris@87:                         dtype=[("f_00", int), ("f_01", float), ("f_02", float)])
Chris@87:         assert_equal(mtest, ctrl)
Chris@87: 
Chris@87:     def test_autostrip(self):
Chris@87:         "Test autostrip"
Chris@87:         data = "01/01/2003  , 1.3,   abcde"
Chris@87:         kwargs = dict(delimiter=",", dtype=None)
Chris@87:         mtest = np.ndfromtxt(TextIO(data), **kwargs)
Chris@87:         ctrl = np.array([('01/01/2003  ', 1.3, '   abcde')],
Chris@87:                         dtype=[('f0', '|S12'), ('f1', float), ('f2', '|S8')])
Chris@87:         assert_equal(mtest, ctrl)
Chris@87:         mtest = np.ndfromtxt(TextIO(data), autostrip=True, **kwargs)
Chris@87:         ctrl = np.array([('01/01/2003', 1.3, 'abcde')],
Chris@87:                         dtype=[('f0', '|S10'), ('f1', float), ('f2', '|S5')])
Chris@87:         assert_equal(mtest, ctrl)
Chris@87: 
Chris@87:     def test_replace_space(self):
Chris@87:         "Test the 'replace_space' option"
Chris@87:         txt = "A.A, B (B), C:C\n1, 2, 3.14"
Chris@87:         # Test default: replace ' ' by '_' and delete non-alphanum chars
Chris@87:         test = np.genfromtxt(TextIO(txt),
Chris@87:                              delimiter=",", names=True, dtype=None)
Chris@87:         ctrl_dtype = [("AA", int), ("B_B", int), ("CC", float)]
Chris@87:         ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
Chris@87:         assert_equal(test, ctrl)
Chris@87:         # Test: no replace, no delete
Chris@87:         test = np.genfromtxt(TextIO(txt),
Chris@87:                              delimiter=",", names=True, dtype=None,
Chris@87:                              replace_space='', deletechars='')
Chris@87:         ctrl_dtype = [("A.A", int), ("B (B)", int), ("C:C", float)]
Chris@87:         ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
Chris@87:         assert_equal(test, ctrl)
Chris@87:         # Test: no delete (spaces are replaced by _)
Chris@87:         test = np.genfromtxt(TextIO(txt),
Chris@87:                              delimiter=",", names=True, dtype=None,
Chris@87:                              deletechars='')
Chris@87:         ctrl_dtype = [("A.A", int), ("B_(B)", int), ("C:C", float)]
Chris@87:         ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_incomplete_names(self):
Chris@87:         "Test w/ incomplete names"
Chris@87:         data = "A,,C\n0,1,2\n3,4,5"
Chris@87:         kwargs = dict(delimiter=",", names=True)
Chris@87:         # w/ dtype=None
Chris@87:         ctrl = np.array([(0, 1, 2), (3, 4, 5)],
Chris@87:                         dtype=[(_, int) for _ in ('A', 'f0', 'C')])
Chris@87:         test = np.ndfromtxt(TextIO(data), dtype=None, **kwargs)
Chris@87:         assert_equal(test, ctrl)
Chris@87:         # w/ default dtype
Chris@87:         ctrl = np.array([(0, 1, 2), (3, 4, 5)],
Chris@87:                         dtype=[(_, float) for _ in ('A', 'f0', 'C')])
Chris@87:         test = np.ndfromtxt(TextIO(data), **kwargs)
Chris@87: 
Chris@87:     def test_names_auto_completion(self):
Chris@87:         "Make sure that names are properly completed"
Chris@87:         data = "1 2 3\n 4 5 6"
Chris@87:         test = np.genfromtxt(TextIO(data),
Chris@87:                              dtype=(int, float, int), names="a")
Chris@87:         ctrl = np.array([(1, 2, 3), (4, 5, 6)],
Chris@87:                         dtype=[('a', int), ('f0', float), ('f1', int)])
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_names_with_usecols_bug1636(self):
Chris@87:         "Make sure we pick up the right names w/ usecols"
Chris@87:         data = "A,B,C,D,E\n0,1,2,3,4\n0,1,2,3,4\n0,1,2,3,4"
Chris@87:         ctrl_names = ("A", "C", "E")
Chris@87:         test = np.genfromtxt(TextIO(data),
Chris@87:                              dtype=(int, int, int), delimiter=",",
Chris@87:                              usecols=(0, 2, 4), names=True)
Chris@87:         assert_equal(test.dtype.names, ctrl_names)
Chris@87:         #
Chris@87:         test = np.genfromtxt(TextIO(data),
Chris@87:                              dtype=(int, int, int), delimiter=",",
Chris@87:                              usecols=("A", "C", "E"), names=True)
Chris@87:         assert_equal(test.dtype.names, ctrl_names)
Chris@87:         #
Chris@87:         test = np.genfromtxt(TextIO(data),
Chris@87:                              dtype=int, delimiter=",",
Chris@87:                              usecols=("A", "C", "E"), names=True)
Chris@87:         assert_equal(test.dtype.names, ctrl_names)
Chris@87: 
Chris@87:     def test_fixed_width_names(self):
Chris@87:         "Test fix-width w/ names"
Chris@87:         data = "    A    B   C\n    0    1 2.3\n   45   67   9."
Chris@87:         kwargs = dict(delimiter=(5, 5, 4), names=True, dtype=None)
Chris@87:         ctrl = np.array([(0, 1, 2.3), (45, 67, 9.)],
Chris@87:                         dtype=[('A', int), ('B', int), ('C', float)])
Chris@87:         test = np.ndfromtxt(TextIO(data), **kwargs)
Chris@87:         assert_equal(test, ctrl)
Chris@87:         #
Chris@87:         kwargs = dict(delimiter=5, names=True, dtype=None)
Chris@87:         ctrl = np.array([(0, 1, 2.3), (45, 67, 9.)],
Chris@87:                         dtype=[('A', int), ('B', int), ('C', float)])
Chris@87:         test = np.ndfromtxt(TextIO(data), **kwargs)
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_filling_values(self):
Chris@87:         "Test missing values"
Chris@87:         data = b"1, 2, 3\n1, , 5\n0, 6, \n"
Chris@87:         kwargs = dict(delimiter=",", dtype=None, filling_values=-999)
Chris@87:         ctrl = np.array([[1, 2, 3], [1, -999, 5], [0, 6, -999]], dtype=int)
Chris@87:         test = np.ndfromtxt(TextIO(data), **kwargs)
Chris@87:         assert_equal(test, ctrl)
Chris@87: 
Chris@87:     def test_comments_is_none(self):
Chris@87:         # Github issue 329 (None was previously being converted to 'None').
Chris@87:         test = np.genfromtxt(TextIO("test1,testNonetherestofthedata"),
Chris@87:                              dtype=None, comments=None, delimiter=',')
Chris@87:         assert_equal(test[1], b'testNonetherestofthedata')
Chris@87:         test = np.genfromtxt(TextIO("test1, testNonetherestofthedata"),
Chris@87:                              dtype=None, comments=None, delimiter=',')
Chris@87:         assert_equal(test[1], b' testNonetherestofthedata')
Chris@87: 
Chris@87:     def test_recfromtxt(self):
Chris@87:         #
Chris@87:         data = TextIO('A,B\n0,1\n2,3')
Chris@87:         kwargs = dict(delimiter=",", missing_values="N/A", names=True)
Chris@87:         test = np.recfromtxt(data, **kwargs)
Chris@87:         control = np.array([(0, 1), (2, 3)],
Chris@87:                            dtype=[('A', np.int), ('B', np.int)])
Chris@87:         self.assertTrue(isinstance(test, np.recarray))
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         data = TextIO('A,B\n0,1\n2,N/A')
Chris@87:         test = np.recfromtxt(data, dtype=None, usemask=True, **kwargs)
Chris@87:         control = ma.array([(0, 1), (2, -1)],
Chris@87:                            mask=[(False, False), (False, True)],
Chris@87:                            dtype=[('A', np.int), ('B', np.int)])
Chris@87:         assert_equal(test, control)
Chris@87:         assert_equal(test.mask, control.mask)
Chris@87:         assert_equal(test.A, [0, 2])
Chris@87: 
Chris@87:     def test_recfromcsv(self):
Chris@87:         #
Chris@87:         data = TextIO('A,B\n0,1\n2,3')
Chris@87:         kwargs = dict(missing_values="N/A", names=True, case_sensitive=True)
Chris@87:         test = np.recfromcsv(data, dtype=None, **kwargs)
Chris@87:         control = np.array([(0, 1), (2, 3)],
Chris@87:                            dtype=[('A', np.int), ('B', np.int)])
Chris@87:         self.assertTrue(isinstance(test, np.recarray))
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         data = TextIO('A,B\n0,1\n2,N/A')
Chris@87:         test = np.recfromcsv(data, dtype=None, usemask=True, **kwargs)
Chris@87:         control = ma.array([(0, 1), (2, -1)],
Chris@87:                            mask=[(False, False), (False, True)],
Chris@87:                            dtype=[('A', np.int), ('B', np.int)])
Chris@87:         assert_equal(test, control)
Chris@87:         assert_equal(test.mask, control.mask)
Chris@87:         assert_equal(test.A, [0, 2])
Chris@87:         #
Chris@87:         data = TextIO('A,B\n0,1\n2,3')
Chris@87:         test = np.recfromcsv(data, missing_values='N/A',)
Chris@87:         control = np.array([(0, 1), (2, 3)],
Chris@87:                            dtype=[('a', np.int), ('b', np.int)])
Chris@87:         self.assertTrue(isinstance(test, np.recarray))
Chris@87:         assert_equal(test, control)
Chris@87:         #
Chris@87:         data = TextIO('A,B\n0,1\n2,3')
Chris@87:         dtype = [('a', np.int), ('b', np.float)]
Chris@87:         test = np.recfromcsv(data, missing_values='N/A', dtype=dtype)
Chris@87:         control = np.array([(0, 1), (2, 3)],
Chris@87:                            dtype=dtype)
Chris@87:         self.assertTrue(isinstance(test, np.recarray))
Chris@87:         assert_equal(test, control)
Chris@87: 
Chris@87:     def test_gft_using_filename(self):
Chris@87:         # Test that we can load data from a filename as well as a file object
Chris@87:         wanted = np.arange(6).reshape((2, 3))
Chris@87:         if sys.version_info[0] >= 3:
Chris@87:             # python 3k is known to fail for '\r'
Chris@87:             linesep = ('\n', '\r\n')
Chris@87:         else:
Chris@87:             linesep = ('\n', '\r\n', '\r')
Chris@87: 
Chris@87:         for sep in linesep:
Chris@87:             data = '0 1 2' + sep + '3 4 5'
Chris@87:             f, name = mkstemp()
Chris@87:             # We can't use NamedTemporaryFile on windows, because we cannot
Chris@87:             # reopen the file.
Chris@87:             try:
Chris@87:                 os.write(f, asbytes(data))
Chris@87:                 assert_array_equal(np.genfromtxt(name), wanted)
Chris@87:             finally:
Chris@87:                 os.close(f)
Chris@87:                 os.unlink(name)
Chris@87: 
Chris@87:     def test_gft_using_generator(self):
Chris@87:         # gft doesn't work with unicode.
Chris@87:         def count():
Chris@87:             for i in range(10):
Chris@87:                 yield asbytes("%d" % i)
Chris@87: 
Chris@87:         res = np.genfromtxt(count())
Chris@87:         assert_array_equal(res, np.arange(10))
Chris@87: 
Chris@87: 
Chris@87: def test_gzip_load():
Chris@87:     a = np.random.random((5, 5))
Chris@87: 
Chris@87:     s = BytesIO()
Chris@87:     f = gzip.GzipFile(fileobj=s, mode="w")
Chris@87: 
Chris@87:     np.save(f, a)
Chris@87:     f.close()
Chris@87:     s.seek(0)
Chris@87: 
Chris@87:     f = gzip.GzipFile(fileobj=s, mode="r")
Chris@87:     assert_array_equal(np.load(f), a)
Chris@87: 
Chris@87: 
Chris@87: def test_gzip_loadtxt():
Chris@87:     # Thanks to another windows brokeness, we can't use
Chris@87:     # NamedTemporaryFile: a file created from this function cannot be
Chris@87:     # reopened by another open call. So we first put the gzipped string
Chris@87:     # of the test reference array, write it to a securely opened file,
Chris@87:     # which is then read from by the loadtxt function
Chris@87:     s = BytesIO()
Chris@87:     g = gzip.GzipFile(fileobj=s, mode='w')
Chris@87:     g.write(b'1 2 3\n')
Chris@87:     g.close()
Chris@87:     s.seek(0)
Chris@87: 
Chris@87:     f, name = mkstemp(suffix='.gz')
Chris@87:     try:
Chris@87:         os.write(f, s.read())
Chris@87:         s.close()
Chris@87:         assert_array_equal(np.loadtxt(name), [1, 2, 3])
Chris@87:     finally:
Chris@87:         os.close(f)
Chris@87:         os.unlink(name)
Chris@87: 
Chris@87: 
Chris@87: def test_gzip_loadtxt_from_string():
Chris@87:     s = BytesIO()
Chris@87:     f = gzip.GzipFile(fileobj=s, mode="w")
Chris@87:     f.write(b'1 2 3\n')
Chris@87:     f.close()
Chris@87:     s.seek(0)
Chris@87: 
Chris@87:     f = gzip.GzipFile(fileobj=s, mode="r")
Chris@87:     assert_array_equal(np.loadtxt(f), [1, 2, 3])
Chris@87: 
Chris@87: 
Chris@87: def test_npzfile_dict():
Chris@87:     s = BytesIO()
Chris@87:     x = np.zeros((3, 3))
Chris@87:     y = np.zeros((3, 3))
Chris@87: 
Chris@87:     np.savez(s, x=x, y=y)
Chris@87:     s.seek(0)
Chris@87: 
Chris@87:     z = np.load(s)
Chris@87: 
Chris@87:     assert_('x' in z)
Chris@87:     assert_('y' in z)
Chris@87:     assert_('x' in z.keys())
Chris@87:     assert_('y' in z.keys())
Chris@87: 
Chris@87:     for f, a in z.items():
Chris@87:         assert_(f in ['x', 'y'])
Chris@87:         assert_equal(a.shape, (3, 3))
Chris@87: 
Chris@87:     assert_(len(z.items()) == 2)
Chris@87: 
Chris@87:     for f in z:
Chris@87:         assert_(f in ['x', 'y'])
Chris@87: 
Chris@87:     assert_('x' in z.keys())
Chris@87: 
Chris@87: 
Chris@87: def test_load_refcount():
Chris@87:     # Check that objects returned by np.load are directly freed based on
Chris@87:     # their refcount, rather than needing the gc to collect them.
Chris@87: 
Chris@87:     f = BytesIO()
Chris@87:     np.savez(f, [1, 2, 3])
Chris@87:     f.seek(0)
Chris@87: 
Chris@87:     gc.collect()
Chris@87:     n_before = len(gc.get_objects())
Chris@87:     np.load(f)
Chris@87:     n_after = len(gc.get_objects())
Chris@87: 
Chris@87:     assert_equal(n_before, n_after)
Chris@87: 
Chris@87: if __name__ == "__main__":
Chris@87:     run_module_suite()