Mercurial > hg > vamp-build-and-test

comparison DEPENDENCIES/mingw32/Python27/Lib/site-packages/numpy/lib/tests/test_io.py @ 87:2a2c65a20a8b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add Python libs and headers
author Chris Cannam
date Wed, 25 Feb 2015 14:05:22 +0000
parents
children
comparison
equal deleted inserted replaced
86:413a9d26189e 87:2a2c65a20a8b
1 from __future__ import division, absolute_import, print_function
2
3 import sys
4 import gzip
5 import os
6 import threading
7 from tempfile import mkstemp, NamedTemporaryFile
8 import time
9 import warnings
10 import gc
11 from io import BytesIO
12 from datetime import datetime
13
14 import numpy as np
15 import numpy.ma as ma
16 from numpy.lib._iotools import (ConverterError, ConverterLockError,
17 ConversionWarning)
18 from numpy.compat import asbytes, asbytes_nested, bytes, asstr
19 from nose import SkipTest
20 from numpy.ma.testutils import (
21 TestCase, assert_equal, assert_array_equal,
22 assert_raises, assert_raises_regex, run_module_suite
23 )
24 from numpy.testing import assert_warns, assert_, build_err_msg
25 from numpy.testing.utils import tempdir
26
27
28 class TextIO(BytesIO):
29 """Helper IO class.
30
31 Writes encode strings to bytes if needed, reads return bytes.
32 This makes it easier to emulate files opened in binary mode
33 without needing to explicitly convert strings to bytes in
34 setting up the test data.
35
36 """
37 def __init__(self, s=""):
38 BytesIO.__init__(self, asbytes(s))
39
40 def write(self, s):
41 BytesIO.write(self, asbytes(s))
42
43 def writelines(self, lines):
44 BytesIO.writelines(self, [asbytes(s) for s in lines])
45
46
47 MAJVER, MINVER = sys.version_info[:2]
48 IS_64BIT = sys.maxsize > 2**32
49
50
51 def strptime(s, fmt=None):
52 """This function is available in the datetime module only
53 from Python >= 2.5.
54
55 """
56 if sys.version_info[0] >= 3:
57 return datetime(*time.strptime(s.decode('latin1'), fmt)[:3])
58 else:
59 return datetime(*time.strptime(s, fmt)[:3])
60
61
62 class RoundtripTest(object):
63 def roundtrip(self, save_func, *args, **kwargs):
64 """
65 save_func : callable
66 Function used to save arrays to file.
67 file_on_disk : bool
68 If true, store the file on disk, instead of in a
69 string buffer.
70 save_kwds : dict
71 Parameters passed to `save_func`.
72 load_kwds : dict
73 Parameters passed to `numpy.load`.
74 args : tuple of arrays
75 Arrays stored to file.
76
77 """
78 save_kwds = kwargs.get('save_kwds', {})
79 load_kwds = kwargs.get('load_kwds', {})
80 file_on_disk = kwargs.get('file_on_disk', False)
81
82 if file_on_disk:
83 target_file = NamedTemporaryFile(delete=False)
84 load_file = target_file.name
85 else:
86 target_file = BytesIO()
87 load_file = target_file
88
89 try:
90 arr = args
91
92 save_func(target_file, *arr, **save_kwds)
93 target_file.flush()
94 target_file.seek(0)
95
96 if sys.platform == 'win32' and not isinstance(target_file, BytesIO):
97 target_file.close()
98
99 arr_reloaded = np.load(load_file, **load_kwds)
100
101 self.arr = arr
102 self.arr_reloaded = arr_reloaded
103 finally:
104 if not isinstance(target_file, BytesIO):
105 target_file.close()
106 # holds an open file descriptor so it can't be deleted on win
107 if not isinstance(arr_reloaded, np.lib.npyio.NpzFile):
108 os.remove(target_file.name)
109
110 def check_roundtrips(self, a):
111 self.roundtrip(a)
112 self.roundtrip(a, file_on_disk=True)
113 self.roundtrip(np.asfortranarray(a))
114 self.roundtrip(np.asfortranarray(a), file_on_disk=True)
115 if a.shape[0] > 1:
116 # neither C nor Fortran contiguous for 2D arrays or more
117 self.roundtrip(np.asfortranarray(a)[1:])
118 self.roundtrip(np.asfortranarray(a)[1:], file_on_disk=True)
119
120 def test_array(self):
121 a = np.array([], float)
122 self.check_roundtrips(a)
123
124 a = np.array([[1, 2], [3, 4]], float)
125 self.check_roundtrips(a)
126
127 a = np.array([[1, 2], [3, 4]], int)
128 self.check_roundtrips(a)
129
130 a = np.array([[1 + 5j, 2 + 6j], [3 + 7j, 4 + 8j]], dtype=np.csingle)
131 self.check_roundtrips(a)
132
133 a = np.array([[1 + 5j, 2 + 6j], [3 + 7j, 4 + 8j]], dtype=np.cdouble)
134 self.check_roundtrips(a)
135
136 def test_array_object(self):
137 if sys.version_info[:2] >= (2, 7):
138 a = np.array([], object)
139 self.check_roundtrips(a)
140
141 a = np.array([[1, 2], [3, 4]], object)
142 self.check_roundtrips(a)
143 # Fails with UnpicklingError: could not find MARK on Python 2.6
144
145 def test_1D(self):
146 a = np.array([1, 2, 3, 4], int)
147 self.roundtrip(a)
148
149 @np.testing.dec.knownfailureif(sys.platform == 'win32', "Fail on Win32")
150 def test_mmap(self):
151 a = np.array([[1, 2.5], [4, 7.3]])
152 self.roundtrip(a, file_on_disk=True, load_kwds={'mmap_mode': 'r'})
153
154 a = np.asfortranarray([[1, 2.5], [4, 7.3]])
155 self.roundtrip(a, file_on_disk=True, load_kwds={'mmap_mode': 'r'})
156
157 def test_record(self):
158 a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
159 self.check_roundtrips(a)
160
161 def test_format_2_0(self):
162 dt = [(("%d" % i) * 100, float) for i in range(500)]
163 a = np.ones(1000, dtype=dt)
164 with warnings.catch_warnings(record=True):
165 warnings.filterwarnings('always', '', UserWarning)
166 self.check_roundtrips(a)
167
168
169 class TestSaveLoad(RoundtripTest, TestCase):
170 def roundtrip(self, *args, **kwargs):
171 RoundtripTest.roundtrip(self, np.save, *args, **kwargs)
172 assert_equal(self.arr[0], self.arr_reloaded)
173 assert_equal(self.arr[0].dtype, self.arr_reloaded.dtype)
174 assert_equal(self.arr[0].flags.fnc, self.arr_reloaded.flags.fnc)
175
176
177 class TestSavezLoad(RoundtripTest, TestCase):
178 def roundtrip(self, *args, **kwargs):
179 RoundtripTest.roundtrip(self, np.savez, *args, **kwargs)
180 try:
181 for n, arr in enumerate(self.arr):
182 reloaded = self.arr_reloaded['arr_%d' % n]
183 assert_equal(arr, reloaded)
184 assert_equal(arr.dtype, reloaded.dtype)
185 assert_equal(arr.flags.fnc, reloaded.flags.fnc)
186 finally:
187 # delete tempfile, must be done here on windows
188 if self.arr_reloaded.fid:
189 self.arr_reloaded.fid.close()
190 os.remove(self.arr_reloaded.fid.name)
191
192 @np.testing.dec.skipif(not IS_64BIT, "Works only with 64bit systems")
193 @np.testing.dec.slow
194 def test_big_arrays(self):
195 L = (1 << 31) + 100000
196 a = np.empty(L, dtype=np.uint8)
197 with tempdir(prefix="numpy_test_big_arrays_") as tmpdir:
198 tmp = os.path.join(tmpdir, "file.npz")
199 np.savez(tmp, a=a)
200 del a
201 npfile = np.load(tmp)
202 a = npfile['a']
203 npfile.close()
204
205 def test_multiple_arrays(self):
206 a = np.array([[1, 2], [3, 4]], float)
207 b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex)
208 self.roundtrip(a, b)
209
210 def test_named_arrays(self):
211 a = np.array([[1, 2], [3, 4]], float)
212 b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex)
213 c = BytesIO()
214 np.savez(c, file_a=a, file_b=b)
215 c.seek(0)
216 l = np.load(c)
217 assert_equal(a, l['file_a'])
218 assert_equal(b, l['file_b'])
219
220 def test_savez_filename_clashes(self):
221 # Test that issue #852 is fixed
222 # and savez functions in multithreaded environment
223
224 def writer(error_list):
225 fd, tmp = mkstemp(suffix='.npz')
226 os.close(fd)
227 try:
228 arr = np.random.randn(500, 500)
229 try:
230 np.savez(tmp, arr=arr)
231 except OSError as err:
232 error_list.append(err)
233 finally:
234 os.remove(tmp)
235
236 errors = []
237 threads = [threading.Thread(target=writer, args=(errors,))
238 for j in range(3)]
239 for t in threads:
240 t.start()
241 for t in threads:
242 t.join()
243
244 if errors:
245 raise AssertionError(errors)
246
247 def test_not_closing_opened_fid(self):
248 # Test that issue #2178 is fixed:
249 # verify could seek on 'loaded' file
250
251 fd, tmp = mkstemp(suffix='.npz')
252 os.close(fd)
253 try:
254 fp = open(tmp, 'wb')
255 np.savez(fp, data='LOVELY LOAD')
256 fp.close()
257
258 fp = open(tmp, 'rb', 10000)
259 fp.seek(0)
260 assert_(not fp.closed)
261 _ = np.load(fp)['data']
262 assert_(not fp.closed)
263 # must not get closed by .load(opened fp)
264 fp.seek(0)
265 assert_(not fp.closed)
266
267 finally:
268 fp.close()
269 os.remove(tmp)
270
271 def test_closing_fid(self):
272 # Test that issue #1517 (too many opened files) remains closed
273 # It might be a "weak" test since failed to get triggered on
274 # e.g. Debian sid of 2012 Jul 05 but was reported to
275 # trigger the failure on Ubuntu 10.04:
276 # http://projects.scipy.org/numpy/ticket/1517#comment:2
277 fd, tmp = mkstemp(suffix='.npz')
278 os.close(fd)
279
280 try:
281 fp = open(tmp, 'wb')
282 np.savez(fp, data='LOVELY LOAD')
283 fp.close()
284 # We need to check if the garbage collector can properly close
285 # numpy npz file returned by np.load when their reference count
286 # goes to zero. Python 3 running in debug mode raises a
287 # ResourceWarning when file closing is left to the garbage
288 # collector, so we catch the warnings. Because ResourceWarning
289 # is unknown in Python < 3.x, we take the easy way out and
290 # catch all warnings.
291 with warnings.catch_warnings():
292 warnings.simplefilter("ignore")
293 for i in range(1, 1025):
294 try:
295 np.load(tmp)["data"]
296 except Exception as e:
297 msg = "Failed to load data from a file: %s" % e
298 raise AssertionError(msg)
299 finally:
300 os.remove(tmp)
301
302 def test_closing_zipfile_after_load(self):
303 # Check that zipfile owns file and can close it.
304 # This needs to pass a file name to load for the
305 # test.
306 with tempdir(prefix="numpy_test_closing_zipfile_after_load_") as tmpdir:
307 fd, tmp = mkstemp(suffix='.npz', dir=tmpdir)
308 os.close(fd)
309 np.savez(tmp, lab='place holder')
310 data = np.load(tmp)
311 fp = data.zip.fp
312 data.close()
313 assert_(fp.closed)
314
315
316 class TestSaveTxt(TestCase):
317 def test_array(self):
318 a = np.array([[1, 2], [3, 4]], float)
319 fmt = "%.18e"
320 c = BytesIO()
321 np.savetxt(c, a, fmt=fmt)
322 c.seek(0)
323 assert_equal(c.readlines(),
324 [asbytes((fmt + ' ' + fmt + '\n') % (1, 2)),
325 asbytes((fmt + ' ' + fmt + '\n') % (3, 4))])
326
327 a = np.array([[1, 2], [3, 4]], int)
328 c = BytesIO()
329 np.savetxt(c, a, fmt='%d')
330 c.seek(0)
331 assert_equal(c.readlines(), [b'1 2\n', b'3 4\n'])
332
333 def test_1D(self):
334 a = np.array([1, 2, 3, 4], int)
335 c = BytesIO()
336 np.savetxt(c, a, fmt='%d')
337 c.seek(0)
338 lines = c.readlines()
339 assert_equal(lines, [b'1\n', b'2\n', b'3\n', b'4\n'])
340
341 def test_record(self):
342 a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
343 c = BytesIO()
344 np.savetxt(c, a, fmt='%d')
345 c.seek(0)
346 assert_equal(c.readlines(), [b'1 2\n', b'3 4\n'])
347
348 def test_delimiter(self):
349 a = np.array([[1., 2.], [3., 4.]])
350 c = BytesIO()
351 np.savetxt(c, a, delimiter=',', fmt='%d')
352 c.seek(0)
353 assert_equal(c.readlines(), [b'1,2\n', b'3,4\n'])
354
355 def test_format(self):
356 a = np.array([(1, 2), (3, 4)])
357 c = BytesIO()
358 # Sequence of formats
359 np.savetxt(c, a, fmt=['%02d', '%3.1f'])
360 c.seek(0)
361 assert_equal(c.readlines(), [b'01 2.0\n', b'03 4.0\n'])
362
363 # A single multiformat string
364 c = BytesIO()
365 np.savetxt(c, a, fmt='%02d : %3.1f')
366 c.seek(0)
367 lines = c.readlines()
368 assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n'])
369
370 # Specify delimiter, should be overiden
371 c = BytesIO()
372 np.savetxt(c, a, fmt='%02d : %3.1f', delimiter=',')
373 c.seek(0)
374 lines = c.readlines()
375 assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n'])
376
377 # Bad fmt, should raise a ValueError
378 c = BytesIO()
379 assert_raises(ValueError, np.savetxt, c, a, fmt=99)
380
381 def test_header_footer(self):
382 """
383 Test the functionality of the header and footer keyword argument.
384 """
385 c = BytesIO()
386 a = np.array([(1, 2), (3, 4)], dtype=np.int)
387 test_header_footer = 'Test header / footer'
388 # Test the header keyword argument
389 np.savetxt(c, a, fmt='%1d', header=test_header_footer)
390 c.seek(0)
391 assert_equal(c.read(),
392 asbytes('# ' + test_header_footer + '\n1 2\n3 4\n'))
393 # Test the footer keyword argument
394 c = BytesIO()
395 np.savetxt(c, a, fmt='%1d', footer=test_header_footer)
396 c.seek(0)
397 assert_equal(c.read(),
398 asbytes('1 2\n3 4\n# ' + test_header_footer + '\n'))
399 # Test the commentstr keyword argument used on the header
400 c = BytesIO()
401 commentstr = '% '
402 np.savetxt(c, a, fmt='%1d',
403 header=test_header_footer, comments=commentstr)
404 c.seek(0)
405 assert_equal(c.read(),
406 asbytes(commentstr + test_header_footer + '\n' + '1 2\n3 4\n'))
407 # Test the commentstr keyword argument used on the footer
408 c = BytesIO()
409 commentstr = '% '
410 np.savetxt(c, a, fmt='%1d',
411 footer=test_header_footer, comments=commentstr)
412 c.seek(0)
413 assert_equal(c.read(),
414 asbytes('1 2\n3 4\n' + commentstr + test_header_footer + '\n'))
415
416 def test_file_roundtrip(self):
417 f, name = mkstemp()
418 os.close(f)
419 try:
420 a = np.array([(1, 2), (3, 4)])
421 np.savetxt(name, a)
422 b = np.loadtxt(name)
423 assert_array_equal(a, b)
424 finally:
425 os.unlink(name)
426
427 def test_complex_arrays(self):
428 ncols = 2
429 nrows = 2
430 a = np.zeros((ncols, nrows), dtype=np.complex128)
431 re = np.pi
432 im = np.e
433 a[:] = re + 1.0j * im
434
435 # One format only
436 c = BytesIO()
437 np.savetxt(c, a, fmt=' %+.3e')
438 c.seek(0)
439 lines = c.readlines()
440 assert_equal(
441 lines,
442 [b' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n',
443 b' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n'])
444
445 # One format for each real and imaginary part
446 c = BytesIO()
447 np.savetxt(c, a, fmt=' %+.3e' * 2 * ncols)
448 c.seek(0)
449 lines = c.readlines()
450 assert_equal(
451 lines,
452 [b' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n',
453 b' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n'])
454
455 # One format for each complex number
456 c = BytesIO()
457 np.savetxt(c, a, fmt=['(%.3e%+.3ej)'] * ncols)
458 c.seek(0)
459 lines = c.readlines()
460 assert_equal(
461 lines,
462 [b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n',
463 b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n'])
464
465 def test_custom_writer(self):
466
467 class CustomWriter(list):
468 def write(self, text):
469 self.extend(text.split(b'\n'))
470
471 w = CustomWriter()
472 a = np.array([(1, 2), (3, 4)])
473 np.savetxt(w, a)
474 b = np.loadtxt(w)
475 assert_array_equal(a, b)
476
477
478 class TestLoadTxt(TestCase):
479 def test_record(self):
480 c = TextIO()
481 c.write('1 2\n3 4')
482 c.seek(0)
483 x = np.loadtxt(c, dtype=[('x', np.int32), ('y', np.int32)])
484 a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
485 assert_array_equal(x, a)
486
487 d = TextIO()
488 d.write('M 64.0 75.0\nF 25.0 60.0')
489 d.seek(0)
490 mydescriptor = {'names': ('gender', 'age', 'weight'),
491 'formats': ('S1', 'i4', 'f4')}
492 b = np.array([('M', 64.0, 75.0),
493 ('F', 25.0, 60.0)], dtype=mydescriptor)
494 y = np.loadtxt(d, dtype=mydescriptor)
495 assert_array_equal(y, b)
496
497 def test_array(self):
498 c = TextIO()
499 c.write('1 2\n3 4')
500
501 c.seek(0)
502 x = np.loadtxt(c, dtype=np.int)
503 a = np.array([[1, 2], [3, 4]], int)
504 assert_array_equal(x, a)
505
506 c.seek(0)
507 x = np.loadtxt(c, dtype=float)
508 a = np.array([[1, 2], [3, 4]], float)
509 assert_array_equal(x, a)
510
511 def test_1D(self):
512 c = TextIO()
513 c.write('1\n2\n3\n4\n')
514 c.seek(0)
515 x = np.loadtxt(c, dtype=int)
516 a = np.array([1, 2, 3, 4], int)
517 assert_array_equal(x, a)
518
519 c = TextIO()
520 c.write('1,2,3,4\n')
521 c.seek(0)
522 x = np.loadtxt(c, dtype=int, delimiter=',')
523 a = np.array([1, 2, 3, 4], int)
524 assert_array_equal(x, a)
525
526 def test_missing(self):
527 c = TextIO()
528 c.write('1,2,3,,5\n')
529 c.seek(0)
530 x = np.loadtxt(c, dtype=int, delimiter=',',
531 converters={3: lambda s: int(s or - 999)})
532 a = np.array([1, 2, 3, -999, 5], int)
533 assert_array_equal(x, a)
534
535 def test_converters_with_usecols(self):
536 c = TextIO()
537 c.write('1,2,3,,5\n6,7,8,9,10\n')
538 c.seek(0)
539 x = np.loadtxt(c, dtype=int, delimiter=',',
540 converters={3: lambda s: int(s or - 999)},
541 usecols=(1, 3,))
542 a = np.array([[2, -999], [7, 9]], int)
543 assert_array_equal(x, a)
544
545 def test_comments(self):
546 c = TextIO()
547 c.write('# comment\n1,2,3,5\n')
548 c.seek(0)
549 x = np.loadtxt(c, dtype=int, delimiter=',',
550 comments='#')
551 a = np.array([1, 2, 3, 5], int)
552 assert_array_equal(x, a)
553
554 def test_skiprows(self):
555 c = TextIO()
556 c.write('comment\n1,2,3,5\n')
557 c.seek(0)
558 x = np.loadtxt(c, dtype=int, delimiter=',',
559 skiprows=1)
560 a = np.array([1, 2, 3, 5], int)
561 assert_array_equal(x, a)
562
563 c = TextIO()
564 c.write('# comment\n1,2,3,5\n')
565 c.seek(0)
566 x = np.loadtxt(c, dtype=int, delimiter=',',
567 skiprows=1)
568 a = np.array([1, 2, 3, 5], int)
569 assert_array_equal(x, a)
570
571 def test_usecols(self):
572 a = np.array([[1, 2], [3, 4]], float)
573 c = BytesIO()
574 np.savetxt(c, a)
575 c.seek(0)
576 x = np.loadtxt(c, dtype=float, usecols=(1,))
577 assert_array_equal(x, a[:, 1])
578
579 a = np.array([[1, 2, 3], [3, 4, 5]], float)
580 c = BytesIO()
581 np.savetxt(c, a)
582 c.seek(0)
583 x = np.loadtxt(c, dtype=float, usecols=(1, 2))
584 assert_array_equal(x, a[:, 1:])
585
586 # Testing with arrays instead of tuples.
587 c.seek(0)
588 x = np.loadtxt(c, dtype=float, usecols=np.array([1, 2]))
589 assert_array_equal(x, a[:, 1:])
590
591 # Checking with dtypes defined converters.
592 data = '''JOE 70.1 25.3
593 BOB 60.5 27.9
594 '''
595 c = TextIO(data)
596 names = ['stid', 'temp']
597 dtypes = ['S4', 'f8']
598 arr = np.loadtxt(c, usecols=(0, 2), dtype=list(zip(names, dtypes)))
599 assert_equal(arr['stid'], [b"JOE", b"BOB"])
600 assert_equal(arr['temp'], [25.3, 27.9])
601
602 def test_fancy_dtype(self):
603 c = TextIO()
604 c.write('1,2,3.0\n4,5,6.0\n')
605 c.seek(0)
606 dt = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
607 x = np.loadtxt(c, dtype=dt, delimiter=',')
608 a = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dt)
609 assert_array_equal(x, a)
610
611 def test_shaped_dtype(self):
612 c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6")
613 dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
614 ('block', int, (2, 3))])
615 x = np.loadtxt(c, dtype=dt)
616 a = np.array([('aaaa', 1.0, 8.0, [[1, 2, 3], [4, 5, 6]])],
617 dtype=dt)
618 assert_array_equal(x, a)
619
620 def test_3d_shaped_dtype(self):
621 c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12")
622 dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
623 ('block', int, (2, 2, 3))])
624 x = np.loadtxt(c, dtype=dt)
625 a = np.array([('aaaa', 1.0, 8.0,
626 [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])],
627 dtype=dt)
628 assert_array_equal(x, a)
629
630 def test_empty_file(self):
631 with warnings.catch_warnings():
632 warnings.filterwarnings("ignore",
633 message="loadtxt: Empty input file:")
634 c = TextIO()
635 x = np.loadtxt(c)
636 assert_equal(x.shape, (0,))
637 x = np.loadtxt(c, dtype=np.int64)
638 assert_equal(x.shape, (0,))
639 assert_(x.dtype == np.int64)
640
641 def test_unused_converter(self):
642 c = TextIO()
643 c.writelines(['1 21\n', '3 42\n'])
644 c.seek(0)
645 data = np.loadtxt(c, usecols=(1,),
646 converters={0: lambda s: int(s, 16)})
647 assert_array_equal(data, [21, 42])
648
649 c.seek(0)
650 data = np.loadtxt(c, usecols=(1,),
651 converters={1: lambda s: int(s, 16)})
652 assert_array_equal(data, [33, 66])
653
654 def test_dtype_with_object(self):
655 "Test using an explicit dtype with an object"
656 from datetime import date
657 import time
658 data = """ 1; 2001-01-01
659 2; 2002-01-31 """
660 ndtype = [('idx', int), ('code', np.object)]
661 func = lambda s: strptime(s.strip(), "%Y-%m-%d")
662 converters = {1: func}
663 test = np.loadtxt(TextIO(data), delimiter=";", dtype=ndtype,
664 converters=converters)
665 control = np.array(
666 [(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
667 dtype=ndtype)
668 assert_equal(test, control)
669
670 def test_uint64_type(self):
671 tgt = (9223372043271415339, 9223372043271415853)
672 c = TextIO()
673 c.write("%s %s" % tgt)
674 c.seek(0)
675 res = np.loadtxt(c, dtype=np.uint64)
676 assert_equal(res, tgt)
677
678 def test_int64_type(self):
679 tgt = (-9223372036854775807, 9223372036854775807)
680 c = TextIO()
681 c.write("%s %s" % tgt)
682 c.seek(0)
683 res = np.loadtxt(c, dtype=np.int64)
684 assert_equal(res, tgt)
685
686 def test_universal_newline(self):
687 f, name = mkstemp()
688 os.write(f, b'1 21\r3 42\r')
689 os.close(f)
690
691 try:
692 data = np.loadtxt(name)
693 assert_array_equal(data, [[1, 21], [3, 42]])
694 finally:
695 os.unlink(name)
696
697 def test_empty_field_after_tab(self):
698 c = TextIO()
699 c.write('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t')
700 c.seek(0)
701 dt = {'names': ('x', 'y', 'z', 'comment'),
702 'formats': ('<i4', '<i4', '<f4', '|S8')}
703 x = np.loadtxt(c, dtype=dt, delimiter='\t')
704 a = np.array([b'start ', b' ', b''])
705 assert_array_equal(x['comment'], a)
706
707 def test_structure_unpack(self):
708 txt = TextIO("M 21 72\nF 35 58")
709 dt = {'names': ('a', 'b', 'c'), 'formats': ('|S1', '<i4', '<f4')}
710 a, b, c = np.loadtxt(txt, dtype=dt, unpack=True)
711 assert_(a.dtype.str == '|S1')
712 assert_(b.dtype.str == '<i4')
713 assert_(c.dtype.str == '<f4')
714 assert_array_equal(a, np.array([b'M', b'F']))
715 assert_array_equal(b, np.array([21, 35]))
716 assert_array_equal(c, np.array([72., 58.]))
717
718 def test_ndmin_keyword(self):
719 c = TextIO()
720 c.write('1,2,3\n4,5,6')
721 c.seek(0)
722 assert_raises(ValueError, np.loadtxt, c, ndmin=3)
723 c.seek(0)
724 assert_raises(ValueError, np.loadtxt, c, ndmin=1.5)
725 c.seek(0)
726 x = np.loadtxt(c, dtype=int, delimiter=',', ndmin=1)
727 a = np.array([[1, 2, 3], [4, 5, 6]])
728 assert_array_equal(x, a)
729
730 d = TextIO()
731 d.write('0,1,2')
732 d.seek(0)
733 x = np.loadtxt(d, dtype=int, delimiter=',', ndmin=2)
734 assert_(x.shape == (1, 3))
735 d.seek(0)
736 x = np.loadtxt(d, dtype=int, delimiter=',', ndmin=1)
737 assert_(x.shape == (3,))
738 d.seek(0)
739 x = np.loadtxt(d, dtype=int, delimiter=',', ndmin=0)
740 assert_(x.shape == (3,))
741
742 e = TextIO()
743 e.write('0\n1\n2')
744 e.seek(0)
745 x = np.loadtxt(e, dtype=int, delimiter=',', ndmin=2)
746 assert_(x.shape == (3, 1))
747 e.seek(0)
748 x = np.loadtxt(e, dtype=int, delimiter=',', ndmin=1)
749 assert_(x.shape == (3,))
750 e.seek(0)
751 x = np.loadtxt(e, dtype=int, delimiter=',', ndmin=0)
752 assert_(x.shape == (3,))
753
754 # Test ndmin kw with empty file.
755 with warnings.catch_warnings():
756 warnings.filterwarnings("ignore",
757 message="loadtxt: Empty input file:")
758 f = TextIO()
759 assert_(np.loadtxt(f, ndmin=2).shape == (0, 1,))
760 assert_(np.loadtxt(f, ndmin=1).shape == (0,))
761
762 def test_generator_source(self):
763 def count():
764 for i in range(10):
765 yield "%d" % i
766
767 res = np.loadtxt(count())
768 assert_array_equal(res, np.arange(10))
769
770 def test_bad_line(self):
771 c = TextIO()
772 c.write('1 2 3\n4 5 6\n2 3')
773 c.seek(0)
774
775 # Check for exception and that exception contains line number
776 assert_raises_regex(ValueError, "3", np.loadtxt, c)
777
778
779 class Testfromregex(TestCase):
780 # np.fromregex expects files opened in binary mode.
781 def test_record(self):
782 c = TextIO()
783 c.write('1.312 foo\n1.534 bar\n4.444 qux')
784 c.seek(0)
785
786 dt = [('num', np.float64), ('val', 'S3')]
787 x = np.fromregex(c, r"([0-9.]+)\s+(...)", dt)
788 a = np.array([(1.312, 'foo'), (1.534, 'bar'), (4.444, 'qux')],
789 dtype=dt)
790 assert_array_equal(x, a)
791
792 def test_record_2(self):
793 c = TextIO()
794 c.write('1312 foo\n1534 bar\n4444 qux')
795 c.seek(0)
796
797 dt = [('num', np.int32), ('val', 'S3')]
798 x = np.fromregex(c, r"(\d+)\s+(...)", dt)
799 a = np.array([(1312, 'foo'), (1534, 'bar'), (4444, 'qux')],
800 dtype=dt)
801 assert_array_equal(x, a)
802
803 def test_record_3(self):
804 c = TextIO()
805 c.write('1312 foo\n1534 bar\n4444 qux')
806 c.seek(0)
807
808 dt = [('num', np.float64)]
809 x = np.fromregex(c, r"(\d+)\s+...", dt)
810 a = np.array([(1312,), (1534,), (4444,)], dtype=dt)
811 assert_array_equal(x, a)
812
813
814 #####--------------------------------------------------------------------------
815
816
817 class TestFromTxt(TestCase):
818 #
819 def test_record(self):
820 "Test w/ explicit dtype"
821 data = TextIO('1 2\n3 4')
822 # data.seek(0)
823 test = np.ndfromtxt(data, dtype=[('x', np.int32), ('y', np.int32)])
824 control = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
825 assert_equal(test, control)
826 #
827 data = TextIO('M 64.0 75.0\nF 25.0 60.0')
828 # data.seek(0)
829 descriptor = {'names': ('gender', 'age', 'weight'),
830 'formats': ('S1', 'i4', 'f4')}
831 control = np.array([('M', 64.0, 75.0), ('F', 25.0, 60.0)],
832 dtype=descriptor)
833 test = np.ndfromtxt(data, dtype=descriptor)
834 assert_equal(test, control)
835
836 def test_array(self):
837 "Test outputing a standard ndarray"
838 data = TextIO('1 2\n3 4')
839 control = np.array([[1, 2], [3, 4]], dtype=int)
840 test = np.ndfromtxt(data, dtype=int)
841 assert_array_equal(test, control)
842 #
843 data.seek(0)
844 control = np.array([[1, 2], [3, 4]], dtype=float)
845 test = np.loadtxt(data, dtype=float)
846 assert_array_equal(test, control)
847
848 def test_1D(self):
849 "Test squeezing to 1D"
850 control = np.array([1, 2, 3, 4], int)
851 #
852 data = TextIO('1\n2\n3\n4\n')
853 test = np.ndfromtxt(data, dtype=int)
854 assert_array_equal(test, control)
855 #
856 data = TextIO('1,2,3,4\n')
857 test = np.ndfromtxt(data, dtype=int, delimiter=',')
858 assert_array_equal(test, control)
859
860 def test_comments(self):
861 "Test the stripping of comments"
862 control = np.array([1, 2, 3, 5], int)
863 # Comment on its own line
864 data = TextIO('# comment\n1,2,3,5\n')
865 test = np.ndfromtxt(data, dtype=int, delimiter=',', comments='#')
866 assert_equal(test, control)
867 # Comment at the end of a line
868 data = TextIO('1,2,3,5# comment\n')
869 test = np.ndfromtxt(data, dtype=int, delimiter=',', comments='#')
870 assert_equal(test, control)
871
872 def test_skiprows(self):
873 "Test row skipping"
874 control = np.array([1, 2, 3, 5], int)
875 kwargs = dict(dtype=int, delimiter=',')
876 #
877 data = TextIO('comment\n1,2,3,5\n')
878 test = np.ndfromtxt(data, skip_header=1, **kwargs)
879 assert_equal(test, control)
880 #
881 data = TextIO('# comment\n1,2,3,5\n')
882 test = np.loadtxt(data, skiprows=1, **kwargs)
883 assert_equal(test, control)
884
885 def test_skip_footer(self):
886 data = ["# %i" % i for i in range(1, 6)]
887 data.append("A, B, C")
888 data.extend(["%i,%3.1f,%03s" % (i, i, i) for i in range(51)])
889 data[-1] = "99,99"
890 kwargs = dict(delimiter=",", names=True, skip_header=5, skip_footer=10)
891 test = np.genfromtxt(TextIO("\n".join(data)), **kwargs)
892 ctrl = np.array([("%f" % i, "%f" % i, "%f" % i) for i in range(41)],
893 dtype=[(_, float) for _ in "ABC"])
894 assert_equal(test, ctrl)
895
896 def test_skip_footer_with_invalid(self):
897 with warnings.catch_warnings():
898 warnings.filterwarnings("ignore")
899 basestr = '1 1\n2 2\n3 3\n4 4\n5 \n6 \n7 \n'
900 # Footer too small to get rid of all invalid values
901 assert_raises(ValueError, np.genfromtxt,
902 TextIO(basestr), skip_footer=1)
903 # except ValueError:
904 # pass
905 a = np.genfromtxt(
906 TextIO(basestr), skip_footer=1, invalid_raise=False)
907 assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
908 #
909 a = np.genfromtxt(TextIO(basestr), skip_footer=3)
910 assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
911 #
912 basestr = '1 1\n2 \n3 3\n4 4\n5 \n6 6\n7 7\n'
913 a = np.genfromtxt(
914 TextIO(basestr), skip_footer=1, invalid_raise=False)
915 assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.], [6., 6.]]))
916 a = np.genfromtxt(
917 TextIO(basestr), skip_footer=3, invalid_raise=False)
918 assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.]]))
919
920 def test_header(self):
921 "Test retrieving a header"
922 data = TextIO('gender age weight\nM 64.0 75.0\nF 25.0 60.0')
923 test = np.ndfromtxt(data, dtype=None, names=True)
924 control = {'gender': np.array([b'M', b'F']),
925 'age': np.array([64.0, 25.0]),
926 'weight': np.array([75.0, 60.0])}
927 assert_equal(test['gender'], control['gender'])
928 assert_equal(test['age'], control['age'])
929 assert_equal(test['weight'], control['weight'])
930
931 def test_auto_dtype(self):
932 "Test the automatic definition of the output dtype"
933 data = TextIO('A 64 75.0 3+4j True\nBCD 25 60.0 5+6j False')
934 test = np.ndfromtxt(data, dtype=None)
935 control = [np.array([b'A', b'BCD']),
936 np.array([64, 25]),
937 np.array([75.0, 60.0]),
938 np.array([3 + 4j, 5 + 6j]),
939 np.array([True, False]), ]
940 assert_equal(test.dtype.names, ['f0', 'f1', 'f2', 'f3', 'f4'])
941 for (i, ctrl) in enumerate(control):
942 assert_equal(test['f%i' % i], ctrl)
943
944 def test_auto_dtype_uniform(self):
945 "Tests whether the output dtype can be uniformized"
946 data = TextIO('1 2 3 4\n5 6 7 8\n')
947 test = np.ndfromtxt(data, dtype=None)
948 control = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
949 assert_equal(test, control)
950
951 def test_fancy_dtype(self):
952 "Check that a nested dtype isn't MIA"
953 data = TextIO('1,2,3.0\n4,5,6.0\n')
954 fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
955 test = np.ndfromtxt(data, dtype=fancydtype, delimiter=',')
956 control = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dtype=fancydtype)
957 assert_equal(test, control)
958
959 def test_names_overwrite(self):
960 "Test overwriting the names of the dtype"
961 descriptor = {'names': ('g', 'a', 'w'),
962 'formats': ('S1', 'i4', 'f4')}
963 data = TextIO(b'M 64.0 75.0\nF 25.0 60.0')
964 names = ('gender', 'age', 'weight')
965 test = np.ndfromtxt(data, dtype=descriptor, names=names)
966 descriptor['names'] = names
967 control = np.array([('M', 64.0, 75.0),
968 ('F', 25.0, 60.0)], dtype=descriptor)
969 assert_equal(test, control)
970
971 def test_commented_header(self):
972 "Check that names can be retrieved even if the line is commented out."
973 data = TextIO("""
974 #gender age weight
975 M 21 72.100000
976 F 35 58.330000
977 M 33 21.99
978 """)
979 # The # is part of the first name and should be deleted automatically.
980 test = np.genfromtxt(data, names=True, dtype=None)
981 ctrl = np.array([('M', 21, 72.1), ('F', 35, 58.33), ('M', 33, 21.99)],
982 dtype=[('gender', '|S1'), ('age', int), ('weight', float)])
983 assert_equal(test, ctrl)
984 # Ditto, but we should get rid of the first element
985 data = TextIO(b"""
986 # gender age weight
987 M 21 72.100000
988 F 35 58.330000
989 M 33 21.99
990 """)
991 test = np.genfromtxt(data, names=True, dtype=None)
992 assert_equal(test, ctrl)
993
994 def test_autonames_and_usecols(self):
995 "Tests names and usecols"
996 data = TextIO('A B C D\n aaaa 121 45 9.1')
997 test = np.ndfromtxt(data, usecols=('A', 'C', 'D'),
998 names=True, dtype=None)
999 control = np.array(('aaaa', 45, 9.1),
1000 dtype=[('A', '|S4'), ('C', int), ('D', float)])
1001 assert_equal(test, control)
1002
1003 def test_converters_with_usecols(self):
1004 "Test the combination user-defined converters and usecol"
1005 data = TextIO('1,2,3,,5\n6,7,8,9,10\n')
1006 test = np.ndfromtxt(data, dtype=int, delimiter=',',
1007 converters={3: lambda s: int(s or - 999)},
1008 usecols=(1, 3,))
1009 control = np.array([[2, -999], [7, 9]], int)
1010 assert_equal(test, control)
1011
1012 def test_converters_with_usecols_and_names(self):
1013 "Tests names and usecols"
1014 data = TextIO('A B C D\n aaaa 121 45 9.1')
1015 test = np.ndfromtxt(data, usecols=('A', 'C', 'D'), names=True,
1016 dtype=None, converters={'C': lambda s: 2 * int(s)})
1017 control = np.array(('aaaa', 90, 9.1),
1018 dtype=[('A', '|S4'), ('C', int), ('D', float)])
1019 assert_equal(test, control)
1020
1021 def test_converters_cornercases(self):
1022 "Test the conversion to datetime."
1023 converter = {
1024 'date': lambda s: strptime(s, '%Y-%m-%d %H:%M:%SZ')}
1025 data = TextIO('2009-02-03 12:00:00Z, 72214.0')
1026 test = np.ndfromtxt(data, delimiter=',', dtype=None,
1027 names=['date', 'stid'], converters=converter)
1028 control = np.array((datetime(2009, 2, 3), 72214.),
1029 dtype=[('date', np.object_), ('stid', float)])
1030 assert_equal(test, control)
1031
1032 def test_converters_cornercases2(self):
1033 "Test the conversion to datetime64."
1034 converter = {
1035 'date': lambda s: np.datetime64(strptime(s, '%Y-%m-%d %H:%M:%SZ'))}
1036 data = TextIO('2009-02-03 12:00:00Z, 72214.0')
1037 test = np.ndfromtxt(data, delimiter=',', dtype=None,
1038 names=['date', 'stid'], converters=converter)
1039 control = np.array((datetime(2009, 2, 3), 72214.),
1040 dtype=[('date', 'datetime64[us]'), ('stid', float)])
1041 assert_equal(test, control)
1042
1043 def test_unused_converter(self):
1044 "Test whether unused converters are forgotten"
1045 data = TextIO("1 21\n 3 42\n")
1046 test = np.ndfromtxt(data, usecols=(1,),
1047 converters={0: lambda s: int(s, 16)})
1048 assert_equal(test, [21, 42])
1049 #
1050 data.seek(0)
1051 test = np.ndfromtxt(data, usecols=(1,),
1052 converters={1: lambda s: int(s, 16)})
1053 assert_equal(test, [33, 66])
1054
1055 def test_invalid_converter(self):
1056 strip_rand = lambda x: float((b'r' in x.lower() and x.split()[-1]) or
1057 (b'r' not in x.lower() and x.strip() or 0.0))
1058 strip_per = lambda x: float((b'%' in x.lower() and x.split()[0]) or
1059 (b'%' not in x.lower() and x.strip() or 0.0))
1060 s = TextIO("D01N01,10/1/2003 ,1 %,R 75,400,600\r\n"
1061 "L24U05,12/5/2003, 2 %,1,300, 150.5\r\n"
1062 "D02N03,10/10/2004,R 1,,7,145.55")
1063 kwargs = dict(
1064 converters={2: strip_per, 3: strip_rand}, delimiter=",",
1065 dtype=None)
1066 assert_raises(ConverterError, np.genfromtxt, s, **kwargs)
1067
1068 def test_tricky_converter_bug1666(self):
1069 "Test some corner case"
1070 s = TextIO('q1,2\nq3,4')
1071 cnv = lambda s: float(s[1:])
1072 test = np.genfromtxt(s, delimiter=',', converters={0: cnv})
1073 control = np.array([[1., 2.], [3., 4.]])
1074 assert_equal(test, control)
1075
1076 def test_dtype_with_converters(self):
1077 dstr = "2009; 23; 46"
1078 test = np.ndfromtxt(TextIO(dstr,),
1079 delimiter=";", dtype=float, converters={0: bytes})
1080 control = np.array([('2009', 23., 46)],
1081 dtype=[('f0', '|S4'), ('f1', float), ('f2', float)])
1082 assert_equal(test, control)
1083 test = np.ndfromtxt(TextIO(dstr,),
1084 delimiter=";", dtype=float, converters={0: float})
1085 control = np.array([2009., 23., 46],)
1086 assert_equal(test, control)
1087
1088 def test_dtype_with_converters_and_usecols(self):
1089 dstr = "1,5,-1,1:1\n2,8,-1,1:n\n3,3,-2,m:n\n"
1090 dmap = {'1:1':0, '1:n':1, 'm:1':2, 'm:n':3}
1091 dtyp = [('E1','i4'),('E2','i4'),('E3','i2'),('N', 'i1')]
1092 conv = {0: int, 1: int, 2: int, 3: lambda r: dmap[r.decode()]}
1093 test = np.recfromcsv(TextIO(dstr,), dtype=dtyp, delimiter=',',
1094 names=None, converters=conv)
1095 control = np.rec.array([[1,5,-1,0], [2,8,-1,1], [3,3,-2,3]], dtype=dtyp)
1096 assert_equal(test, control)
1097 dtyp = [('E1','i4'),('E2','i4'),('N', 'i1')]
1098 test = np.recfromcsv(TextIO(dstr,), dtype=dtyp, delimiter=',',
1099 usecols=(0,1,3), names=None, converters=conv)
1100 control = np.rec.array([[1,5,0], [2,8,1], [3,3,3]], dtype=dtyp)
1101 assert_equal(test, control)
1102
1103 def test_dtype_with_object(self):
1104 "Test using an explicit dtype with an object"
1105 from datetime import date
1106 import time
1107 data = """ 1; 2001-01-01
1108 2; 2002-01-31 """
1109 ndtype = [('idx', int), ('code', np.object)]
1110 func = lambda s: strptime(s.strip(), "%Y-%m-%d")
1111 converters = {1: func}
1112 test = np.genfromtxt(TextIO(data), delimiter=";", dtype=ndtype,
1113 converters=converters)
1114 control = np.array(
1115 [(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
1116 dtype=ndtype)
1117 assert_equal(test, control)
1118 #
1119 ndtype = [('nest', [('idx', int), ('code', np.object)])]
1120 try:
1121 test = np.genfromtxt(TextIO(data), delimiter=";",
1122 dtype=ndtype, converters=converters)
1123 except NotImplementedError:
1124 pass
1125 else:
1126 errmsg = "Nested dtype involving objects should be supported."
1127 raise AssertionError(errmsg)
1128
1129 def test_userconverters_with_explicit_dtype(self):
1130 "Test user_converters w/ explicit (standard) dtype"
1131 data = TextIO('skip,skip,2001-01-01,1.0,skip')
1132 test = np.genfromtxt(data, delimiter=",", names=None, dtype=float,
1133 usecols=(2, 3), converters={2: bytes})
1134 control = np.array([('2001-01-01', 1.)],
1135 dtype=[('', '|S10'), ('', float)])
1136 assert_equal(test, control)
1137
1138 def test_spacedelimiter(self):
1139 "Test space delimiter"
1140 data = TextIO("1 2 3 4 5\n6 7 8 9 10")
1141 test = np.ndfromtxt(data)
1142 control = np.array([[1., 2., 3., 4., 5.],
1143 [6., 7., 8., 9., 10.]])
1144 assert_equal(test, control)
1145
1146 def test_integer_delimiter(self):
1147 "Test using an integer for delimiter"
1148 data = " 1 2 3\n 4 5 67\n890123 4"
1149 test = np.genfromtxt(TextIO(data), delimiter=3)
1150 control = np.array([[1, 2, 3], [4, 5, 67], [890, 123, 4]])
1151 assert_equal(test, control)
1152
1153 def test_missing(self):
1154 data = TextIO('1,2,3,,5\n')
1155 test = np.ndfromtxt(data, dtype=int, delimiter=',',
1156 converters={3: lambda s: int(s or - 999)})
1157 control = np.array([1, 2, 3, -999, 5], int)
1158 assert_equal(test, control)
1159
1160 def test_missing_with_tabs(self):
1161 "Test w/ a delimiter tab"
1162 txt = "1\t2\t3\n\t2\t\n1\t\t3"
1163 test = np.genfromtxt(TextIO(txt), delimiter="\t",
1164 usemask=True,)
1165 ctrl_d = np.array([(1, 2, 3), (np.nan, 2, np.nan), (1, np.nan, 3)],)
1166 ctrl_m = np.array([(0, 0, 0), (1, 0, 1), (0, 1, 0)], dtype=bool)
1167 assert_equal(test.data, ctrl_d)
1168 assert_equal(test.mask, ctrl_m)
1169
1170 def test_usecols(self):
1171 "Test the selection of columns"
1172 # Select 1 column
1173 control = np.array([[1, 2], [3, 4]], float)
1174 data = TextIO()
1175 np.savetxt(data, control)
1176 data.seek(0)
1177 test = np.ndfromtxt(data, dtype=float, usecols=(1,))
1178 assert_equal(test, control[:, 1])
1179 #
1180 control = np.array([[1, 2, 3], [3, 4, 5]], float)
1181 data = TextIO()
1182 np.savetxt(data, control)
1183 data.seek(0)
1184 test = np.ndfromtxt(data, dtype=float, usecols=(1, 2))
1185 assert_equal(test, control[:, 1:])
1186 # Testing with arrays instead of tuples.
1187 data.seek(0)
1188 test = np.ndfromtxt(data, dtype=float, usecols=np.array([1, 2]))
1189 assert_equal(test, control[:, 1:])
1190
1191 def test_usecols_as_css(self):
1192 "Test giving usecols with a comma-separated string"
1193 data = "1 2 3\n4 5 6"
1194 test = np.genfromtxt(TextIO(data),
1195 names="a, b, c", usecols="a, c")
1196 ctrl = np.array([(1, 3), (4, 6)], dtype=[(_, float) for _ in "ac"])
1197 assert_equal(test, ctrl)
1198
1199 def test_usecols_with_structured_dtype(self):
1200 "Test usecols with an explicit structured dtype"
1201 data = TextIO("JOE 70.1 25.3\nBOB 60.5 27.9")
1202 names = ['stid', 'temp']
1203 dtypes = ['S4', 'f8']
1204 test = np.ndfromtxt(
1205 data, usecols=(0, 2), dtype=list(zip(names, dtypes)))
1206 assert_equal(test['stid'], [b"JOE", b"BOB"])
1207 assert_equal(test['temp'], [25.3, 27.9])
1208
1209 def test_usecols_with_integer(self):
1210 "Test usecols with an integer"
1211 test = np.genfromtxt(TextIO(b"1 2 3\n4 5 6"), usecols=0)
1212 assert_equal(test, np.array([1., 4.]))
1213
1214 def test_usecols_with_named_columns(self):
1215 "Test usecols with named columns"
1216 ctrl = np.array([(1, 3), (4, 6)], dtype=[('a', float), ('c', float)])
1217 data = "1 2 3\n4 5 6"
1218 kwargs = dict(names="a, b, c")
1219 test = np.genfromtxt(TextIO(data), usecols=(0, -1), **kwargs)
1220 assert_equal(test, ctrl)
1221 test = np.genfromtxt(TextIO(data),
1222 usecols=('a', 'c'), **kwargs)
1223 assert_equal(test, ctrl)
1224
1225 def test_empty_file(self):
1226 "Test that an empty file raises the proper warning."
1227 with warnings.catch_warnings():
1228 warnings.filterwarnings("ignore",
1229 message="genfromtxt: Empty input file:")
1230 data = TextIO()
1231 test = np.genfromtxt(data)
1232 assert_equal(test, np.array([]))
1233
1234 def test_fancy_dtype_alt(self):
1235 "Check that a nested dtype isn't MIA"
1236 data = TextIO('1,2,3.0\n4,5,6.0\n')
1237 fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
1238 test = np.mafromtxt(data, dtype=fancydtype, delimiter=',')
1239 control = ma.array([(1, (2, 3.0)), (4, (5, 6.0))], dtype=fancydtype)
1240 assert_equal(test, control)
1241
1242 def test_shaped_dtype(self):
1243 c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6")
1244 dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
1245 ('block', int, (2, 3))])
1246 x = np.ndfromtxt(c, dtype=dt)
1247 a = np.array([('aaaa', 1.0, 8.0, [[1, 2, 3], [4, 5, 6]])],
1248 dtype=dt)
1249 assert_array_equal(x, a)
1250
1251 def test_withmissing(self):
1252 data = TextIO('A,B\n0,1\n2,N/A')
1253 kwargs = dict(delimiter=",", missing_values="N/A", names=True)
1254 test = np.mafromtxt(data, dtype=None, **kwargs)
1255 control = ma.array([(0, 1), (2, -1)],
1256 mask=[(False, False), (False, True)],
1257 dtype=[('A', np.int), ('B', np.int)])
1258 assert_equal(test, control)
1259 assert_equal(test.mask, control.mask)
1260 #
1261 data.seek(0)
1262 test = np.mafromtxt(data, **kwargs)
1263 control = ma.array([(0, 1), (2, -1)],
1264 mask=[(False, False), (False, True)],
1265 dtype=[('A', np.float), ('B', np.float)])
1266 assert_equal(test, control)
1267 assert_equal(test.mask, control.mask)
1268
1269 def test_user_missing_values(self):
1270 data = "A, B, C\n0, 0., 0j\n1, N/A, 1j\n-9, 2.2, N/A\n3, -99, 3j"
1271 basekwargs = dict(dtype=None, delimiter=",", names=True,)
1272 mdtype = [('A', int), ('B', float), ('C', complex)]
1273 #
1274 test = np.mafromtxt(TextIO(data), missing_values="N/A",
1275 **basekwargs)
1276 control = ma.array([(0, 0.0, 0j), (1, -999, 1j),
1277 (-9, 2.2, -999j), (3, -99, 3j)],
1278 mask=[(0, 0, 0), (0, 1, 0), (0, 0, 1), (0, 0, 0)],
1279 dtype=mdtype)
1280 assert_equal(test, control)
1281 #
1282 basekwargs['dtype'] = mdtype
1283 test = np.mafromtxt(TextIO(data),
1284 missing_values={0: -9, 1: -99, 2: -999j}, **basekwargs)
1285 control = ma.array([(0, 0.0, 0j), (1, -999, 1j),
1286 (-9, 2.2, -999j), (3, -99, 3j)],
1287 mask=[(0, 0, 0), (0, 1, 0), (1, 0, 1), (0, 1, 0)],
1288 dtype=mdtype)
1289 assert_equal(test, control)
1290 #
1291 test = np.mafromtxt(TextIO(data),
1292 missing_values={0: -9, 'B': -99, 'C': -999j},
1293 **basekwargs)
1294 control = ma.array([(0, 0.0, 0j), (1, -999, 1j),
1295 (-9, 2.2, -999j), (3, -99, 3j)],
1296 mask=[(0, 0, 0), (0, 1, 0), (1, 0, 1), (0, 1, 0)],
1297 dtype=mdtype)
1298 assert_equal(test, control)
1299
1300 def test_user_filling_values(self):
1301 "Test with missing and filling values"
1302 ctrl = np.array([(0, 3), (4, -999)], dtype=[('a', int), ('b', int)])
1303 data = "N/A, 2, 3\n4, ,???"
1304 kwargs = dict(delimiter=",",
1305 dtype=int,
1306 names="a,b,c",
1307 missing_values={0: "N/A", 'b': " ", 2: "???"},
1308 filling_values={0: 0, 'b': 0, 2: -999})
1309 test = np.genfromtxt(TextIO(data), **kwargs)
1310 ctrl = np.array([(0, 2, 3), (4, 0, -999)],
1311 dtype=[(_, int) for _ in "abc"])
1312 assert_equal(test, ctrl)
1313 #
1314 test = np.genfromtxt(TextIO(data), usecols=(0, -1), **kwargs)
1315 ctrl = np.array([(0, 3), (4, -999)], dtype=[(_, int) for _ in "ac"])
1316 assert_equal(test, ctrl)
1317
1318 data2 = "1,2,*,4\n5,*,7,8\n"
1319 test = np.genfromtxt(TextIO(data2), delimiter=',', dtype=int,
1320 missing_values="*", filling_values=0)
1321 ctrl = np.array([[1, 2, 0, 4], [5, 0, 7, 8]])
1322 assert_equal(test, ctrl)
1323 test = np.genfromtxt(TextIO(data2), delimiter=',', dtype=int,
1324 missing_values="*", filling_values=-1)
1325 ctrl = np.array([[1, 2, -1, 4], [5, -1, 7, 8]])
1326 assert_equal(test, ctrl)
1327
1328 def test_withmissing_float(self):
1329 data = TextIO('A,B\n0,1.5\n2,-999.00')
1330 test = np.mafromtxt(data, dtype=None, delimiter=',',
1331 missing_values='-999.0', names=True,)
1332 control = ma.array([(0, 1.5), (2, -1.)],
1333 mask=[(False, False), (False, True)],
1334 dtype=[('A', np.int), ('B', np.float)])
1335 assert_equal(test, control)
1336 assert_equal(test.mask, control.mask)
1337
1338 def test_with_masked_column_uniform(self):
1339 "Test masked column"
1340 data = TextIO('1 2 3\n4 5 6\n')
1341 test = np.genfromtxt(data, dtype=None,
1342 missing_values='2,5', usemask=True)
1343 control = ma.array([[1, 2, 3], [4, 5, 6]], mask=[[0, 1, 0], [0, 1, 0]])
1344 assert_equal(test, control)
1345
1346 def test_with_masked_column_various(self):
1347 "Test masked column"
1348 data = TextIO('True 2 3\nFalse 5 6\n')
1349 test = np.genfromtxt(data, dtype=None,
1350 missing_values='2,5', usemask=True)
1351 control = ma.array([(1, 2, 3), (0, 5, 6)],
1352 mask=[(0, 1, 0), (0, 1, 0)],
1353 dtype=[('f0', bool), ('f1', bool), ('f2', int)])
1354 assert_equal(test, control)
1355
1356 def test_invalid_raise(self):
1357 "Test invalid raise"
1358 data = ["1, 1, 1, 1, 1"] * 50
1359 for i in range(5):
1360 data[10 * i] = "2, 2, 2, 2 2"
1361 data.insert(0, "a, b, c, d, e")
1362 mdata = TextIO("\n".join(data))
1363 #
1364 kwargs = dict(delimiter=",", dtype=None, names=True)
1365 # XXX: is there a better way to get the return value of the callable in
1366 # assert_warns ?
1367 ret = {}
1368
1369 def f(_ret={}):
1370 _ret['mtest'] = np.ndfromtxt(mdata, invalid_raise=False, **kwargs)
1371 assert_warns(ConversionWarning, f, _ret=ret)
1372 mtest = ret['mtest']
1373 assert_equal(len(mtest), 45)
1374 assert_equal(mtest, np.ones(45, dtype=[(_, int) for _ in 'abcde']))
1375 #
1376 mdata.seek(0)
1377 assert_raises(ValueError, np.ndfromtxt, mdata,
1378 delimiter=",", names=True)
1379
1380 def test_invalid_raise_with_usecols(self):
1381 "Test invalid_raise with usecols"
1382 data = ["1, 1, 1, 1, 1"] * 50
1383 for i in range(5):
1384 data[10 * i] = "2, 2, 2, 2 2"
1385 data.insert(0, "a, b, c, d, e")
1386 mdata = TextIO("\n".join(data))
1387 kwargs = dict(delimiter=",", dtype=None, names=True,
1388 invalid_raise=False)
1389 # XXX: is there a better way to get the return value of the callable in
1390 # assert_warns ?
1391 ret = {}
1392
1393 def f(_ret={}):
1394 _ret['mtest'] = np.ndfromtxt(mdata, usecols=(0, 4), **kwargs)
1395 assert_warns(ConversionWarning, f, _ret=ret)
1396 mtest = ret['mtest']
1397 assert_equal(len(mtest), 45)
1398 assert_equal(mtest, np.ones(45, dtype=[(_, int) for _ in 'ae']))
1399 #
1400 mdata.seek(0)
1401 mtest = np.ndfromtxt(mdata, usecols=(0, 1), **kwargs)
1402 assert_equal(len(mtest), 50)
1403 control = np.ones(50, dtype=[(_, int) for _ in 'ab'])
1404 control[[10 * _ for _ in range(5)]] = (2, 2)
1405 assert_equal(mtest, control)
1406
1407 def test_inconsistent_dtype(self):
1408 "Test inconsistent dtype"
1409 data = ["1, 1, 1, 1, -1.1"] * 50
1410 mdata = TextIO("\n".join(data))
1411
1412 converters = {4: lambda x: "(%s)" % x}
1413 kwargs = dict(delimiter=",", converters=converters,
1414 dtype=[(_, int) for _ in 'abcde'],)
1415 assert_raises(ValueError, np.genfromtxt, mdata, **kwargs)
1416
1417 def test_default_field_format(self):
1418 "Test default format"
1419 data = "0, 1, 2.3\n4, 5, 6.7"
1420 mtest = np.ndfromtxt(TextIO(data),
1421 delimiter=",", dtype=None, defaultfmt="f%02i")
1422 ctrl = np.array([(0, 1, 2.3), (4, 5, 6.7)],
1423 dtype=[("f00", int), ("f01", int), ("f02", float)])
1424 assert_equal(mtest, ctrl)
1425
1426 def test_single_dtype_wo_names(self):
1427 "Test single dtype w/o names"
1428 data = "0, 1, 2.3\n4, 5, 6.7"
1429 mtest = np.ndfromtxt(TextIO(data),
1430 delimiter=",", dtype=float, defaultfmt="f%02i")
1431 ctrl = np.array([[0., 1., 2.3], [4., 5., 6.7]], dtype=float)
1432 assert_equal(mtest, ctrl)
1433
1434 def test_single_dtype_w_explicit_names(self):
1435 "Test single dtype w explicit names"
1436 data = "0, 1, 2.3\n4, 5, 6.7"
1437 mtest = np.ndfromtxt(TextIO(data),
1438 delimiter=",", dtype=float, names="a, b, c")
1439 ctrl = np.array([(0., 1., 2.3), (4., 5., 6.7)],
1440 dtype=[(_, float) for _ in "abc"])
1441 assert_equal(mtest, ctrl)
1442
1443 def test_single_dtype_w_implicit_names(self):
1444 "Test single dtype w implicit names"
1445 data = "a, b, c\n0, 1, 2.3\n4, 5, 6.7"
1446 mtest = np.ndfromtxt(TextIO(data),
1447 delimiter=",", dtype=float, names=True)
1448 ctrl = np.array([(0., 1., 2.3), (4., 5., 6.7)],
1449 dtype=[(_, float) for _ in "abc"])
1450 assert_equal(mtest, ctrl)
1451
1452 def test_easy_structured_dtype(self):
1453 "Test easy structured dtype"
1454 data = "0, 1, 2.3\n4, 5, 6.7"
1455 mtest = np.ndfromtxt(TextIO(data), delimiter=",",
1456 dtype=(int, float, float), defaultfmt="f_%02i")
1457 ctrl = np.array([(0, 1., 2.3), (4, 5., 6.7)],
1458 dtype=[("f_00", int), ("f_01", float), ("f_02", float)])
1459 assert_equal(mtest, ctrl)
1460
1461 def test_autostrip(self):
1462 "Test autostrip"
1463 data = "01/01/2003 , 1.3, abcde"
1464 kwargs = dict(delimiter=",", dtype=None)
1465 mtest = np.ndfromtxt(TextIO(data), **kwargs)
1466 ctrl = np.array([('01/01/2003 ', 1.3, ' abcde')],
1467 dtype=[('f0', '|S12'), ('f1', float), ('f2', '|S8')])
1468 assert_equal(mtest, ctrl)
1469 mtest = np.ndfromtxt(TextIO(data), autostrip=True, **kwargs)
1470 ctrl = np.array([('01/01/2003', 1.3, 'abcde')],
1471 dtype=[('f0', '|S10'), ('f1', float), ('f2', '|S5')])
1472 assert_equal(mtest, ctrl)
1473
1474 def test_replace_space(self):
1475 "Test the 'replace_space' option"
1476 txt = "A.A, B (B), C:C\n1, 2, 3.14"
1477 # Test default: replace ' ' by '_' and delete non-alphanum chars
1478 test = np.genfromtxt(TextIO(txt),
1479 delimiter=",", names=True, dtype=None)
1480 ctrl_dtype = [("AA", int), ("B_B", int), ("CC", float)]
1481 ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
1482 assert_equal(test, ctrl)
1483 # Test: no replace, no delete
1484 test = np.genfromtxt(TextIO(txt),
1485 delimiter=",", names=True, dtype=None,
1486 replace_space='', deletechars='')
1487 ctrl_dtype = [("A.A", int), ("B (B)", int), ("C:C", float)]
1488 ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
1489 assert_equal(test, ctrl)
1490 # Test: no delete (spaces are replaced by _)
1491 test = np.genfromtxt(TextIO(txt),
1492 delimiter=",", names=True, dtype=None,
1493 deletechars='')
1494 ctrl_dtype = [("A.A", int), ("B_(B)", int), ("C:C", float)]
1495 ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
1496 assert_equal(test, ctrl)
1497
1498 def test_incomplete_names(self):
1499 "Test w/ incomplete names"
1500 data = "A,,C\n0,1,2\n3,4,5"
1501 kwargs = dict(delimiter=",", names=True)
1502 # w/ dtype=None
1503 ctrl = np.array([(0, 1, 2), (3, 4, 5)],
1504 dtype=[(_, int) for _ in ('A', 'f0', 'C')])
1505 test = np.ndfromtxt(TextIO(data), dtype=None, **kwargs)
1506 assert_equal(test, ctrl)
1507 # w/ default dtype
1508 ctrl = np.array([(0, 1, 2), (3, 4, 5)],
1509 dtype=[(_, float) for _ in ('A', 'f0', 'C')])
1510 test = np.ndfromtxt(TextIO(data), **kwargs)
1511
1512 def test_names_auto_completion(self):
1513 "Make sure that names are properly completed"
1514 data = "1 2 3\n 4 5 6"
1515 test = np.genfromtxt(TextIO(data),
1516 dtype=(int, float, int), names="a")
1517 ctrl = np.array([(1, 2, 3), (4, 5, 6)],
1518 dtype=[('a', int), ('f0', float), ('f1', int)])
1519 assert_equal(test, ctrl)
1520
1521 def test_names_with_usecols_bug1636(self):
1522 "Make sure we pick up the right names w/ usecols"
1523 data = "A,B,C,D,E\n0,1,2,3,4\n0,1,2,3,4\n0,1,2,3,4"
1524 ctrl_names = ("A", "C", "E")
1525 test = np.genfromtxt(TextIO(data),
1526 dtype=(int, int, int), delimiter=",",
1527 usecols=(0, 2, 4), names=True)
1528 assert_equal(test.dtype.names, ctrl_names)
1529 #
1530 test = np.genfromtxt(TextIO(data),
1531 dtype=(int, int, int), delimiter=",",
1532 usecols=("A", "C", "E"), names=True)
1533 assert_equal(test.dtype.names, ctrl_names)
1534 #
1535 test = np.genfromtxt(TextIO(data),
1536 dtype=int, delimiter=",",
1537 usecols=("A", "C", "E"), names=True)
1538 assert_equal(test.dtype.names, ctrl_names)
1539
1540 def test_fixed_width_names(self):
1541 "Test fix-width w/ names"
1542 data = " A B C\n 0 1 2.3\n 45 67 9."
1543 kwargs = dict(delimiter=(5, 5, 4), names=True, dtype=None)
1544 ctrl = np.array([(0, 1, 2.3), (45, 67, 9.)],
1545 dtype=[('A', int), ('B', int), ('C', float)])
1546 test = np.ndfromtxt(TextIO(data), **kwargs)
1547 assert_equal(test, ctrl)
1548 #
1549 kwargs = dict(delimiter=5, names=True, dtype=None)
1550 ctrl = np.array([(0, 1, 2.3), (45, 67, 9.)],
1551 dtype=[('A', int), ('B', int), ('C', float)])
1552 test = np.ndfromtxt(TextIO(data), **kwargs)
1553 assert_equal(test, ctrl)
1554
1555 def test_filling_values(self):
1556 "Test missing values"
1557 data = b"1, 2, 3\n1, , 5\n0, 6, \n"
1558 kwargs = dict(delimiter=",", dtype=None, filling_values=-999)
1559 ctrl = np.array([[1, 2, 3], [1, -999, 5], [0, 6, -999]], dtype=int)
1560 test = np.ndfromtxt(TextIO(data), **kwargs)
1561 assert_equal(test, ctrl)
1562
1563 def test_comments_is_none(self):
1564 # Github issue 329 (None was previously being converted to 'None').
1565 test = np.genfromtxt(TextIO("test1,testNonetherestofthedata"),
1566 dtype=None, comments=None, delimiter=',')
1567 assert_equal(test[1], b'testNonetherestofthedata')
1568 test = np.genfromtxt(TextIO("test1, testNonetherestofthedata"),
1569 dtype=None, comments=None, delimiter=',')
1570 assert_equal(test[1], b' testNonetherestofthedata')
1571
1572 def test_recfromtxt(self):
1573 #
1574 data = TextIO('A,B\n0,1\n2,3')
1575 kwargs = dict(delimiter=",", missing_values="N/A", names=True)
1576 test = np.recfromtxt(data, **kwargs)
1577 control = np.array([(0, 1), (2, 3)],
1578 dtype=[('A', np.int), ('B', np.int)])
1579 self.assertTrue(isinstance(test, np.recarray))
1580 assert_equal(test, control)
1581 #
1582 data = TextIO('A,B\n0,1\n2,N/A')
1583 test = np.recfromtxt(data, dtype=None, usemask=True, **kwargs)
1584 control = ma.array([(0, 1), (2, -1)],
1585 mask=[(False, False), (False, True)],
1586 dtype=[('A', np.int), ('B', np.int)])
1587 assert_equal(test, control)
1588 assert_equal(test.mask, control.mask)
1589 assert_equal(test.A, [0, 2])
1590
1591 def test_recfromcsv(self):
1592 #
1593 data = TextIO('A,B\n0,1\n2,3')
1594 kwargs = dict(missing_values="N/A", names=True, case_sensitive=True)
1595 test = np.recfromcsv(data, dtype=None, **kwargs)
1596 control = np.array([(0, 1), (2, 3)],
1597 dtype=[('A', np.int), ('B', np.int)])
1598 self.assertTrue(isinstance(test, np.recarray))
1599 assert_equal(test, control)
1600 #
1601 data = TextIO('A,B\n0,1\n2,N/A')
1602 test = np.recfromcsv(data, dtype=None, usemask=True, **kwargs)
1603 control = ma.array([(0, 1), (2, -1)],
1604 mask=[(False, False), (False, True)],
1605 dtype=[('A', np.int), ('B', np.int)])
1606 assert_equal(test, control)
1607 assert_equal(test.mask, control.mask)
1608 assert_equal(test.A, [0, 2])
1609 #
1610 data = TextIO('A,B\n0,1\n2,3')
1611 test = np.recfromcsv(data, missing_values='N/A',)
1612 control = np.array([(0, 1), (2, 3)],
1613 dtype=[('a', np.int), ('b', np.int)])
1614 self.assertTrue(isinstance(test, np.recarray))
1615 assert_equal(test, control)
1616 #
1617 data = TextIO('A,B\n0,1\n2,3')
1618 dtype = [('a', np.int), ('b', np.float)]
1619 test = np.recfromcsv(data, missing_values='N/A', dtype=dtype)
1620 control = np.array([(0, 1), (2, 3)],
1621 dtype=dtype)
1622 self.assertTrue(isinstance(test, np.recarray))
1623 assert_equal(test, control)
1624
1625 def test_gft_using_filename(self):
1626 # Test that we can load data from a filename as well as a file object
1627 wanted = np.arange(6).reshape((2, 3))
1628 if sys.version_info[0] >= 3:
1629 # python 3k is known to fail for '\r'
1630 linesep = ('\n', '\r\n')
1631 else:
1632 linesep = ('\n', '\r\n', '\r')
1633
1634 for sep in linesep:
1635 data = '0 1 2' + sep + '3 4 5'
1636 f, name = mkstemp()
1637 # We can't use NamedTemporaryFile on windows, because we cannot
1638 # reopen the file.
1639 try:
1640 os.write(f, asbytes(data))
1641 assert_array_equal(np.genfromtxt(name), wanted)
1642 finally:
1643 os.close(f)
1644 os.unlink(name)
1645
1646 def test_gft_using_generator(self):
1647 # gft doesn't work with unicode.
1648 def count():
1649 for i in range(10):
1650 yield asbytes("%d" % i)
1651
1652 res = np.genfromtxt(count())
1653 assert_array_equal(res, np.arange(10))
1654
1655
1656 def test_gzip_load():
1657 a = np.random.random((5, 5))
1658
1659 s = BytesIO()
1660 f = gzip.GzipFile(fileobj=s, mode="w")
1661
1662 np.save(f, a)
1663 f.close()
1664 s.seek(0)
1665
1666 f = gzip.GzipFile(fileobj=s, mode="r")
1667 assert_array_equal(np.load(f), a)
1668
1669
1670 def test_gzip_loadtxt():
1671 # Thanks to another windows brokeness, we can't use
1672 # NamedTemporaryFile: a file created from this function cannot be
1673 # reopened by another open call. So we first put the gzipped string
1674 # of the test reference array, write it to a securely opened file,
1675 # which is then read from by the loadtxt function
1676 s = BytesIO()
1677 g = gzip.GzipFile(fileobj=s, mode='w')
1678 g.write(b'1 2 3\n')
1679 g.close()
1680 s.seek(0)
1681
1682 f, name = mkstemp(suffix='.gz')
1683 try:
1684 os.write(f, s.read())
1685 s.close()
1686 assert_array_equal(np.loadtxt(name), [1, 2, 3])
1687 finally:
1688 os.close(f)
1689 os.unlink(name)
1690
1691
1692 def test_gzip_loadtxt_from_string():
1693 s = BytesIO()
1694 f = gzip.GzipFile(fileobj=s, mode="w")
1695 f.write(b'1 2 3\n')
1696 f.close()
1697 s.seek(0)
1698
1699 f = gzip.GzipFile(fileobj=s, mode="r")
1700 assert_array_equal(np.loadtxt(f), [1, 2, 3])
1701
1702
1703 def test_npzfile_dict():
1704 s = BytesIO()
1705 x = np.zeros((3, 3))
1706 y = np.zeros((3, 3))
1707
1708 np.savez(s, x=x, y=y)
1709 s.seek(0)
1710
1711 z = np.load(s)
1712
1713 assert_('x' in z)
1714 assert_('y' in z)
1715 assert_('x' in z.keys())
1716 assert_('y' in z.keys())
1717
1718 for f, a in z.items():
1719 assert_(f in ['x', 'y'])
1720 assert_equal(a.shape, (3, 3))
1721
1722 assert_(len(z.items()) == 2)
1723
1724 for f in z:
1725 assert_(f in ['x', 'y'])
1726
1727 assert_('x' in z.keys())
1728
1729
1730 def test_load_refcount():
1731 # Check that objects returned by np.load are directly freed based on
1732 # their refcount, rather than needing the gc to collect them.
1733
1734 f = BytesIO()
1735 np.savez(f, [1, 2, 3])
1736 f.seek(0)
1737
1738 gc.collect()
1739 n_before = len(gc.get_objects())
1740 np.load(f)
1741 n_after = len(gc.get_objects())
1742
1743 assert_equal(n_before, n_after)
1744
1745 if __name__ == "__main__":
1746 run_module_suite()