Spaces:
Running
Running
File size: 8,073 Bytes
122d3ff |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
import h5py
import numpy
import numpy.testing
import pytest
from .common import ut, TestCase
class TestWriteDirectChunk(TestCase):
def test_write_direct_chunk(self):
filename = self.mktemp().encode()
with h5py.File(filename, "w") as filehandle:
dataset = filehandle.create_dataset("data", (100, 100, 100),
maxshape=(None, 100, 100),
chunks=(1, 100, 100),
dtype='float32')
# writing
array = numpy.zeros((10, 100, 100))
for index in range(10):
a = numpy.random.rand(100, 100).astype('float32')
dataset.id.write_direct_chunk((index, 0, 0), a.tobytes(), filter_mask=1)
array[index] = a
# checking
with h5py.File(filename, "r") as filehandle:
for i in range(10):
read_data = filehandle["data"][i]
numpy.testing.assert_array_equal(array[i], read_data)
@ut.skipIf('gzip' not in h5py.filters.encode, "DEFLATE is not installed")
class TestReadDirectChunk(TestCase):
def test_read_compressed_offsets(self):
filename = self.mktemp().encode()
with h5py.File(filename, "w") as filehandle:
frame = numpy.arange(16).reshape(4, 4)
frame_dataset = filehandle.create_dataset("frame",
data=frame,
compression="gzip",
compression_opts=9)
dataset = filehandle.create_dataset("compressed_chunked",
data=[frame, frame, frame],
compression="gzip",
compression_opts=9,
chunks=(1, ) + frame.shape)
filter_mask, compressed_frame = frame_dataset.id.read_direct_chunk((0, 0))
# No filter must be disabled
self.assertEqual(filter_mask, 0)
for i in range(dataset.shape[0]):
filter_mask, data = dataset.id.read_direct_chunk((i, 0, 0))
self.assertEqual(compressed_frame, data)
# No filter must be disabled
self.assertEqual(filter_mask, 0)
def test_read_uncompressed_offsets(self):
filename = self.mktemp().encode()
frame = numpy.arange(16).reshape(4, 4)
with h5py.File(filename, "w") as filehandle:
dataset = filehandle.create_dataset("frame",
maxshape=(1,) + frame.shape,
shape=(1,) + frame.shape,
compression="gzip",
compression_opts=9)
# Write uncompressed data
DISABLE_ALL_FILTERS = 0xFFFFFFFF
dataset.id.write_direct_chunk((0, 0, 0), frame.tobytes(), filter_mask=DISABLE_ALL_FILTERS)
# FIXME: Here we have to close the file and load it back else
# a runtime error occurs:
# RuntimeError: Can't get storage size of chunk (chunk storage is not allocated)
with h5py.File(filename, "r") as filehandle:
dataset = filehandle["frame"]
filter_mask, compressed_frame = dataset.id.read_direct_chunk((0, 0, 0))
# At least 1 filter is supposed to be disabled
self.assertNotEqual(filter_mask, 0)
self.assertEqual(compressed_frame, frame.tobytes())
def test_read_write_chunk(self):
filename = self.mktemp().encode()
with h5py.File(filename, "w") as filehandle:
# create a reference
frame = numpy.arange(16).reshape(4, 4)
frame_dataset = filehandle.create_dataset("source",
data=frame,
compression="gzip",
compression_opts=9)
# configure an empty dataset
filter_mask, compressed_frame = frame_dataset.id.read_direct_chunk((0, 0))
dataset = filehandle.create_dataset("created",
shape=frame_dataset.shape,
maxshape=frame_dataset.shape,
chunks=frame_dataset.chunks,
dtype=frame_dataset.dtype,
compression="gzip",
compression_opts=9)
# copy the data
dataset.id.write_direct_chunk((0, 0), compressed_frame, filter_mask=filter_mask)
# checking
with h5py.File(filename, "r") as filehandle:
dataset = filehandle["created"][...]
numpy.testing.assert_array_equal(dataset, frame)
class TestReadDirectChunkToOut:
def test_uncompressed_data(self, writable_file):
ref_data = numpy.arange(16).reshape(4, 4)
dataset = writable_file.create_dataset(
"uncompressed", data=ref_data, chunks=ref_data.shape)
out = bytearray(ref_data.nbytes)
filter_mask, chunk = dataset.id.read_direct_chunk((0, 0), out=out)
assert numpy.array_equal(
numpy.frombuffer(out, dtype=ref_data.dtype).reshape(ref_data.shape),
ref_data,
)
assert filter_mask == 0
assert len(chunk) == ref_data.nbytes
@pytest.mark.skipif(
h5py.version.hdf5_version_tuple < (1, 10, 5),
reason="chunk info requires HDF5 >= 1.10.5",
)
@pytest.mark.skipif(
'gzip' not in h5py.filters.encode,
reason="DEFLATE is not installed",
)
def test_compressed_data(self, writable_file):
ref_data = numpy.arange(16).reshape(4, 4)
dataset = writable_file.create_dataset(
"gzip",
data=ref_data,
chunks=ref_data.shape,
compression="gzip",
compression_opts=9,
)
chunk_info = dataset.id.get_chunk_info(0)
out = bytearray(chunk_info.size)
filter_mask, chunk = dataset.id.read_direct_chunk(
chunk_info.chunk_offset,
out=out,
)
assert filter_mask == chunk_info.filter_mask
assert len(chunk) == chunk_info.size
assert out == dataset.id.read_direct_chunk(chunk_info.chunk_offset)[1]
def test_fail_buffer_too_small(self, writable_file):
ref_data = numpy.arange(16).reshape(4, 4)
dataset = writable_file.create_dataset(
"uncompressed", data=ref_data, chunks=ref_data.shape)
out = bytearray(ref_data.nbytes // 2)
with pytest.raises(ValueError):
dataset.id.read_direct_chunk((0, 0), out=out)
def test_fail_buffer_readonly(self, writable_file):
ref_data = numpy.arange(16).reshape(4, 4)
dataset = writable_file.create_dataset(
"uncompressed", data=ref_data, chunks=ref_data.shape)
out = bytes(ref_data.nbytes)
with pytest.raises(BufferError):
dataset.id.read_direct_chunk((0, 0), out=out)
def test_fail_buffer_not_contiguous(self, writable_file):
ref_data = numpy.arange(16).reshape(4, 4)
dataset = writable_file.create_dataset(
"uncompressed", data=ref_data, chunks=ref_data.shape)
array = numpy.empty(ref_data.shape + (2,), dtype=ref_data.dtype)
out = array[:, :, ::2] # Array is not contiguous
with pytest.raises(ValueError):
dataset.id.read_direct_chunk((0, 0), out=out)
|