Merge pull request #33 from jurekkow/feature/fix-bigtiff

Feature/fix bigtiff

apeer_ometiff_library/io.py

@@ -1,3 +1,4 @@
+import struct
 from pathlib import Path
 from types import TracebackType
 from typing import Union, Optional, Type, Tuple, List
@@ -43,9 +44,9 @@ def read_multi_series(self) -> Tuple[List[np.ndarray], str]:
         omexml_string = self._tiff_file.ome_metadata
         arrays = [
             _ensure_correct_dimensions(
-                self._tiff_file.asarray(series=series), omexml_string
+                self._tiff_file.asarray(key=image_key), omexml_string
             )
-            for series in range(
+            for image_key in range(
                 omexmlClass.OMEXML(self._tiff_file.ome_metadata).image_count
             )
         ]
@@ -230,9 +231,8 @@ def write_ometiff(output_path, array, omexml_string = None, compression=None):
     if omexml_string is None:
         omexml_string = gen_xml(array)
 
-    if sys.version < "3.7":
-        tifffile.imwrite(output_path, array,  photometric = "minisblack", description=omexml_string, metadata=None,
-                         compress=compression)
-    else:
-        tifffile.imwrite(output_path, array,  photometric = "minisblack", description=omexml_string, metadata=None,
-                         compression=compression)
+    data_limit_nbytes = 2 ** 32
+    tiff_metadata_nbytes = 2 ** 25
+    bigtiff = array.nbytes > data_limit_nbytes - tiff_metadata_nbytes
+    tifffile.imwrite(output_path, array, photometric="minisblack", description=omexml_string, metadata=None,
+                     compress=compression, bigtiff=bigtiff)

azure-pipelines.yml

@@ -16,9 +16,10 @@ steps:
 
 - script: |
     python -m pip install --upgrade pip
-    pip install -r requirements.txt
+    pip install .
     pip install twine setuptools wheel
-  displayName: 'Install dependencies'
+    python -m unittest discover tests/
+  displayName: 'Install dependencies and run tests'
 
 - script: |
     python setup.py sdist bdist_wheel

setup.py

@@ -2,12 +2,12 @@
 from setuptools import setup
 
 setup(name='apeer-ometiff-library',
-      version='1.9.0',
+      version='1.10.0',
       description='Library to read and write ometiff images',
       url='https://github.com/apeer-micro/apeer-ometiff-library',
       author='apeer-micro',
       packages=setuptools.find_packages(),
-      install_requires=['numpy','tifffile'],
+      install_requires=['numpy==1.18.5','tifffile==2020.6.3', 'imagecodecs==2020.5.30'],
       license='MIT',
       classifiers=[
           "Programming Language :: Python :: 3",

tests/test_io.py

@@ -1,10 +1,15 @@
+import math
+import os
 import unittest
 from pathlib import Path
 
 import numpy as np
 import tifffile
 
-from apeer_ometiff_library.io import OmeTiffFile
+from apeer_ometiff_library.io import OmeTiffFile, write_ometiff
+
+# Change to True if you want to include log running tests in the test suite
+_RUN_SLOW_TESTS = False
 
 
 class TestOmeTiffFile(unittest.TestCase):
@@ -15,29 +20,8 @@ def setUp(self):
     def _set_up_ometiff(self):
         self.ometiff_path = Path("tmp.ome.tiff")
         self.ometiff_path.touch()
-        self.ome_tiff_metadata = """<?xml version='1.0' encoding='utf-8'?>
-        <OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-             xsi:schemaLocation="http://www.openmicroscopy.org/Schemas/OME/2016-06 http://www.openmicroscopy.org/Schemas/OME/2016-06/ome.xsd"
-             UUID="urn:uuid:11227ecd-e960-42e4-95c8-39e6cec94150">
-            <Image ID="0" Name="IMAGE0">
-                <AcquisitionDate>2022-05-13T11:25:25.212054</AcquisitionDate>
-                <Pixels DimensionOrder="XYCZT" ID="0" SizeC="1" SizeT="1" SizeX="64" SizeY="32" SizeZ="1" Type="uint8"
-                        BigEndian="true">
-                    <Channel ID="Channel:0:0" SamplesPerPixel="1" Name="C:0">
-                        <LightPath/>
-                    </Channel>
-                    <TiffData FirstT="0" FirstZ="0" FirstC="0" IFD="0" PlaneCount="1"/>
-                </Pixels>
-            </Image>
-        </OME>""".encode()
-        self.ometiff_array = 255 * np.ones((1, 1, 1, 32, 64), np.uint8)
-        with tifffile.TiffWriter(self.ometiff_path) as tiff_writer:
-            tiff_writer.write(
-                self.ometiff_array,
-                photometric="minisblack",
-                description=self.ome_tiff_metadata,
-                metadata=None,
-            )
+        self.ometiff_array = 255 * np.ones((2, 3, 4, 32, 64), np.uint8)
+        write_ometiff(str(self.ometiff_path), self.ometiff_array)
 
     def _set_up_multi_series_ometiff(self):
         self.multi_series_ometiff_path = Path("multi_series_tmp.ome.tiff")
@@ -69,17 +53,25 @@ def _set_up_multi_series_ometiff(self):
         </OME>""".encode()
         self.multi_series_ometiff_array0 = 255 * np.ones((1, 1, 1, 32, 64), np.uint8)
         self.multi_series_ometiff_array1 = np.zeros((1, 1, 1, 64, 32), np.uint8)
+
+        # apeeer-ome-tiff library does not support writing multi-page files
+        # hence test data needs to be created with tifffile directly
         with tifffile.TiffWriter(self.multi_series_ometiff_path) as tiff_writer:
-            tiff_writer.write(
+            tiff_writer.save(
                 self.multi_series_ometiff_array0,
                 photometric="minisblack",
-                metadata=None,
+                description=self.multi_series_metadata,
+                metadata={},
             )
-            tiff_writer.write(
+
+        with tifffile.TiffWriter(
+            self.multi_series_ometiff_path, append=True
+        ) as tiff_writer:
+            tiff_writer.save(
                 self.multi_series_ometiff_array1,
                 photometric="minisblack",
-                description=self.multi_series_metadata,
-                metadata=None,
+                subfiletype=1,
+                metadata={},
             )
 
     def tearDown(self) -> None:
@@ -105,3 +97,67 @@ def test_read_multi_series(self):
                 arrays,
                 [self.multi_series_ometiff_array0, self.multi_series_ometiff_array1],
             )
+
+
+class TestOmeTiffWrite(unittest.TestCase):
+    def setUp(self) -> None:
+        self._test_array = np.ones((1, 7, 2, 256, 256), dtype=np.uint8)
+        self._output_path = "test.ome.tiff"
+        self._big_test_array = None
+
+    def _set_up_big_test_array(self):
+        if self._big_test_array is not None:
+            return
+        # size of a square uint8 image that would be 4GB, hence will be treated as bigtiff upon a save
+        square_bigtiff_size = int(math.sqrt(4 * 2**30))
+        biggtiff_shape = (1, 1, 1, square_bigtiff_size, square_bigtiff_size)
+        self._big_test_array = np.random.randint(0, 255, biggtiff_shape, dtype=np.uint8)
+
+    def test_write(self):
+        write_ometiff(output_path=self._output_path, array=self._test_array)
+
+        with OmeTiffFile(self._output_path) as ome_tiff_file:
+            array, omexml_string = ome_tiff_file.read()
+            np.testing.assert_equal(array, self._test_array)
+
+    def test_write_compress(self):
+        write_ometiff(
+            output_path=self._output_path,
+            array=self._test_array,
+            compression="adobe_deflate",
+        )
+
+        with OmeTiffFile(self._output_path) as ome_tiff_file:
+            array, omexml_string = ome_tiff_file.read()
+            np.testing.assert_equal(array, self._test_array)
+            os.remove(self._output_path)
+
+    @unittest.skipUnless(
+        _RUN_SLOW_TESTS,
+        "Test is skipped by default because it is slow due to bigtiff data",
+    )
+    def test_write_bigtiff(self):
+        self._set_up_big_test_array()
+        write_ometiff(output_path=self._output_path, array=self._big_test_array)
+
+        with OmeTiffFile(self._output_path) as ome_tiff_file:
+            array, omexml_string = ome_tiff_file.read()
+            np.testing.assert_equal(array, self._big_test_array)
+            os.remove(self._output_path)
+
+    @unittest.skipUnless(
+        _RUN_SLOW_TESTS,
+        "Test is skipped by default because it is slow due to bigtiff data",
+    )
+    def test_write_bigtiff_compress(self):
+        self._set_up_big_test_array()
+        write_ometiff(
+            output_path=self._output_path,
+            array=self._big_test_array,
+            compression="adobe_deflate",
+        )
+
+        with OmeTiffFile(self._output_path) as ome_tiff_file:
+            array, omexml_string = ome_tiff_file.read()
+            np.testing.assert_equal(array, self._big_test_array)
+            os.remove(self._output_path)