From 8a464940ae6280494a1288d98a4827ddabc5b75b Mon Sep 17 00:00:00 2001 From: Emil Nowosielski Date: Thu, 23 Jun 2022 15:53:09 +0200 Subject: [PATCH 1/8] Fix big tiff issue --- apeer_ometiff_library/io.py | 9 +++------ requirements.txt | 2 -- setup.py | 4 ++-- tests/test_io.py | 39 ++++++++++++++++++++++++++++--------- tests/test_processing.py | 2 -- 5 files changed, 35 insertions(+), 21 deletions(-) delete mode 100644 requirements.txt diff --git a/apeer_ometiff_library/io.py b/apeer_ometiff_library/io.py index 2db3f7c..0f0acf1 100755 --- a/apeer_ometiff_library/io.py +++ b/apeer_ometiff_library/io.py @@ -230,9 +230,6 @@ def write_ometiff(output_path, array, omexml_string = None, compression=None): if omexml_string is None: omexml_string = gen_xml(array) - if sys.version < "3.7": - tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, - compress=compression) - else: - tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, - compression=compression) + tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, + compress=compression) + diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 89c1aa7..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -numpy==1.16.5 -tifffile==2020.9.3 \ No newline at end of file diff --git a/setup.py b/setup.py index b3af5fc..b2ccc53 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,12 @@ from setuptools import setup setup(name='apeer-ometiff-library', - version='1.9.0', + version='1.10.0', description='Library to read and write ometiff images', url='https://github.com/apeer-micro/apeer-ometiff-library', author='apeer-micro', packages=setuptools.find_packages(), - install_requires=['numpy','tifffile'], + install_requires=['numpy==1.18.5','tifffile==2020.6.3'], license='MIT', classifiers=[ "Programming Language :: Python :: 3", diff --git a/tests/test_io.py b/tests/test_io.py index f260277..6cfe53f 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,10 +1,11 @@ +import os import unittest from pathlib import Path import numpy as np import tifffile -from apeer_ometiff_library.io import OmeTiffFile +from apeer_ometiff_library.io import OmeTiffFile, write_ometiff class TestOmeTiffFile(unittest.TestCase): @@ -21,7 +22,7 @@ def _set_up_ometiff(self): UUID="urn:uuid:11227ecd-e960-42e4-95c8-39e6cec94150"> 2022-05-13T11:25:25.212054 - @@ -30,13 +31,13 @@ def _set_up_ometiff(self): """.encode() - self.ometiff_array = 255 * np.ones((1, 1, 1, 32, 64), np.uint8) + self.ometiff_array = 255 * np.ones((2, 3, 4, 32, 64), np.uint8) with tifffile.TiffWriter(self.ometiff_path) as tiff_writer: - tiff_writer.write( + tiff_writer.save( self.ometiff_array, photometric="minisblack", description=self.ome_tiff_metadata, - metadata=None, + metadata={}, ) def _set_up_multi_series_ometiff(self): @@ -70,16 +71,16 @@ def _set_up_multi_series_ometiff(self): self.multi_series_ometiff_array0 = 255 * np.ones((1, 1, 1, 32, 64), np.uint8) self.multi_series_ometiff_array1 = np.zeros((1, 1, 1, 64, 32), np.uint8) with tifffile.TiffWriter(self.multi_series_ometiff_path) as tiff_writer: - tiff_writer.write( + tiff_writer.save( self.multi_series_ometiff_array0, photometric="minisblack", - metadata=None, + metadata={}, ) - tiff_writer.write( + tiff_writer.save( self.multi_series_ometiff_array1, photometric="minisblack", description=self.multi_series_metadata, - metadata=None, + metadata={}, ) def tearDown(self) -> None: @@ -105,3 +106,23 @@ def test_read_multi_series(self): arrays, [self.multi_series_ometiff_array0, self.multi_series_ometiff_array1], ) + +class TestOmeTiffWrite(unittest.TestCase): + def setUp(self) -> None: + self._test_array = np.ones((1, 7, 2, 256, 256)) + self._output_path = 'test.ome.tiff' + + def test_write(self): + write_ometiff(output_path=self._output_path, array=self._test_array) + + with OmeTiffFile(self._output_path) as ome_tiff_file: + array, omexml_string = ome_tiff_file.read() + np.testing.assert_equal(array, self._test_array) + + def test_write_compress(self): + write_ometiff(output_path=self._output_path, array=self._test_array, compression="adobe_deflate") + + with OmeTiffFile(self._output_path) as ome_tiff_file: + array, omexml_string = ome_tiff_file.read() + np.testing.assert_equal(array, self._test_array) + os.remove(self._output_path) \ No newline at end of file diff --git a/tests/test_processing.py b/tests/test_processing.py index 362e29f..a3e3a5a 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -1,12 +1,10 @@ import unittest -import mock import sys import os sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import itertools as it import numpy as np -import skimage from apeer_ometiff_library import io class TestsCore(unittest.TestCase): From 4b80757aab398b539b3ad693e494bda02b69076f Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Mon, 27 Jun 2022 11:24:03 +0200 Subject: [PATCH 2/8] fix requirements for the package --- azure-pipelines.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1cdf8fb..b2642ca 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ steps: - script: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install . pip install twine setuptools wheel displayName: 'Install dependencies' diff --git a/setup.py b/setup.py index b2ccc53..7068919 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ url='https://github.com/apeer-micro/apeer-ometiff-library', author='apeer-micro', packages=setuptools.find_packages(), - install_requires=['numpy==1.18.5','tifffile==2020.6.3'], + install_requires=['numpy==1.18.5','tifffile==2020.6.3', 'imagecodecs==2020.5.30'], license='MIT', classifiers=[ "Programming Language :: Python :: 3", From f65486896516a515ed69285610d7713cdaec15ed Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Mon, 27 Jun 2022 11:25:20 +0200 Subject: [PATCH 3/8] make multi-page reading compatible with tifffile==2020.6.3 --- apeer_ometiff_library/io.py | 4 ++-- tests/test_io.py | 32 +++++++++----------------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/apeer_ometiff_library/io.py b/apeer_ometiff_library/io.py index 0f0acf1..75c384c 100755 --- a/apeer_ometiff_library/io.py +++ b/apeer_ometiff_library/io.py @@ -43,9 +43,9 @@ def read_multi_series(self) -> Tuple[List[np.ndarray], str]: omexml_string = self._tiff_file.ome_metadata arrays = [ _ensure_correct_dimensions( - self._tiff_file.asarray(series=series), omexml_string + self._tiff_file.asarray(key=image_key), omexml_string ) - for series in range( + for image_key in range( omexmlClass.OMEXML(self._tiff_file.ome_metadata).image_count ) ] diff --git a/tests/test_io.py b/tests/test_io.py index 6cfe53f..dc0776c 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -16,29 +16,9 @@ def setUp(self): def _set_up_ometiff(self): self.ometiff_path = Path("tmp.ome.tiff") self.ometiff_path.touch() - self.ome_tiff_metadata = """ - - - 2022-05-13T11:25:25.212054 - - - - - - - - """.encode() self.ometiff_array = 255 * np.ones((2, 3, 4, 32, 64), np.uint8) - with tifffile.TiffWriter(self.ometiff_path) as tiff_writer: - tiff_writer.save( - self.ometiff_array, - photometric="minisblack", - description=self.ome_tiff_metadata, - metadata={}, - ) + write_ometiff(str(self.ometiff_path), self.ometiff_array) + def _set_up_multi_series_ometiff(self): self.multi_series_ometiff_path = Path("multi_series_tmp.ome.tiff") @@ -70,16 +50,22 @@ def _set_up_multi_series_ometiff(self): """.encode() self.multi_series_ometiff_array0 = 255 * np.ones((1, 1, 1, 32, 64), np.uint8) self.multi_series_ometiff_array1 = np.zeros((1, 1, 1, 64, 32), np.uint8) + + # apeeer-ome-tiff library does not support writing multi-page files + # hence test data needs to be created with tifffile directly with tifffile.TiffWriter(self.multi_series_ometiff_path) as tiff_writer: tiff_writer.save( self.multi_series_ometiff_array0, photometric="minisblack", + description=self.multi_series_metadata, metadata={}, ) + + with tifffile.TiffWriter(self.multi_series_ometiff_path, append=True) as tiff_writer: tiff_writer.save( self.multi_series_ometiff_array1, photometric="minisblack", - description=self.multi_series_metadata, + subfiletype=1, metadata={}, ) From 42f2a6b11f79a314cc1d8a54df725092eecc4c80 Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Mon, 27 Jun 2022 11:47:00 +0200 Subject: [PATCH 4/8] fix retry write for bigtiffs --- apeer_ometiff_library/io.py | 10 ++++-- tests/test_io.py | 61 +++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/apeer_ometiff_library/io.py b/apeer_ometiff_library/io.py index 75c384c..956b537 100755 --- a/apeer_ometiff_library/io.py +++ b/apeer_ometiff_library/io.py @@ -1,3 +1,4 @@ +import struct from pathlib import Path from types import TracebackType from typing import Union, Optional, Type, Tuple, List @@ -230,6 +231,11 @@ def write_ometiff(output_path, array, omexml_string = None, compression=None): if omexml_string is None: omexml_string = gen_xml(array) - tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, - compress=compression) + try: + tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, + compress=compression) + except struct.error: + tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, + compress=compression, bigtiff=True) + diff --git a/tests/test_io.py b/tests/test_io.py index dc0776c..cbdd577 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,3 +1,4 @@ +import math import os import unittest from pathlib import Path @@ -7,6 +8,9 @@ from apeer_ometiff_library.io import OmeTiffFile, write_ometiff +# Change to False if you want to skip log running tests +_RUN_SLOW_TESTS = True + class TestOmeTiffFile(unittest.TestCase): def setUp(self): @@ -19,7 +23,6 @@ def _set_up_ometiff(self): self.ometiff_array = 255 * np.ones((2, 3, 4, 32, 64), np.uint8) write_ometiff(str(self.ometiff_path), self.ometiff_array) - def _set_up_multi_series_ometiff(self): self.multi_series_ometiff_path = Path("multi_series_tmp.ome.tiff") self.multi_series_ometiff_path.touch() @@ -61,7 +64,9 @@ def _set_up_multi_series_ometiff(self): metadata={}, ) - with tifffile.TiffWriter(self.multi_series_ometiff_path, append=True) as tiff_writer: + with tifffile.TiffWriter( + self.multi_series_ometiff_path, append=True + ) as tiff_writer: tiff_writer.save( self.multi_series_ometiff_array1, photometric="minisblack", @@ -93,10 +98,20 @@ def test_read_multi_series(self): [self.multi_series_ometiff_array0, self.multi_series_ometiff_array1], ) + class TestOmeTiffWrite(unittest.TestCase): def setUp(self) -> None: - self._test_array = np.ones((1, 7, 2, 256, 256)) - self._output_path = 'test.ome.tiff' + self._test_array = np.ones((1, 7, 2, 256, 256), dtype=np.uint8) + self._output_path = "test.ome.tiff" + self._big_test_array = None + + def _set_up_big_test_array(self): + if self._big_test_array is not None: + return + # size of a square uint8 image that would be 4GB, hence will be treated as bigtiff upon a save + square_bigtiff_size = int(math.sqrt(4 * 2**30)) + biggtiff_shape = (1, 1, 1, square_bigtiff_size, square_bigtiff_size) + self._big_test_array = np.random.randint(0, 255, biggtiff_shape, dtype=np.uint8) def test_write(self): write_ometiff(output_path=self._output_path, array=self._test_array) @@ -106,9 +121,43 @@ def test_write(self): np.testing.assert_equal(array, self._test_array) def test_write_compress(self): - write_ometiff(output_path=self._output_path, array=self._test_array, compression="adobe_deflate") + write_ometiff( + output_path=self._output_path, + array=self._test_array, + compression="adobe_deflate", + ) with OmeTiffFile(self._output_path) as ome_tiff_file: array, omexml_string = ome_tiff_file.read() np.testing.assert_equal(array, self._test_array) - os.remove(self._output_path) \ No newline at end of file + os.remove(self._output_path) + + @unittest.skipUnless( + _RUN_SLOW_TESTS, + "Test is skipped by default because it is slow due to bigtiff data", + ) + def test_write_bigtiff(self): + self._set_up_big_test_array() + write_ometiff(output_path=self._output_path, array=self._big_test_array) + + with OmeTiffFile(self._output_path) as ome_tiff_file: + array, omexml_string = ome_tiff_file.read() + np.testing.assert_equal(array, self._big_test_array) + os.remove(self._output_path) + + @unittest.skipUnless( + _RUN_SLOW_TESTS, + "Test is skipped by default because it is slow due to bigtiff data", + ) + def test_write_bigtiff_compress(self): + self._set_up_big_test_array() + write_ometiff( + output_path=self._output_path, + array=self._big_test_array, + compression="adobe_deflate", + ) + + with OmeTiffFile(self._output_path) as ome_tiff_file: + array, omexml_string = ome_tiff_file.read() + np.testing.assert_equal(array, self._big_test_array) + os.remove(self._output_path) From 7cad7a14b0ab12a19c7c474ac08a8d7a9fd3e94d Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Mon, 27 Jun 2022 11:50:01 +0200 Subject: [PATCH 5/8] add running tests upon a build --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b2642ca..196f8f1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,8 @@ steps: python -m pip install --upgrade pip pip install . pip install twine setuptools wheel - displayName: 'Install dependencies' + python -m unittest discover tests/ + displayName: 'Install dependencies and run tests' - script: | python setup.py sdist bdist_wheel From 9e8aed0cf465f4d08799eca8833e97b3dc342051 Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Mon, 27 Jun 2022 11:52:33 +0200 Subject: [PATCH 6/8] remove useless test module --- tests/test_processing.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 tests/test_processing.py diff --git a/tests/test_processing.py b/tests/test_processing.py deleted file mode 100644 index a3e3a5a..0000000 --- a/tests/test_processing.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest -import sys -import os -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -import itertools as it -import numpy as np -from apeer_ometiff_library import io - -class TestsCore(unittest.TestCase): - - def test_funcName_condition_expectedResult(self): - pass - -if __name__ == '__main__': - unittest.main() \ No newline at end of file From 0c1623d10ec13db9a3edc49cdf8352dd897d5cbb Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Mon, 27 Jun 2022 12:06:10 +0200 Subject: [PATCH 7/8] skip slow tests by default due to OOMs on the build machine --- tests/test_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_io.py b/tests/test_io.py index cbdd577..9229d5d 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -8,8 +8,8 @@ from apeer_ometiff_library.io import OmeTiffFile, write_ometiff -# Change to False if you want to skip log running tests -_RUN_SLOW_TESTS = True +# Change to True if you want to include log running tests in the test suite +_RUN_SLOW_TESTS = False class TestOmeTiffFile(unittest.TestCase): From cd844a6ae90421c13b3f11a830a48ce5fdcad1c8 Mon Sep 17 00:00:00 2001 From: Jerzy Kowalski Date: Tue, 28 Jun 2022 10:34:57 +0200 Subject: [PATCH 8/8] check whether data is a bigtiff candidate to save execution time --- apeer_ometiff_library/io.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/apeer_ometiff_library/io.py b/apeer_ometiff_library/io.py index 956b537..3f11c1b 100755 --- a/apeer_ometiff_library/io.py +++ b/apeer_ometiff_library/io.py @@ -231,11 +231,8 @@ def write_ometiff(output_path, array, omexml_string = None, compression=None): if omexml_string is None: omexml_string = gen_xml(array) - try: - tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, - compress=compression) - except struct.error: - tifffile.imwrite(output_path, array, photometric = "minisblack", description=omexml_string, metadata=None, - compress=compression, bigtiff=True) - - + data_limit_nbytes = 2 ** 32 + tiff_metadata_nbytes = 2 ** 25 + bigtiff = array.nbytes > data_limit_nbytes - tiff_metadata_nbytes + tifffile.imwrite(output_path, array, photometric="minisblack", description=omexml_string, metadata=None, + compress=compression, bigtiff=bigtiff)