Exclude metadata from ingestion (#106)

* Excluding original metadata from_bioimg path

* Install openslide-tools failing tests

* Test CI failing

Author: ktsitsi

.github/workflows/ci.yml

@@ -32,7 +32,9 @@ jobs:
           pytest-mock
     
     - name: Install openslide for non-Win
-      run: micromamba install openslide
+      run: |
+        sudo apt install openslide-tools
+        micromamba install openslide
       if: matrix.os != 'windows-latest'
 
     - name: Install openslide for Win 
@@ -61,9 +63,10 @@ jobs:
         # default is C: ,thus we create a temp_test folder for pytest's tmp_dir to run on D: as well 
         if [ "$RUNNER_OS" == "Linux" ]; then
           pytest -v --cov=tiledb --cov-report=term-missing --durations=0 tests/ > coverage.txt
-          cat coverage.txt
+          exit_code=$?
           TEST_COVERAGE="$(grep '^TOTAL' coverage.txt | awk -v N=4 '{print $N}')"
           echo "COVERAGE=$TEST_COVERAGE" >> $GITHUB_OUTPUT
+          exit $exit_code
         else
           mkdir test_temp
           pytest --basetemp=test_temp -v --cov=tiledb --cov-report=term-missing --durations=0 tests/

tests/__init__.py

@@ -14,6 +14,10 @@ def get_schema(x_size, y_size, c_size=3, compressor=tiledb.ZstdFilter(level=0)):
     dims = []
     x_tile = min(x_size, 1024)
     y_tile = min(y_size, 1024)
+    # WEBP Compressor does not accept specific dtypes so for dimensions we use the default
+    dim_compressor = tiledb.ZstdFilter(level=0)
+    if not isinstance(compressor, tiledb.WebpFilter):
+        dim_compressor = compressor
     if isinstance(compressor, tiledb.WebpFilter):
         x_size *= c_size
         x_tile *= c_size
@@ -30,10 +34,34 @@ def get_schema(x_size, y_size, c_size=3, compressor=tiledb.ZstdFilter(level=0)):
                 lossless=compressor.lossless,
             )
     else:
-        dims.append(tiledb.Dim("C", (0, c_size - 1), tile=c_size, dtype=np.uint32))
+        dims.append(
+            tiledb.Dim(
+                "C",
+                (0, c_size - 1),
+                tile=c_size,
+                dtype=np.uint32,
+                filters=tiledb.FilterList([compressor]),
+            )
+        )
 
-    dims.append(tiledb.Dim("Y", (0, y_size - 1), tile=y_tile, dtype=np.uint32))
-    dims.append(tiledb.Dim("X", (0, x_size - 1), tile=x_tile, dtype=np.uint32))
+    dims.append(
+        tiledb.Dim(
+            "Y",
+            (0, y_size - 1),
+            tile=y_tile,
+            dtype=np.uint32,
+            filters=tiledb.FilterList([dim_compressor]),
+        )
+    )
+    dims.append(
+        tiledb.Dim(
+            "X",
+            (0, x_size - 1),
+            tile=x_tile,
+            dtype=np.uint32,
+            filters=tiledb.FilterList([dim_compressor]),
+        )
+    )
 
     return tiledb.ArraySchema(
         domain=tiledb.Domain(*dims),

tests/integration/converters/test_ome_tiff.py

@@ -96,6 +96,43 @@ def test_ome_tiff_converter_group_metadata(tmp_path, filename):
             assert shape[level_axes.index("Y")] == level_height
 
 
+@pytest.mark.parametrize(
+    "filename,num_series", [("CMU-1-Small-Region.ome.tiff", 3), ("UTM2GTIF.tiff", 1)]
+)
+@pytest.mark.parametrize("preserve_axes", [False, True])
+@pytest.mark.parametrize("chunked,max_workers", [(False, 0), (True, 0), (True, 4)])
+@pytest.mark.parametrize(
+    "compressor",
+    [
+        tiledb.ZstdFilter(level=0),
+        tiledb.WebpFilter(WebpInputFormat.WEBP_RGB, lossless=False),
+        tiledb.WebpFilter(WebpInputFormat.WEBP_RGB, lossless=True),
+        tiledb.WebpFilter(WebpInputFormat.WEBP_NONE, lossless=True),
+    ],
+)
+def test_ome_tiff_converter_exclude_original_metadata(
+    tmp_path, filename, num_series, preserve_axes, chunked, max_workers, compressor
+):
+    if isinstance(compressor, tiledb.WebpFilter) and filename == "UTM2GTIF.tiff":
+        pytest.skip(f"WebPFilter cannot be applied to {filename}")
+
+    input_path = get_path(filename)
+    tiledb_path = tmp_path / "to_tiledb"
+    OMETiffConverter.to_tiledb(
+        input_path,
+        str(tiledb_path),
+        preserve_axes=preserve_axes,
+        chunked=chunked,
+        max_workers=max_workers,
+        compressor=compressor,
+        log=False,
+        exclude_metadata=True,
+    )
+
+    with TileDBOpenSlide(str(tiledb_path)) as t:
+        assert t.properties["original_metadata"] == "{}"
+
+
 @pytest.mark.parametrize(
     "filename,num_series", [("CMU-1-Small-Region.ome.tiff", 3), ("UTM2GTIF.tiff", 1)]
 )

tiledb/bioimg/converters/base.py

@@ -296,6 +296,7 @@ def to_tiledb(
         preserve_axes: bool = False,
         chunked: bool = False,
         max_workers: int = 0,
+        exclude_metadata: bool = False,
         compressor: Optional[Union[Mapping[int, Any], Any]] = None,
         log: Optional[Union[bool, logging.Logger]] = None,
         reader_kwargs: Optional[Mapping[str, Any]] = None,
@@ -317,6 +318,7 @@ def to_tiledb(
             original ones.
         :param max_workers: Maximum number of threads that can be used for conversion.
             Applicable only if chunked=True.
+        :param exclude_metadata: If true, drop original metadata of the images and exclude them from being ingested.
         :param compressor: TileDB compression filter mapping for each level
         :param log: verbose logging, defaults to None. Allows passing custom logging.Logger or boolean.
             If None or bool=False it initiates an INFO level logging. If bool=True then a logger is instantiated in
@@ -465,7 +467,8 @@ def to_tiledb(
             metadata["channels"] = {f"{ATTR_NAME}": metadata["channels"]}
             logger.debug(f'Metadata channels: {metadata["channels"]}')
 
-            original_metadata = reader.original_metadata
+            if not exclude_metadata:
+                original_metadata = reader.original_metadata
 
         with rw_group:
             rw_group.w_group.meta.update(

tiledb/bioimg/helpers.py

@@ -97,9 +97,19 @@ def get_schema(
 
     dims = []
     assert len(dim_names) == len(dim_shape), (dim_names, dim_shape)
+    # WEBP Compressor does not accept specific dtypes so for dimensions we use the default
+    dim_compressor = tiledb.ZstdFilter(level=0)
+    if not isinstance(compressor, tiledb.WebpFilter):
+        dim_compressor = compressor
     for dim_name, dim_size in zip(dim_names, dim_shape):
         dim_tile = min(dim_size, max_tiles[dim_name])
-        dim = tiledb.Dim(dim_name, (0, dim_size - 1), dim_tile, dtype=dim_dtype)
+        dim = tiledb.Dim(
+            dim_name,
+            (0, dim_size - 1),
+            dim_tile,
+            dtype=dim_dtype,
+            filters=[dim_compressor],
+        )
         dims.append(dim)
     attr = tiledb.Attr(name=ATTR_NAME, dtype=attr_dtype, filters=[compressor])
     return tiledb.ArraySchema(domain=tiledb.Domain(*dims), attrs=[attr])

tiledb/bioimg/wrappers.py

@@ -16,6 +16,7 @@ def from_bioimg(
     converter: Converters = Converters.OMETIFF,
     *,
     verbose: bool = False,
+    exclude_metadata: bool = False,
     **kwargs: Any,
 ) -> Type[ImageConverter]:
     """
@@ -32,17 +33,29 @@ def from_bioimg(
     if converter is Converters.OMETIFF:
         logger.info("Converting OME-TIFF file")
         return OMETiffConverter.to_tiledb(
-            source=src, output_path=dest, log=logger, **kwargs
+            source=src,
+            output_path=dest,
+            log=logger,
+            exclude_metadata=exclude_metadata,
+            **kwargs,
         )
     elif converter is Converters.OMEZARR:
         logger.info("Converting OME-Zarr file")
         return OMEZarrConverter.to_tiledb(
-            source=src, output_path=dest, log=logger, **kwargs
+            source=src,
+            output_path=dest,
+            log=logger,
+            exclude_metadata=exclude_metadata,
+            **kwargs,
         )
     else:
         logger.info("Converting Openslide")
         return OpenSlideConverter.to_tiledb(
-            source=src, output_path=dest, log=logger, **kwargs
+            source=src,
+            output_path=dest,
+            log=logger,
+            exclude_metadata=exclude_metadata,
+            **kwargs,
         )