VFS pushdown to exportation (#132)

* VFS pushdown to exportation and minio testing

* Misc

* Support source dest configs ingestion (#134)

* Support source and dest configurations

* Adding backwards compatibility for config arg overriding the source and dest

Author: ktsitsi

tiledb/bioimg/converters/base.py

@@ -50,12 +50,10 @@
 )
 from ..openslide import TileDBOpenSlide
 from ..version import version as PKG_VERSION
-from . import DATASET_TYPE, FMT_VERSION
+from . import DATASET_TYPE, DEFAULT_SCRATCH_SPACE, FMT_VERSION
 from .axes import Axes
 from .tiles import iter_tiles, num_tiles
 
-DEFAULT_SCRATCH_SPACE = "/dev/shm"
-
 
 class ImageReader(ABC):
     @abstractmethod
@@ -64,8 +62,10 @@ def __init__(
         input_path: str,
         *,
         logger: Optional[logging.Logger],
-        config: Optional[tiledb.Config] = None,
-        ctx: Optional[tiledb.Ctx] = None,
+        source_config: Optional[tiledb.Config] = None,
+        source_ctx: Optional[tiledb.Ctx] = None,
+        dest_config: Optional[tiledb.Config] = None,
+        dest_ctx: Optional[tiledb.Config] = None,
         **kwargs: Any,
     ):
         """Initialize this ImageReader"""
@@ -76,6 +76,16 @@ def __enter__(self) -> ImageReader:
     def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         pass
 
+    @property
+    @abstractmethod
+    def source_ctx(self) -> tiledb.Ctx:
+        """The ctx of the source path of this image reader."""
+
+    @property
+    @abstractmethod
+    def dest_ctx(self) -> tiledb.Ctx:
+        """The ctx of the dest path of this image reader."""
+
     @property
     @abstractmethod
     def logger(self) -> Optional[logging.Logger]:
@@ -309,7 +319,7 @@ def to_tiledb(
         exclude_metadata: bool = False,
         compressor: Optional[Union[Mapping[int, Any], Any]] = None,
         log: Optional[Union[bool, logging.Logger]] = None,
-        reader_kwargs: Optional[Mapping[str, Any]] = None,
+        reader_kwargs: Optional[MutableMapping[str, Any]] = None,
         pyramid_kwargs: Optional[Mapping[str, Any]] = None,
     ) -> Type[ImageConverter]:
         """
@@ -360,6 +370,15 @@ def to_tiledb(
             default_verbose = False
             logger = get_logger_wrapper(default_verbose)
 
+        # Backwards compatibility config v0.2.13
+        if reader_kwargs:
+            common_cfg = reader_kwargs.get("config", None)
+            if common_cfg:
+                # Overwrite the source and destination configs with the common
+                reader_kwargs["source_config"] = reader_kwargs[
+                    "dest_config"
+                ] = common_cfg
+
         if isinstance(source, ImageReader):
             if cls._ImageReaderType != source.__class__:
                 raise ValueError("Image reader should match converter on source format")
@@ -379,7 +398,7 @@ def to_tiledb(
             max_tiles.update(tiles)
         logger.debug(f"Updated max tiles:{max_tiles}")
 
-        rw_group = ReadWriteGroup(output_path)
+        rw_group = ReadWriteGroup(output_path, ctx=reader.dest_ctx)
 
         metadata = {}
         original_metadata = {}
@@ -498,7 +517,10 @@ def to_tiledb(
                 fmt_version=FMT_VERSION,
                 channels=json.dumps(reader.channels),
                 levels=jsonpickle.encode(
-                    sorted(iter_levels_meta(rw_group.r_group), key=itemgetter("level")),
+                    sorted(
+                        iter_levels_meta(rw_group.r_group, ctx=reader.dest_ctx),
+                        key=itemgetter("level"),
+                    ),
                     unpicklable=False,
                 ),
                 metadata=jsonpickle.encode(metadata, unpicklable=False),
@@ -579,9 +601,9 @@ def _convert_level_to_tiledb(
     # get or create TileDB array uri
     uri, created = rw_group.get_or_create(f"l_{level}.tdb", schema)
 
-    if created or not validate_ingestion(uri):
+    if created or not validate_ingestion(uri, ctx=reader.dest_ctx):
         # write image and metadata to TileDB array
-        with open_bioimg(uri, "w") as out_array:
+        with open_bioimg(uri, "w", ctx=reader.dest_ctx) as out_array:
             out_array.meta.update(reader.level_metadata(level), level=level)
             inv_axes_mapper = axes_mapper.inverse
             if chunked:

tiledb/bioimg/converters/ome_tiff.py

@@ -32,8 +32,10 @@ def __init__(
         input_path: str,
         *,
         logger: Optional[logging.Logger] = None,
-        config: Optional[Config] = None,
-        ctx: Optional[Ctx] = None,
+        source_config: Optional[Config] = None,
+        source_ctx: Optional[Ctx] = None,
+        dest_config: Optional[Config] = None,
+        dest_ctx: Optional[Ctx] = None,
         extra_tags: Sequence[Union[str, int]] = (),
     ):
         """
@@ -47,9 +49,11 @@ def __init__(
 
         # Use VFS for all paths local or remote for reading the input image
         self._input_path = input_path
-        self._ctx = _get_ctx(ctx, config)
-        self._cfg = self._ctx.config()
-        self._vfs = VFS(config=self._cfg, ctx=self._ctx)
+        self._source_ctx = _get_ctx(source_ctx, source_config)
+        self._source_cfg = self._source_ctx.config()
+        self._dest_ctx = _get_ctx(dest_ctx, dest_config)
+        self._dest_cfg = self._dest_ctx.config()
+        self._vfs = VFS(config=self._source_cfg, ctx=self._source_ctx)
         self._vfs_fh = self._vfs.open(input_path, mode="rb")
         self._tiff = tifffile.TiffFile(self._vfs_fh)
         # XXX ignore all but the first series
@@ -61,6 +65,14 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         self._tiff.close()
         self._vfs.close(file=self._vfs_fh)
 
+    @property
+    def source_ctx(self) -> Ctx:
+        return self._source_ctx
+
+    @property
+    def dest_ctx(self) -> Ctx:
+        return self._dest_ctx
+
     @property
     def logger(self) -> Optional[logging.Logger]:
         return self._logger

tiledb/bioimg/converters/ome_zarr.py

@@ -36,24 +36,36 @@ def __init__(
         input_path: str,
         *,
         logger: Optional[logging.Logger] = None,
-        config: Optional[Config] = None,
-        ctx: Optional[Ctx] = None,
+        source_config: Optional[Config] = None,
+        source_ctx: Optional[Ctx] = None,
+        dest_config: Optional[Config] = None,
+        dest_ctx: Optional[Ctx] = None,
     ):
         """
         OME-Zarr image reader
         :param input_path: The path to the Zarr image
         """
         self._logger = get_logger_wrapper(False) if not logger else logger
-        self._ctx = _get_ctx(ctx, config)
-        self._cfg = self._ctx.config()
-        storage_options = translate_config_to_s3fs(self._cfg)
+        self._source_ctx = _get_ctx(source_ctx, source_config)
+        self._source_cfg = self._source_ctx.config()
+        self._dest_ctx = _get_ctx(dest_ctx, dest_config)
+        self._dest_cfg = self._dest_ctx.config()
+        storage_options = translate_config_to_s3fs(self._source_cfg)
         input_fh = zarr.storage.FSStore(
             input_path, check=True, create=True, **storage_options
         )
         self._root_node = next(Reader(ZarrLocation(input_fh))())
         self._multiscales = cast(Multiscales, self._root_node.load(Multiscales))
         self._omero = cast(Optional[OMERO], self._root_node.load(OMERO))
 
+    @property
+    def source_ctx(self) -> Ctx:
+        return self._source_ctx
+
+    @property
+    def dest_ctx(self) -> Ctx:
+        return self._dest_ctx
+
     @property
     def logger(self) -> Optional[logging.Logger]:
         return self._logger

tiledb/bioimg/converters/openslide.py

@@ -43,21 +43,25 @@ def __init__(
         input_path: str,
         *,
         logger: Optional[logging.Logger] = None,
-        config: Optional[Config] = None,
-        ctx: Optional[Ctx] = None,
+        source_config: Optional[Config] = None,
+        source_ctx: Optional[Ctx] = None,
+        dest_config: Optional[Config] = None,
+        dest_ctx: Optional[Ctx] = None,
         scratch_space: str = DEFAULT_SCRATCH_SPACE,
     ):
         """
         OpenSlide image reader
         :param input_path: The path to the OpenSlide image
 
         """
-        self._ctx = _get_ctx(ctx, config)
-        self._cfg = self._ctx.config()
+        self._source_ctx = _get_ctx(source_ctx, source_config)
+        self._source_cfg = self._source_ctx.config()
+        self._dest_ctx = _get_ctx(dest_ctx, dest_config)
+        self._dest_cfg = self._dest_ctx.config()
         self._logger = get_logger_wrapper(False) if not logger else logger
         if is_remote_protocol(input_path):
             resolved_path = cache_filepath(
-                input_path, config, ctx, self._logger, scratch_space
+                input_path, source_config, source_ctx, self._logger, scratch_space
             )
         else:
             resolved_path = input_path
@@ -66,6 +70,14 @@ def __init__(
     def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         self._osd.close()
 
+    @property
+    def source_ctx(self) -> Ctx:
+        return self._source_ctx
+
+    @property
+    def dest_ctx(self) -> Ctx:
+        return self._dest_ctx
+
     @property
     def logger(self) -> Optional[logging.Logger]:
         return self._logger

tiledb/bioimg/helpers.py

@@ -61,7 +61,7 @@ def get_or_create(self, name: str, schema: tiledb.ArraySchema) -> Tuple[str, boo
         else:
             uri = os.path.join(self._uri, name).replace("\\", "/")
 
-            if not tiledb.array_exists(uri):
+            if not tiledb.array_exists(uri, ctx=self._ctx):
                 tiledb.Array.create(uri, schema, ctx=self._ctx)
                 create = True
             else:
@@ -90,7 +90,7 @@ def get_or_create(self, name: str, schema: tiledb.ArraySchema) -> Tuple[str, boo
         return uri, create
 
 
-def validate_ingestion(uri: str) -> bool:
+def validate_ingestion(uri: str, ctx: tiledb.Ctx = None) -> bool:
     """
     This function validates that they array has been stored properly
     by checking the existence of array fragments and
@@ -103,12 +103,12 @@ def validate_ingestion(uri: str) -> bool:
     Returns boolean
     -------
     """
-    fragments_list_info = tiledb.array_fragments(uri)
+    fragments_list_info = tiledb.array_fragments(uri, ctx=ctx)
     if not len(fragments_list_info):
         # If no fragments are present
         return False
     else:
-        with tiledb.open(uri) as validation_array:
+        with tiledb.open(uri, ctx=ctx) as validation_array:
             ned = fragments_list_info.nonempty_domain
             consolidated_ranges = merge_ned_ranges(ned)
             domains = [d.domain for d in validation_array.schema.domain]

tiledb/bioimg/openslide.py

@@ -15,6 +15,7 @@
 
 import tiledb
 from tiledb import Config, Ctx
+from tiledb.highlevel import _get_ctx
 
 from . import ATTR_NAME
 from .converters.axes import Axes
@@ -43,7 +44,9 @@ def __init__(
 
         :param uri: uri of a tiledb.Group containing the image
         """
-        self._group = tiledb.Group(uri, config=config, ctx=ctx)
+        self._ctx = _get_ctx(ctx, config)
+        self._cfg = self._ctx.config()
+        self._group = tiledb.Group(uri, ctx=self._ctx)
         pixel_depth = self._group.meta.get("pixel_depth", "")
         pixel_depth = dict(json.loads(pixel_depth)) if pixel_depth else {}
         self._levels = sorted(

tiledb/bioimg/wrappers.py

@@ -47,7 +47,18 @@ def from_bioimg(
     :param kwargs: keyword arguments for custom ingestion behaviour
     :return: The converter class that was used for the ingestion
     """
+
     logger = get_logger_wrapper(verbose)
+    reader_kwargs = kwargs.get("reader_kwargs", {})
+
+    # Get the config for the source
+    reader_kwargs["source_config"] = kwargs.pop("source_config", None)
+
+    # Get the config for the destination (if exists) otherwise match it with source config
+    reader_kwargs["dest_config"] = kwargs.pop(
+        "dest_config", reader_kwargs["source_config"]
+    )
+
     if converter is Converters.OMETIFF:
         if not _tiff_exc:
             logger.info("Converting OME-TIFF file")
@@ -57,6 +68,7 @@ def from_bioimg(
                 log=logger,
                 exclude_metadata=exclude_metadata,
                 tile_scale=tile_scale,
+                reader_kwargs=reader_kwargs,
                 **kwargs,
             )
         else:
@@ -70,6 +82,7 @@ def from_bioimg(
                 log=logger,
                 exclude_metadata=exclude_metadata,
                 tile_scale=tile_scale,
+                reader_kwargs=reader_kwargs,
                 **kwargs,
             )
         else:
@@ -83,6 +96,7 @@ def from_bioimg(
                 log=logger,
                 exclude_metadata=exclude_metadata,
                 tile_scale=tile_scale,
+                reader_kwargs=reader_kwargs,
                 **kwargs,
             )
         else: