SC67 remove datasets dependency (#148)

* SC67 remove datasets dependency

* SC_67 remove datasets depedency

* SC_67 bump spatialdata

* lock file

* uv lock in cache

* run tests

* update registry

* uv lock

* sync locked before running tests

* update matrix ci cd

Author: ArneDefauw

.github/workflows/run_tests.yml

@@ -43,16 +43,25 @@ jobs:
         with:
           filter: blob:none
           fetch-depth: 0
+
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
-          cache-dependency-glob: pyproject.toml
-      - name: run tests using uv and pytest
+          enable-cache: true
+          cache-dependency-glob: |
+            **/uv.lock
+            **/pyproject.toml
+
+      - name: Sync env (locked, --lock makes it fail immediatelly if uv.lock out of date with pyproject.toml)
+        run: uv sync --locked --extra dev --python ${{ matrix.python }}
+
+      - name: run tests using uv and pytest (locked, no sync)
         env:
           MPLBACKEND: agg
           PLATFORM: ${{ matrix.os }}
           DISPLAY: :42
-        run: uv run --extra dev --python ${{ matrix.python }} pytest --cov harpy --cov-branch --cov-report=xml src/harpy/_tests
+        run: uv run --no-sync pytest --cov harpy --cov-branch --cov-report=xml src/harpy/_tests
+
       - name: Upload coverage
         uses: codecov/codecov-action@v4
         with:

benchmarks/benchmarks.py

@@ -29,7 +29,7 @@ def setup_cache(self):
         """Set up cache."""
         tile_repeat = (1, 10, 10)  # will results in image of size tile_repeat * ( 22, 512, 512 )
 
-        sdata = hp.datasets.pixie_example(["fov0"])
+        sdata = hp.datasets.pixie_example()
 
         # create artificial large dataset
         arr = da.tile(sdata["raw_image_fov0"].data, tile_repeat)

docs/tutorials/general/FlowSOM_for_pixel_and_cell_clustering.ipynb

@@ -27,7 +27,7 @@ classifiers = [
     "Topic :: Software Development :: Testing"
 ]
 dependencies = [
-    "spatialdata>=0.2.6",
+    "spatialdata>=0.4.0",
     "xarray>=2024.10.0",
     "dask>=2024.4.1,<=2024.11.2",
     "distributed",
@@ -43,7 +43,6 @@ dependencies = [
     "magicgui",
     "pyrootutils",
     "universal_pathlib",
-    "datasets>=2.16.0",
     "crick",
     "lazy-loader>=0.4",
     # spatialdata_io depends on specific version of spatialdata: ImportError: cannot import name 'rasterize_bins_link_table_to_labels' from 'spatialdata' at readers/visium_hd.py", line 21
@@ -109,7 +108,6 @@ extra = [
 ]
 # dev dependencies
 test = [
-    "datasets",
     "opencv-python",
     "pytest",
     "pytest-cov",

experiments/example_pixel_cell_clustering_ark.ipynb

@@ -1,6 +1,14 @@
+import os
+
 import anndata as ad
 import pandas as pd
-from spatialdata import SpatialData
+import pooch
+from spatialdata import SpatialData, read_zarr
+
+from harpy.datasets import get_registry
+from harpy.utils.pylogger import get_pylogger
+
+log = get_pylogger(__name__)
 
 
 def _get_df(path):
@@ -29,8 +37,12 @@ def _get_adata(path, label_prefix="label_whole_"):
     return adata
 
 
-def pixie_example(fovs: list | None = None, with_pixel_output=True, with_cells_output=True) -> SpatialData:
+def pixie_example_datasets(fovs: list | None = None, with_pixel_output=True, with_cells_output=True) -> SpatialData:
     """Example pixie dataset."""
+    log.warning(
+        "pixie_example_datasets() is deprecated: it relies on Hugging Face 'datasets' loading scripts, "
+        "which were removed in datasets>=4.0.0. Use 'hp.datasets.pixie_example()' or pin 'datasets<4.0.0'."
+    )
     import glob
     import os
     from pathlib import Path
@@ -125,5 +137,15 @@ def pixie_example(fovs: list | None = None, with_pixel_output=True, with_cells_o
     return sdata
 
 
+def pixie_example() -> SpatialData:
+    """Example pixie dataset, loaded from s3 bucket"""
+    # Fetch and unzip the file
+    registry = get_registry()
+    unzip_path = registry.fetch("proteomics/pixie/sdata_pixie.zarr.zip", processor=pooch.Unzip())
+    sdata = read_zarr(os.path.commonpath(unzip_path))
+    sdata.path = None
+    return sdata
+
+
 if __name__ == "__main__":
-    pixie_example([0])
+    pixie_example()

pyproject.toml

@@ -51,6 +51,7 @@ def get_registry(path: str | Path | None = None) -> Pooch:
             "transcriptomics/xenium/Xenium_V1_humanLung_Cancer_FFPE/Xenium_V1_humanLung_Cancer_FFPE_he_image.ome.tif": "0348c3aeac4be3770fd3385a0cd99c3948b25b8d5e2d853add6c95e07a8baaf1",
             "transcriptomics/xenium/Xenium_V1_humanLung_Cancer_FFPE/Xenium_V1_humanLung_Cancer_FFPE_he_imagealignment.csv": "9b8e47bf81ca1aa447dc01a476ba6315cdf9ea66d7abb357d91e06ccf67735a7",
             "transcriptomics/xenium/Xenium_V1_humanLung_Cancer_FFPE/Xenium_V1_humanLung_Cancer_FFPE_outs.zip": "865c3805a959b51514555300df61328fc05055b939b9bece43522b8918314e1d",
+            "proteomics/pixie/sdata_pixie.zarr.zip": "cdcb17e4e18165d5cfca2c07a899a0e5b19efea26c25826370ef2bf77ea6ad13",
             "proteomics/mibi_tof/sdata_multi_channel.zarr.zip": "930fd2574666b90d5d6660ad8b52d47afffc9522704b9e6fef39d11c9cfff06e",
             "proteomics/macsima/sdata_multi_channel.zarr.zip": "26187fe62b75c3411f948cbcea52abd24b318992509e768c2ed2f55ddcb99f28",
             "proteomics/macsima/tonsil_all.zarr.zip": "f3444332c9decb318c6eff03e719ee35df6d7399da2a04a17dbed5833bfd9ed7",