Skip to content

Zarr3 #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 10 additions & 26 deletions funlib/persistence/arrays/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,6 @@

logger = logging.getLogger(__name__)


class ArrayNotFoundError(Exception):
"""Exception raised when an array is not found in the dataset."""

def __init__(self, message: str = "Array not found in the dataset"):
self.message = message
super().__init__(self.message)


def open_ds(
store,
mode: str = "r",
Expand Down Expand Up @@ -107,10 +98,7 @@ def open_ds(
else get_default_metadata_format()
)

try:
data = zarr.open(store, mode=mode, **kwargs)
except zarr.errors.PathNotFoundError:
raise ArrayNotFoundError(f"Nothing found at path {store}")
data = zarr.open(store, mode=mode, **kwargs)

metadata = metadata_format.parse(
data.shape,
Expand Down Expand Up @@ -239,7 +227,7 @@ def prepare_ds(

try:
existing_array = open_ds(store, mode="r", **kwargs)
except ArrayNotFoundError:
except FileNotFoundError:
existing_array = None

if existing_array is not None:
Expand Down Expand Up @@ -349,18 +337,14 @@ def prepare_ds(
)

# create the dataset
try:
ds = zarr.open_array(
store=store,
shape=shape,
chunks=chunk_shape,
dtype=dtype,
dimension_separator="/",
mode=mode,
**kwargs,
)
except zarr.errors.ArrayNotFoundError:
raise ArrayNotFoundError(f"Nothing found at path {store}")
ds = zarr.open_array(
store=store,
shape=shape,
chunks=chunk_shape,
dtype=dtype,
mode=mode,
**kwargs,
)

default_metadata_format = get_default_metadata_format()
our_metadata = {
Expand Down
2 changes: 1 addition & 1 deletion funlib/persistence/arrays/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ def recurse(
# base case
if len(keys) == 0:
# this key returns the data we want
if isinstance(data, (dict, zarr.attrs.Attributes)):
if isinstance(data, (dict, zarr.core.attributes.Attributes)):
return data.get(str(current_key), None)
elif isinstance(data, list):
assert isinstance(current_key, int), current_key
Expand Down
7 changes: 5 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ authors = [
]
dynamic = ['version']

requires-python = ">=3.10"
requires-python = ">=3.11"
classifiers = ["Programming Language :: Python :: 3"]
keywords = []

dependencies = [
"zarr>=2,<3",
"zarr>=3,<4",
# ImportError: cannot import name 'cbuffer_sizes' from 'numcodecs.blosc'
# We can pin zarr to >2.18.7 but then we have to drop python 3.10
# pin numcodecs to avoid breaking change
Expand Down Expand Up @@ -63,3 +63,6 @@ explicit_package_bases = true
[[tool.mypy.overrides]]
module = ["zarr.*", "iohub.*"]
ignore_missing_imports = true

[tool.uv.sources]
iohub = { git = "https://github.com/czbiohub-sf/iohub", rev = "zarr3-dev" }
6 changes: 2 additions & 4 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,17 @@ def can_connect_to_psql():
psql_param,
)
)
def provider_factory(request, tmpdir):
def provider_factory(request, tmp_path):
# provides a factory function to generate graph provider
# can provide either mongodb graph provider or file graph provider
# if file graph provider, will generate graph in a temporary directory
# to avoid artifacts

tmpdir = Path(tmpdir)

def sqlite_provider_factory(
mode, directed=None, total_roi=None, node_attrs=None, edge_attrs=None
):
return SQLiteGraphDataBase(
tmpdir / "test_sqlite_graph.db",
tmp_path / "test_sqlite_graph.db",
position_attribute="position",
mode=mode,
directed=directed,
Expand Down
18 changes: 9 additions & 9 deletions tests/test_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import pytest

from funlib.geometry import Coordinate, Roi
from funlib.persistence.arrays.datasets import ArrayNotFoundError, open_ds, prepare_ds
from funlib.persistence.arrays.datasets import open_ds, prepare_ds
from funlib.persistence.arrays.metadata import MetaDataFormat

stores = {
Expand All @@ -16,8 +16,8 @@


@pytest.mark.parametrize("store", stores.keys())
def test_metadata(tmpdir, store):
store = tmpdir / store
def test_metadata(tmp_path, store):
store = tmp_path / store

# test prepare_ds creates array if it does not exist and mode is write
array = prepare_ds(
Expand All @@ -34,10 +34,10 @@ def test_metadata(tmpdir, store):

@pytest.mark.parametrize("store", stores.keys())
@pytest.mark.parametrize("dtype", [np.float32, np.uint8, np.uint64])
def test_helpers(tmpdir, store, dtype):
def test_helpers(tmp_path, store, dtype):
shape = Coordinate(1, 1, 10, 20, 30)
chunk_shape = Coordinate(2, 3, 10, 10, 10)
store = tmpdir / store
store = tmp_path / store
metadata = MetaDataFormat().parse(
shape,
{
Expand All @@ -50,7 +50,7 @@ def test_helpers(tmpdir, store, dtype):
)

# test prepare_ds fails if array does not exist and mode is read
with pytest.raises(ArrayNotFoundError):
with pytest.raises(FileNotFoundError):
prepare_ds(
store,
shape,
Expand Down Expand Up @@ -218,9 +218,9 @@ def test_helpers(tmpdir, store, dtype):

@pytest.mark.parametrize("store", stores.keys())
@pytest.mark.parametrize("dtype", [np.float32, np.uint8, np.uint64])
def test_open_ds(tmpdir, store, dtype):
def test_open_ds(tmp_path, store, dtype):
shape = Coordinate(1, 1, 10, 20, 30)
store = tmpdir / store
store = tmp_path / store
metadata = MetaDataFormat().parse(
shape,
{
Expand All @@ -233,7 +233,7 @@ def test_open_ds(tmpdir, store, dtype):
)

# test open_ds fails if array does not exist and mode is read
with pytest.raises(ArrayNotFoundError):
with pytest.raises(FileNotFoundError):
open_ds(
store,
offset=metadata.offset,
Expand Down
6 changes: 3 additions & 3 deletions tests/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,15 @@ def test_empty_metadata():
assert metadata.types == ["space", "space", "space", "space", "space"]


def test_default_metadata_format(tmpdir):
def test_default_metadata_format(tmp_path):
set_default_metadata_format(metadata_formats["simple"])
metadata = metadata_formats["simple"].parse(
(10, 2, 100, 100, 100),
json.loads(open(fixtures_dir / metadata_jsons["simple"]).read()),
)

prepare_ds(
tmpdir / "test.zarr/test",
tmp_path / "test.zarr/test",
(10, 2, 100, 100, 100),
offset=metadata.offset,
voxel_size=metadata.voxel_size,
Expand All @@ -110,7 +110,7 @@ def test_default_metadata_format(tmpdir):
mode="w",
)

zarr_attrs = dict(**zarr.open(str(tmpdir / "test.zarr/test")).attrs)
zarr_attrs = dict(**zarr.open(tmp_path / "test.zarr/test").attrs)
assert zarr_attrs["offset"] == [100, 200, 400]
assert zarr_attrs["resolution"] == [1, 2, 3]
assert zarr_attrs["extras/axes"] == ["sample^", "channel^", "t", "y", "x"]
Expand Down
Loading