From be8149fb20a43fc739297adeca39da75c3882877 Mon Sep 17 00:00:00 2001 From: Andrew Davison Date: Sat, 23 May 2026 17:44:25 +0200 Subject: [PATCH 1/2] Harmonise Drive and Bucket client interfaces --- README.md | 63 +++++- ebrains_drive/__init__.py | 20 +- ebrains_drive/base.py | 71 +++++++ ebrains_drive/bucket.py | 61 +++++- ebrains_drive/buckets.py | 109 +++++++++++ ebrains_drive/client.py | 70 ++----- ebrains_drive/file.py | 33 ++++ ebrains_drive/files.py | 211 +++++++++++++++++++- ebrains_drive/repo.py | 19 +- ebrains_drive/repos.py | 20 ++ tests/test_harmonization.py | 375 ++++++++++++++++++++++++++++++++++++ 11 files changed, 980 insertions(+), 72 deletions(-) create mode 100644 ebrains_drive/base.py create mode 100644 tests/test_harmonization.py diff --git a/README.md b/README.md index 50baf079b4..087cd7267b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ Example usage (refer to docs for more): # 2.2 or via from ebrains_drive.client import DriveApiClient client = DriveApiClient(username="hbp_username", password="password") + # 2.3 or, to talk to the Bucket (Data Proxy) backend: + bucket_client = ebrains_drive.connect('hbp_username', 'password', target="bucket") # 3. Working with Collab drives (libraries / repos) @@ -76,11 +78,17 @@ Example Usage: # access existing bucket bucket = client.buckets.get_bucket("existing_collab_name") - # or create a new collab + bucket - bucket = client.create_new("new_collab_name") + # list buckets you have access to (optionally filter by name substring) + my_buckets = client.buckets.list_buckets() + matching = client.buckets.list_buckets(search="atlas") - # upload new file + # create a new collab + bucket (canonical entry point) + bucket = client.buckets.create_bucket("new_collab_name") + # client.create_new(...) still works but is deprecated. + + # upload new file (path or file-like object both accepted) bucket.upload("/home/jovyan/test.txt", "test/foobar.txt") + bucket.upload_local_file("/home/jovyan/test.txt", "test/foobar.txt") # Or upload from from in memory: from io import StringIO @@ -101,18 +109,61 @@ Example Usage: from time import sleep sleep(1) - # list the contents + # list the contents (flat) files = [f for f in bucket.ls(prefix="test")] + # or browse as a directory tree (uses the data-proxy `delimiter` parameter) + root = bucket.get_dir("/") + for entry in root.ls(): + # entry is either a DataproxyFile (object) or a BucketDir (subprefix) + print(entry) + + # navigate further + subdir = root.get_dir("test") + for entry in subdir.ls(): + print(entry) + # get the uploaded file file_handle = bucket.get_file("foobar.txt") file_content = file_handle.get_content() + # ...with a progress bar: + file_content = file_handle.get_content(progress=True) + + # native server-side copy (no client-side transfer) + file_handle.copy_to(dst_name="foobar-backup.txt") - # delete a bucket, and also delete the wiki associated with it) - client.delete_bucket("new_collab_name", delete_wiki=True) + # delete a bucket (and the wiki associated with it) + bucket.delete() # canonical instance method + client.buckets.delete_bucket("new_collab_name", delete_wiki=True) # manager-level + # client.delete_bucket(...) still works but is deprecated. ``` +### Harmonised cross-backend interface + +The Drive (Seafile) and Bucket (Data-Proxy) clients now share a common +interface so the same code can target either backend: + +| concept | Drive | Bucket | +| --- | --- | --- | +| factory | `connect(target="drive")` | `connect(target="bucket")` | +| manager | `client.repos` | `client.buckets` | +| list | `client.repos.list()` / `list_repos()` | `client.buckets.list()` / `list_buckets()` | +| get | `client.repos.get(id)` / `get_repo(id)` | `client.buckets.get(name)` / `get_bucket(name)` | +| create | `client.repos.create(name)` / `create_repo(name)` | `client.buckets.create(name)` / `create_bucket(name)` | +| delete by name | `client.repos.delete(id)` | `client.buckets.delete(name)` / `delete_bucket(name)` | +| container delete | `repo.delete()` | `bucket.delete()` | +| readonly check | `repo.is_readonly()` | `bucket.is_readonly()` | +| upload (root) | `repo.upload(filelike_or_path, name)` / `repo.upload_local_file(path)` | `bucket.upload(filelike_or_path, name)` / `bucket.upload_local_file(path)` | +| directory view | `repo.get_dir("/")` returning `SeafDir` | `bucket.get_dir("/")` returning `BucketDir` | +| download with progress bar | `file.get_content(progress=True)` | `file.get_content(progress=True)` | +| URL helper | `client.file.get_file_by_url(url)` | `client.file.get_file_by_url(url)` | + +Structural protocols are also published in `ebrains_drive.base` +(`StorageClient`, `ContainerManager`, `Container`, `StorageObject`) so +backend-agnostic code can be type-checked with `isinstance(...)` at +runtime. + Read access of public buckets can be done without supplying a token: ```python diff --git a/ebrains_drive/__init__.py b/ebrains_drive/__init__.py index ac73e6795c..fd0887ffad 100644 --- a/ebrains_drive/__init__.py +++ b/ebrains_drive/__init__.py @@ -10,6 +10,20 @@ from ebrains_drive.client import DriveApiClient, BucketApiClient -def connect(username=None, password=None, token=None, env=""): - client = DriveApiClient(username, password, token, env) - return client +def connect(username=None, password=None, token=None, env="", target="drive"): + """Return an authenticated EBRAINS client. + + :param target: ``"drive"`` (default) returns a + :class:`ebrains_drive.client.DriveApiClient`; ``"bucket"`` returns + a :class:`ebrains_drive.client.BucketApiClient`. The default + preserves backwards compatibility. + + For anonymous read access to public buckets, construct + :class:`BucketApiClient` directly rather than going through this + factory: ``connect()`` always authenticates. + """ + if target == "drive": + return DriveApiClient(username, password, token, env) + if target == "bucket": + return BucketApiClient(username, password, token, env) + raise ValueError(f"Unknown target: {target!r}. Expected 'drive' or 'bucket'.") diff --git a/ebrains_drive/base.py b/ebrains_drive/base.py new file mode 100644 index 0000000000..7e8d73842a --- /dev/null +++ b/ebrains_drive/base.py @@ -0,0 +1,71 @@ +"""Structural protocols for the harmonised Drive / Bucket interface. + +These :class:`typing.Protocol` definitions describe the surface area that +the EBRAINS Drive (Seafile) and EBRAINS Bucket (Data Proxy) clients have +in common. They are structural: concrete classes such as +:class:`ebrains_drive.repos.Repos`, :class:`ebrains_drive.buckets.Buckets`, +:class:`ebrains_drive.repo.Repo`, :class:`ebrains_drive.bucket.Bucket`, +:class:`ebrains_drive.files.SeafFile`, and +:class:`ebrains_drive.files.DataproxyFile` satisfy these protocols without +needing to inherit from them. + +Use the protocols as type hints when writing code that works against +either backend, e.g.:: + + def archive(obj: StorageObject) -> bytes: + return obj.get_content() +""" + +from typing import Any, Iterable, Protocol, Union, runtime_checkable + + +@runtime_checkable +class StorageClient(Protocol): + """The common surface of :class:`DriveApiClient` and :class:`BucketApiClient`.""" + + server: str + + def send_request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Any: ... + + +@runtime_checkable +class StorageObject(Protocol): + """A file-like object in either backend.""" + + name: str + + def get_content(self) -> bytes: ... + + def get_download_link(self) -> str: ... + + def delete(self) -> Any: ... + + +@runtime_checkable +class Container(Protocol): + """A repo (Drive) or bucket (Bucket).""" + + name: str + + def get_file(self, path: str) -> StorageObject: ... + + def upload(self, filelike_or_path: Any, filename: str) -> Any: ... + + def upload_local_file(self, filepath: str, name: Union[str, None] = None, overwrite: bool = False) -> Any: ... + + def delete(self) -> Any: ... + + def is_readonly(self) -> bool: ... + + +@runtime_checkable +class ContainerManager(Protocol): + """The collection of containers exposed by a client (``client.repos`` / ``client.buckets``).""" + + def get(self, name: str) -> Container: ... + + def list(self) -> Iterable[Container]: ... + + def create(self, name: str, **kwargs: Any) -> Container: ... + + def delete(self, name: str) -> Any: ... diff --git a/ebrains_drive/bucket.py b/ebrains_drive/bucket.py index 4e3800f792..deb221fa18 100644 --- a/ebrains_drive/bucket.py +++ b/ebrains_drive/bucket.py @@ -1,7 +1,8 @@ +import os from typing import Iterable import requests from ebrains_drive.exceptions import DoesNotExist, InvalidParameter, UpstreamAPIException -from ebrains_drive.files import DataproxyFile +from ebrains_drive.files import BucketDir, DataproxyFile from ebrains_drive.utils import on_401_raise_unauthorized from io import IOBase from typing import Union @@ -20,8 +21,8 @@ def __init__( self, client, name: str, - objects_count: int, - bytes: int, + objects_count: int = None, + bytes: int = None, last_modified: str = None, is_public: bool = None, is_initialized: bool = None, @@ -106,3 +107,57 @@ def upload(self, filelike: Union[str, IOBase], filename: str, **kwargs): filehandle = filelike if isinstance(filelike, IOBase) else open(filelike, "rb") resp = requests.request("PUT", upload_url, data=filehandle, **kwargs) resp.raise_for_status() + + def upload_local_file(self, filepath: str, name: str = None, overwrite: bool = False, **kwargs): + """Upload a local file to this bucket. + + Mirrors :meth:`ebrains_drive.files.SeafDir.upload_local_file`. + + :param filepath: path to the local file on disk. + :param name: name to store the file under in the bucket; defaults + to the basename of ``filepath``. + :param overwrite: when ``False`` (default), raise + :class:`FileExistsError` if an object with that name already + exists in the bucket. When ``True``, the existing object will + be overwritten on upload. + """ + name = name or os.path.basename(filepath) + if not overwrite: + try: + self.get_file(name) + except DoesNotExist: + pass + else: + raise FileExistsError(f"Object with name = `{name}` already exists in bucket `{self.name}`.") + with open(filepath, "rb") as fp: + self.upload(fp, name, **kwargs) + + def delete(self): + """Delete this bucket. + + Mirrors :meth:`ebrains_drive.repo.Repo.delete`. Delegates to + :meth:`ebrains_drive.buckets.Buckets.delete_bucket` on the owning + client so the call site is uniform across backends. + """ + return self.client.buckets.delete_bucket(self.name) + + def is_readonly(self) -> bool: + """Return ``True`` when the current credentials cannot write to this bucket. + + Conservative: dataset buckets and anonymously-accessed public + buckets report read-only because their :attr:`role` is ``None``. + Returns ``True`` when :attr:`role` is ``None`` or ``"viewer"``; + otherwise ``False``. + """ + return self.role in (None, "viewer") + + def get_dir(self, path: str) -> BucketDir: + """Return a virtual directory view rooted at ``path``. + + The data-proxy backend is a flat object store; this method exposes + the same prefix-based tree view the DataProxy GUI uses. Mirrors + :meth:`ebrains_drive.repo.Repo.get_dir`. No HTTP round-trip on + construction — listing happens lazily. + """ + prefix = path.strip("/") + return BucketDir(self.client, self, prefix) diff --git a/ebrains_drive/buckets.py b/ebrains_drive/buckets.py index f4b3acdb4f..894ebed31d 100644 --- a/ebrains_drive/buckets.py +++ b/ebrains_drive/buckets.py @@ -1,3 +1,4 @@ +import time from ebrains_drive.exceptions import ClientHttpError, Unauthorized from ebrains_drive.utils import on_401_raise_unauthorized from ebrains_drive.bucket import Bucket @@ -41,3 +42,111 @@ def get_dataset(self, dataset_id: str, *, public: bool = False, request_access: sleep(5) attempt_no = attempt_no + 1 print(f"Checking permission, attempt {attempt_no}") + + # ----- Harmonised aliases (mirror Repos.get/list/create/delete) ----- + + def get(self, name: str, *, public: bool = False) -> Bucket: + """Alias for :meth:`get_bucket`. Part of the harmonised + :class:`ebrains_drive.base.ContainerManager` surface.""" + return self.get_bucket(name, public=public) + + @on_401_raise_unauthorized( + "Failed. Note: Buckets.create_bucket needs to have clb.drive:write as a part of scope." + ) + def create_bucket(self, bucket_name: str, title: str = None, description: str = "Created by ebrains_drive"): + """Create a new bucket. + + Canonical entry point — :meth:`ebrains_drive.client.BucketApiClient.create_new` + is a deprecated wrapper around this method. + + Creates a new wiki/collab first (or accepts an existing one), + then initialises the bucket. Bucket initialisation is retried up + to five times because the wiki backend usually needs a few + seconds before it will accept bucket creation. + + :param bucket_name: name of the bucket (and wiki, if it needs creating). + :param title: title of the wiki to create; defaults to ``bucket_name``. + :param description: description for the wiki to create. + """ + self.client.send_request( + "POST", + f"https://wiki{self.client.suffix}.ebrains.eu/rest/v1/collabs", + json={ + "name": bucket_name, + "title": title or bucket_name, + "description": description, + "drive": True, + "chat": True, + "public": False, + }, + expected=(201, 409), + ) + + fuse = 5 + while True: + try: + self.client.send_request("POST", "/v1/buckets", json={"bucket_name": bucket_name}, expected=201) + break + except Exception as e: + if fuse < 0: + raise e from e + fuse -= 1 + time.sleep(1) + return self.get_bucket(bucket_name) + + def create(self, name: str, **kwargs): + """Alias for :meth:`create_bucket`.""" + return self.create_bucket(name, **kwargs) + + @on_401_raise_unauthorized( + "Failed. Note: Buckets.delete_bucket needs to have clb.drive:write as a part of scope." + ) + def delete_bucket(self, bucket_name: str, *, delete_wiki: bool = False): + """Delete an existing bucket. + + Canonical entry point — :meth:`ebrains_drive.client.BucketApiClient.delete_bucket` + is a deprecated wrapper around this method, and + :meth:`ebrains_drive.bucket.Bucket.delete` delegates here. + + :param bucket_name: name of the bucket to delete. + :param delete_wiki: when ``True``, also delete the associated wiki/collab. + """ + self.client.send_request("DELETE", f"/v1/buckets/{bucket_name}", expected=(200,)) + if delete_wiki: + self.client.send_request( + "DELETE", f"https://wiki.ebrains.eu/rest/v1/collabs/{bucket_name}", expected=(200,) + ) + + def delete(self, name: str, **kwargs): + """Alias for :meth:`delete_bucket`.""" + return self.delete_bucket(name, **kwargs) + + @on_401_raise_unauthorized("Unauthorized. List endpoint requires authentication for non-public buckets.") + def list_buckets(self, search: str = None): + """List buckets accessible to the current credentials. + + Calls ``GET /v1/buckets`` which returns a lightweight entry per + bucket — ``name``, ``role``, ``is_public``. The returned + :class:`Bucket` instances have ``objects_count`` and ``bytes`` + set to ``None`` (call :meth:`get_bucket` to fetch full stats). + + :param search: optional substring filter on bucket name. + """ + params = {"search": search} if search else None + resp = self.client.get("/v1/buckets", params=params) + return [ + Bucket( + self.client, + name=entry["name"], + objects_count=None, + bytes=None, + is_public=entry.get("is_public"), + role=entry.get("role"), + target="buckets", + ) + for entry in resp.json() + ] + + def list(self, search: str = None): + """Alias for :meth:`list_buckets`.""" + return self.list_buckets(search=search) diff --git a/ebrains_drive/client.py b/ebrains_drive/client.py index 94739afdb7..65cb7daeef 100644 --- a/ebrains_drive/client.py +++ b/ebrains_drive/client.py @@ -3,17 +3,17 @@ import base64 import json import time +import warnings from copy import copy from typing import Callable from functools import wraps import requests -from ebrains_drive.utils import on_401_raise_unauthorized from ebrains_drive.exceptions import ClientHttpError, TokenExpired, Unauthorized from ebrains_drive.repos import Repos from ebrains_drive.buckets import Buckets -from ebrains_drive.file import File +from ebrains_drive.file import BucketFile, File class ClientBase(ABC): @@ -199,63 +199,25 @@ def __init__(self, username=None, password=None, token=_I_AM_A_PUBLIC_BUCKET, en self.server = f"https://data-proxy{self.suffix}.ebrains.eu/api" self.buckets = Buckets(self) + self.file = BucketFile(self) - @on_401_raise_unauthorized( - "Failed. Note: BucketApiClient.create_new needs to have clb.drive:write as a part of scope." - ) def create_new(self, bucket_name: str, title=None, description="Created by ebrains_drive"): - """ - Create a new bucket by first attempting to create a new wiki/collab. On 201 (created) - or 409 (conflict) initialize the bucket of the said wiki. The request to initialize the bucket - will be retried up to 5 times, as it usually takes a few minutes for the newly initialized wiki - to allow buckets to be created. - - :param:`bucket_name` the name of the to-be-created bucket (and wiki if needed) - - :param:`title` the title of the to-be-created wiki (if unset, defaults to `bucket_name` param) - - :param:`description` description of the to-be-created wiki. - """ - - self.send_request( - "POST", - f"https://wiki{self.suffix}.ebrains.eu/rest/v1/collabs", - json={ - "name": bucket_name, - "title": title or bucket_name, - "description": description, - "drive": True, - "chat": True, - "public": False, - }, - expected=(201, 409), + """Deprecated. Use :meth:`ebrains_drive.buckets.Buckets.create_bucket` instead.""" + warnings.warn( + "BucketApiClient.create_new is deprecated; use client.buckets.create_bucket(...) instead.", + DeprecationWarning, + stacklevel=2, ) + return self.buckets.create_bucket(bucket_name, title=title, description=description) - fuse = 5 - while True: - try: - self.send_request("POST", "/v1/buckets", json={"bucket_name": bucket_name}, expected=201) - break - except Exception as e: - if fuse < 0: - raise e from e - fuse -= 1 - time.sleep(1) - - @on_401_raise_unauthorized( - "Failed. Note: BucketApiClient.delete_bucket needs to have clb.drive:write as a part of scope." - ) def delete_bucket(self, bucket_name: str, *, delete_wiki=False): - """ - Deletes an existing bucket. - - :param:`bucket_name` name of the bucket (and - if delete_wiki is set - of the wiki) to be deleted - - :param:`delete_wiki` if the wiki should also be deleted. - """ - self.send_request("DELETE", f"/v1/buckets/{bucket_name}", expected=(200,)) - if delete_wiki: - self.send_request("DELETE", f"https://wiki.ebrains.eu/rest/v1/collabs/{bucket_name}", expected=(200,)) + """Deprecated. Use :meth:`ebrains_drive.buckets.Buckets.delete_bucket` or :meth:`Bucket.delete` instead.""" + warnings.warn( + "BucketApiClient.delete_bucket is deprecated; use client.buckets.delete_bucket(...) or bucket.delete() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.buckets.delete_bucket(bucket_name, delete_wiki=delete_wiki) def send_request(self, method: str, url: str, *args, **kwargs): diff --git a/ebrains_drive/file.py b/ebrains_drive/file.py index 6ddb3d1bf3..e7021a3a38 100644 --- a/ebrains_drive/file.py +++ b/ebrains_drive/file.py @@ -5,6 +5,39 @@ from ebrains_drive.files import SeafFile +class BucketFile(object): + """URL/path resolution helper for the EBRAINS Bucket (Data Proxy) backend. + + Mirrors :class:`File` (the Drive equivalent) so the same attribute + name — ``client.file`` — works on both + :class:`ebrains_drive.client.DriveApiClient` and + :class:`ebrains_drive.client.BucketApiClient`. + """ + + def __init__(self, client): + self.client = client + + def get_file_by_url(self, file_url): + """Resolve a data-proxy URL to a :class:`DataproxyFile`. + + Recognises both bucket and dataset URLs, e.g.:: + + https://data-proxy.ebrains.eu/api/v1/buckets// + https://data-proxy.ebrains.eu/api/v1/datasets// + """ + regex = r".*/v1/(buckets|datasets)/([^/]+)/(.+)$" + matches = re.search(regex, file_url) + if matches is None: + raise ValueError("Parameter `file_url` does not have expected data-proxy format!") + + target, container_name, file_path = matches.group(1), matches.group(2), matches.group(3) + if target == "datasets": + container = self.client.buckets.get_dataset(container_name) + else: + container = self.client.buckets.get_bucket(container_name) + return container.get_file(file_path) + + class File(object): def __init__(self, client): self.client = client diff --git a/ebrains_drive/files.py b/ebrains_drive/files.py index bb18e304a9..ca1fb0f99c 100644 --- a/ebrains_drive/files.py +++ b/ebrains_drive/files.py @@ -92,6 +92,9 @@ def copyTo(self, dst_dir, dst_repo_id=None): resp = self._copy_move_task("copy", dirent_type, dst_dir, dst_repo_id) return resp.status_code == 200 + # snake_case alias for symmetry with the rest of the API + copy_to = copyTo + def moveTo(self, dst_dir, dst_repo_id=None): """Move file/folder to other directory (also to a different repo)""" if dst_repo_id is None: @@ -111,6 +114,8 @@ def moveTo(self, dst_dir, dst_repo_id=None): self.__dict__[key] = new_dirent.__dict__[key] return succeeded + move_to = moveTo + def get_share_link(self): dirent_type = "dir" if self.isdir else "file" url = f"/api2/repos/{self.repo.id}/{dirent_type}/shared-link/" @@ -242,13 +247,18 @@ def _get_download_token(self): def upload(self, fileobj, filename): """Upload a file to this folder. - :param:fileobj :class:`File` like object - :param:filename The name of the file + :param fileobj: file-like object, ``bytes``, or path (``str`` / ``PathLike``) + to a local file. The path overload mirrors + :meth:`ebrains_drive.bucket.Bucket.upload`. + :param filename: The name of the file as stored in the folder. Return a :class:`SeafFile` object of the newly uploaded file. """ if isinstance(fileobj, bytes): fileobj = io.BytesIO(fileobj) + elif isinstance(fileobj, (str, os.PathLike)): + with open(fileobj, "rb") as fp: + return self.upload(fp, filename) upload_url = self._get_upload_link() files = { "file": (filename, fileobj), @@ -338,10 +348,25 @@ def _get_download_link(self): resp = self.client.get(url) return re.match(r'"(.*)"', resp.text).group(1) - def get_content(self): - """Get the content of the file""" + def get_content(self, *, progress=False): + """Get the content of the file. + + :param progress: when ``True``, stream the response and display a + ``tqdm`` progress bar. Mirrors + :meth:`ebrains_drive.files.DataproxyFile.get_content`. + """ url = self._get_download_link() - return self.client.get(url).content + if not progress: + return self.client.get(url).content + + resp = self.client.get(url, stream=True) + total = resp.headers.get("content-length") + content = bytearray() + with tqdm(total=int(total) if total else None, leave=True) as bar: + for chunk in resp.iter_content(4096): + content.extend(chunk) + bar.update(len(chunk)) + return bytes(content) class DataproxyFile: @@ -400,3 +425,179 @@ def delete(self): assert len(json_resp.get("failures")) == 0 else: assert "has been removed" in json_resp["detail"] + + @on_401_raise_unauthorized("Unauthorized") + def copy_to(self, dst_name: str = None, *, dst_bucket: str = None): + """Copy this object to another location. + + Uses the data-proxy's native copy endpoint (``PUT + /v1/buckets/{name}/{object}/copy``). Roughly analogous to + :meth:`ebrains_drive.files._SeafDirentBase.copy_to`. + + :param dst_name: destination object name; defaults to the same name + (only meaningful when ``dst_bucket`` is set). + :param dst_bucket: destination bucket name; defaults to the same + bucket (only meaningful when ``dst_name`` is set). + """ + if dst_name is None and dst_bucket is None: + raise ValueError("copy_to requires at least one of dst_name or dst_bucket") + params = {} + if dst_bucket is not None: + params["to"] = dst_bucket + if dst_name is not None: + params["name"] = dst_name + resp = self.client.put( + f"/v1/{self.bucket.target}/{self.bucket.dataproxy_entity_name}/{self.name}/copy", + params=params, + ) + return resp.status_code == 200 + + +class BucketDir: + """A virtual directory view over a flat data-proxy bucket. + + The data-proxy backend stores objects without any directory concept, + but the DataProxy GUI presents objects with ``/`` in their names as + a tree. ``BucketDir`` mirrors that convention so cross-backend code + can traverse directories the same way for both + :class:`ebrains_drive.repo.Repo` (via :class:`SeafDir`) and + :class:`ebrains_drive.bucket.Bucket`. + + Instances are constructed via :meth:`ebrains_drive.bucket.Bucket.get_dir` + or :meth:`get_dir` on another :class:`BucketDir`. No HTTP round-trip + happens at construction time; child listings are fetched lazily. + + ``prefix`` is always stored without a leading slash. The root of the + bucket is ``prefix=""``. A non-empty prefix never has a trailing + slash internally — it is appended only when constructing API queries. + """ + + isdir = True + + def __init__(self, client, bucket, prefix: str = ""): + self.client = client + self.bucket = bucket + self.prefix = prefix.strip("/") + + @property + def name(self) -> str: + """The last path segment of this directory, or ``""`` for the bucket root.""" + if not self.prefix: + return "" + return self.prefix.rsplit("/", 1)[-1] + + @property + def path(self) -> str: + """Absolute path of this directory within the bucket (always starts with ``/``).""" + return "/" + self.prefix if self.prefix else "/" + + def _qualify(self, name: str) -> str: + name = name.lstrip("/") + if not self.prefix: + return name + return f"{self.prefix}/{name}" + + def _api_prefix(self) -> str: + return f"{self.prefix}/" if self.prefix else "" + + def __str__(self): + return "BucketDir[bucket=%s, path=%s]" % (self.bucket.name, self.path) + + __repr__ = __str__ + + def ls(self, *, recursive: bool = False): + """List the entries in this directory. + + :param recursive: when ``False`` (default), yield only the + immediate children of this prefix — each is either a + :class:`BucketDir` (a sub-prefix) or a :class:`DataproxyFile` + (an object directly under this prefix). Uses the data-proxy + ``delimiter`` parameter so the server returns grouped + prefixes directly. When ``True``, yield every + :class:`DataproxyFile` under this prefix at any depth. + """ + if recursive: + yield from self.bucket.ls(prefix=self._api_prefix() or None) + return + + marker = None + depth = self._api_prefix() + LIMIT = 100 + while True: + resp = self.client.get( + f"/v1/{self.bucket.target}/{self.bucket.dataproxy_entity_name}", + params={"limit": LIMIT, "marker": marker, "prefix": depth or None, "delimiter": "/"}, + ) + objects = resp.json().get("objects", []) + if not objects: + return + for obj in objects: + if "subdir" in obj: + child_prefix = obj["subdir"].rstrip("/") + marker = obj["subdir"] + yield BucketDir(self.client, self.bucket, child_prefix) + else: + marker = obj.get("name") + yield DataproxyFile.from_json(self.client, self.bucket, obj) + + def get_file(self, name: str) -> "DataproxyFile": + """Return the :class:`DataproxyFile` at ``/``.""" + return self.bucket.get_file(self._qualify(name)) + + def get_dir(self, name: str) -> "BucketDir": + """Return a child :class:`BucketDir`. No HTTP request is made.""" + return BucketDir(self.client, self.bucket, self._qualify(name)) + + def mkdir(self, name: str) -> "BucketDir": + """Return a new :class:`BucketDir` for a child prefix. + + Object-storage directories exist purely by convention — they + spring into existence when an object is uploaded under that + prefix. This method performs no HTTP request and changes no + server-side state. + """ + return self.get_dir(name) + + def upload(self, filelike_or_path, filename: str, **kwargs): + """Upload a file under this directory. + + Joins this directory's prefix with ``filename`` and delegates to + :meth:`ebrains_drive.bucket.Bucket.upload`. + """ + return self.bucket.upload(filelike_or_path, self._qualify(filename), **kwargs) + + def upload_local_file(self, filepath: str, name: str = None, overwrite: bool = False, **kwargs): + """Upload a local file under this directory. + + Mirrors :meth:`SeafDir.upload_local_file`. + """ + name = name or os.path.basename(filepath) + return self.bucket.upload_local_file(filepath, self._qualify(name), overwrite=overwrite, **kwargs) + + def check_exists(self, name: str): + """Return the matching :class:`DataproxyFile` / :class:`BucketDir`, or ``False``. + + Mirrors :meth:`SeafDir.check_exists`. + """ + target = self._qualify(name).rstrip("/") + for entry in self.ls(): + entry_name = entry.path.lstrip("/") if isinstance(entry, BucketDir) else entry.name + if entry_name == target: + return entry + return False + + def delete(self, *, recursive: bool = False): + """Delete every object under this prefix. + + Destructive: requires explicit ``recursive=True``. Raises + :class:`ebrains_drive.exceptions.OperationError` otherwise. + """ + if not recursive: + from ebrains_drive.exceptions import OperationError + + raise OperationError( + "BucketDir.delete() refuses to delete a directory implicitly. " + "Pass recursive=True to delete every object under this prefix." + ) + for obj in self.bucket.ls(prefix=self._api_prefix() or None): + obj.delete() diff --git a/ebrains_drive/repo.py b/ebrains_drive/repo.py index 5fd42421db..476517bf0b 100644 --- a/ebrains_drive/repo.py +++ b/ebrains_drive/repo.py @@ -57,7 +57,7 @@ def from_json(cls, client, repo_json): return cls(client, **repo_json) def is_readonly(self): - return "w" not in self.perm + return "w" not in self.permission @raise_does_not_exist("The requested file does not exist") def get_file(self, path): @@ -92,6 +92,23 @@ def delete(self): """Remove this repo. Only the repo owner can do this""" self.client.delete("/api2/repos/" + self.id) + def upload(self, filelike_or_path, filename): + """Upload a file to the root of this repo. + + Convenience shortcut for ``self.get_dir("/").upload(...)`` that + mirrors :meth:`ebrains_drive.bucket.Bucket.upload`. Accepts a + file-like object, a ``bytes`` value, or a path to a local file. + """ + return self.get_dir("/").upload(filelike_or_path, filename) + + def upload_local_file(self, filepath, name=None, overwrite=False): + """Upload a local file to the root of this repo. + + Convenience shortcut for ``self.get_dir("/").upload_local_file(...)`` + that mirrors :meth:`ebrains_drive.bucket.Bucket.upload_local_file`. + """ + return self.get_dir("/").upload_local_file(filepath, name=name, overwrite=overwrite) + def list_history(self): """List the history of this repo diff --git a/ebrains_drive/repos.py b/ebrains_drive/repos.py index b0b3ee6c51..b6518d8958 100644 --- a/ebrains_drive/repos.py +++ b/ebrains_drive/repos.py @@ -94,6 +94,26 @@ def get_default_repo(self): assert repo_id, f"Expected repo_id to be populated, but wasn't" return self.get_repo(repo_id) + # ----- Harmonised aliases (mirror Buckets.get/list/create/delete) ----- + + def get(self, repo_id): + """Alias for :meth:`get_repo`. Part of the harmonised + :class:`ebrains_drive.base.ContainerManager` surface.""" + return self.get_repo(repo_id) + + def list(self): + """Alias for :meth:`list_repos`.""" + return self.list_repos() + + def create(self, name, password=None): + """Alias for :meth:`create_repo`.""" + return self.create_repo(name, password=password) + + def delete(self, repo_id): + """Delete the repo identified by ``repo_id``. Mirrors + :meth:`ebrains_drive.buckets.Buckets.delete_bucket`.""" + return self.get_repo(repo_id).delete() + def get_repo_by_local_path(self, local_path): """ Get the repo that contains `local_path` when the Drive diff --git a/tests/test_harmonization.py b/tests/test_harmonization.py new file mode 100644 index 0000000000..1465bcd550 --- /dev/null +++ b/tests/test_harmonization.py @@ -0,0 +1,375 @@ +"""Tests for the harmonised Drive / Bucket surface. + +These tests exercise the additions documented in +``docs`` and the README: + + * shared :mod:`ebrains_drive.base` Protocols + * ``ebrains_drive.connect(target=...)`` entry point + * :class:`Bucket.is_readonly` and the bug-fixed :class:`Repo.is_readonly` + * :class:`BucketDir` virtual-directory view + * Manager-level aliases (``get`` / ``list`` / ``create`` / ``delete``) + * Bucket lifecycle aliases (``Buckets.create_bucket`` / + ``Buckets.delete_bucket`` / ``Bucket.delete``) and the + ``DeprecationWarning`` emitted by the legacy client methods. +""" + +import pytest +import warnings +from io import BytesIO +from unittest.mock import MagicMock, patch + +import ebrains_drive +from ebrains_drive.base import Container, ContainerManager, StorageObject +from ebrains_drive.bucket import Bucket +from ebrains_drive.buckets import Buckets +from ebrains_drive.client import BucketApiClient, DriveApiClient, _I_AM_A_PUBLIC_BUCKET +from ebrains_drive.exceptions import DoesNotExist, OperationError +from ebrains_drive.files import BucketDir, DataproxyFile +from ebrains_drive.repo import Repo + + +class MockResp: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + self.headers = {} + + def json(self): + return self._payload + + def raise_for_status(self): + pass + + +class MockClient: + suffix = "" + + def __init__(self): + self.get = MagicMock() + self.post = MagicMock() + self.put = MagicMock() + self.delete = MagicMock() + self.send_request = MagicMock() + + +@pytest.fixture +def mock_client(): + return MockClient() + + +bucket_json = { + "name": "foo", + "objects_count": 12, + "bytes": 112233, + "is_public": False, + "role": "editor", +} + + +# --------------------------- connect() ------------------------------ # + + +def test_connect_target_drive_default(): + with patch("ebrains_drive.DriveApiClient") as Drive, patch("ebrains_drive.BucketApiClient") as Bucket_: + ebrains_drive.connect(token="dummy") + Drive.assert_called_once_with(None, None, "dummy", "") + Bucket_.assert_not_called() + + +def test_connect_target_bucket(): + with patch("ebrains_drive.DriveApiClient") as Drive, patch("ebrains_drive.BucketApiClient") as Bucket_: + ebrains_drive.connect(token="dummy", target="bucket") + Bucket_.assert_called_once_with(None, None, "dummy", "") + Drive.assert_not_called() + + +def test_connect_target_unknown_raises(): + with pytest.raises(ValueError, match="Unknown target"): + ebrains_drive.connect(token="dummy", target="banana") + + +# --------------------------- is_readonly() -------------------------- # + + +@pytest.mark.parametrize( + "permission,expected", + [("r", True), ("rw", False), ("rw", False)], +) +def test_repo_is_readonly(permission, expected): + """Bug fix: previously raised AttributeError because of ``self.perm``.""" + repo = Repo(client=MockClient(), permission=permission) + assert repo.is_readonly() is expected + + +@pytest.mark.parametrize( + "role,expected", + [(None, True), ("viewer", True), ("editor", False), ("administrator", False)], +) +def test_bucket_is_readonly(role, expected): + bucket = Bucket(MockClient(), name="foo", objects_count=0, bytes=0, role=role) + assert bucket.is_readonly() is expected + + +# --------------------------- Protocols ------------------------------ # + + +def test_repo_satisfies_container_protocol(): + repo = Repo(client=MockClient(), id="x", name="n", permission="rw") + assert isinstance(repo, Container) + + +def test_bucket_satisfies_container_protocol(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + assert isinstance(bucket, Container) + + +def test_dataproxy_file_satisfies_storage_object_protocol(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + f = DataproxyFile( + mock_client, bucket, hash="h", last_modified="now", bytes=1, name="x", content_type="text/plain" + ) + assert isinstance(f, StorageObject) + + +def test_buckets_satisfies_container_manager_protocol(mock_client): + assert isinstance(Buckets(mock_client), ContainerManager) + + +# --------------------------- Manager aliases ------------------------ # + + +def test_buckets_get_alias_delegates(mock_client): + mock_client.get.return_value = MockResp(bucket_json) + bkts = Buckets(mock_client) + result = bkts.get("foo") + assert isinstance(result, Bucket) + mock_client.get.assert_called_with("/v1/buckets/foo/stat") + + +def test_buckets_list_calls_endpoint(mock_client): + mock_client.get.return_value = MockResp( + [ + {"name": "a", "role": "editor", "is_public": False}, + {"name": "b", "role": None, "is_public": True}, + ] + ) + bkts = Buckets(mock_client) + out = bkts.list_buckets() + assert [b.name for b in out] == ["a", "b"] + assert out[0].objects_count is None + mock_client.get.assert_called_with("/v1/buckets", params=None) + + +def test_buckets_list_with_search(mock_client): + mock_client.get.return_value = MockResp([]) + Buckets(mock_client).list_buckets(search="atlas") + mock_client.get.assert_called_with("/v1/buckets", params={"search": "atlas"}) + + +# --------------------------- Bucket.delete -------------------------- # + + +def test_bucket_delete_delegates_to_manager(mock_client): + bkts = Buckets(mock_client) + mock_client.buckets = bkts + mock_client.send_request.return_value = MockResp({}) + bucket = Bucket.from_json(mock_client, bucket_json) + bucket.delete() + mock_client.send_request.assert_called_with("DELETE", "/v1/buckets/foo", expected=(200,)) + + +# --------------------------- Deprecation ---------------------------- # + + +def test_create_new_emits_deprecation_warning(): + client = BucketApiClient.__new__(BucketApiClient) + client._token = _I_AM_A_PUBLIC_BUCKET + client.suffix = "" + client.buckets = MagicMock() + client.buckets.create_bucket.return_value = "ok" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + client.create_new("name") + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + client.buckets.create_bucket.assert_called_with("name", title=None, description="Created by ebrains_drive") + + +def test_delete_bucket_client_emits_deprecation_warning(): + client = BucketApiClient.__new__(BucketApiClient) + client._token = _I_AM_A_PUBLIC_BUCKET + client.suffix = "" + client.buckets = MagicMock() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + client.delete_bucket("name", delete_wiki=True) + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + client.buckets.delete_bucket.assert_called_with("name", delete_wiki=True) + + +# --------------------------- BucketDir ------------------------------ # + + +def test_bucket_get_dir_returns_root(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + root = bucket.get_dir("/") + assert isinstance(root, BucketDir) + assert root.prefix == "" + assert root.path == "/" + + +def test_bucket_get_dir_nested(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + d = bucket.get_dir("/sub/folder/") + assert d.prefix == "sub/folder" + assert d.name == "folder" + assert d.path == "/sub/folder" + + +def test_bucket_dir_ls_non_recursive_groups_subdirs(mock_client): + """Non-recursive ``ls`` issues a request with delimiter="/" and dispatches + mixed StorageObject / StorageDirWithObjectsResponse entries.""" + mock_client.get.side_effect = [ + MockResp( + { + "objects": [ + { + "name": "a.txt", + "hash": "h", + "last_modified": "n", + "bytes": 1, + "content_type": "text/plain", + }, + {"subdir": "dir1/", "bytes": None, "last_modified": None, "objects_count": None}, + {"subdir": "dir2/", "bytes": None, "last_modified": None, "objects_count": None}, + ] + } + ), + MockResp({"objects": []}), + ] + bucket = Bucket.from_json(mock_client, bucket_json) + entries = list(bucket.get_dir("/").ls()) + types = [type(e).__name__ for e in entries] + assert types == ["DataproxyFile", "BucketDir", "BucketDir"] + assert entries[0].name == "a.txt" + assert entries[1].prefix == "dir1" + assert entries[2].prefix == "dir2" + # Verify the delimiter param was actually sent + call_params = mock_client.get.call_args_list[0].kwargs["params"] + assert call_params["delimiter"] == "/" + + +def test_bucket_dir_ls_recursive_delegates_to_flat(mock_client): + """Recursive ``ls`` delegates to ``Bucket.ls(prefix=...)`` (no delimiter).""" + mock_client.get.return_value = MockResp({"objects": []}) + bucket = Bucket.from_json(mock_client, bucket_json) + list(bucket.get_dir("/sub").ls(recursive=True)) + call_params = mock_client.get.call_args_list[0].kwargs["params"] + assert "delimiter" not in call_params or call_params.get("delimiter") is None + assert call_params["prefix"] == "sub/" + + +def test_bucket_dir_mkdir_is_local_only(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + new = bucket.get_dir("/").mkdir("future") + assert isinstance(new, BucketDir) + assert new.prefix == "future" + mock_client.get.assert_not_called() + mock_client.post.assert_not_called() + mock_client.put.assert_not_called() + + +def test_bucket_dir_delete_requires_recursive(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + with pytest.raises(OperationError, match="recursive=True"): + bucket.get_dir("/sub").delete() + + +def test_bucket_dir_delete_recursive_deletes_all(mock_client): + f1 = MagicMock(spec=DataproxyFile) + f2 = MagicMock(spec=DataproxyFile) + bucket = Bucket.from_json(mock_client, bucket_json) + with patch.object(bucket, "ls", return_value=iter([f1, f2])) as mock_ls: + bucket.get_dir("/sub").delete(recursive=True) + mock_ls.assert_called_with(prefix="sub/") + f1.delete.assert_called_once() + f2.delete.assert_called_once() + + +def test_bucket_dir_upload_qualifies_filename(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + with patch.object(bucket, "upload") as mock_upload: + bucket.get_dir("/sub").upload(BytesIO(b"x"), "leaf.txt") + args, _ = mock_upload.call_args + assert args[1] == "sub/leaf.txt" + + +# --------------------------- copy_to -------------------------------- # + + +def test_dataproxy_file_copy_to_calls_native_endpoint(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + f = DataproxyFile(mock_client, bucket, hash="h", last_modified="n", bytes=1, name="src", content_type="t") + mock_client.put.return_value = MockResp({}, status_code=200) + ok = f.copy_to(dst_name="dst") + assert ok is True + mock_client.put.assert_called_with("/v1/buckets/foo/src/copy", params={"name": "dst"}) + + +def test_dataproxy_file_copy_to_other_bucket(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + f = DataproxyFile(mock_client, bucket, hash="h", last_modified="n", bytes=1, name="src", content_type="t") + mock_client.put.return_value = MockResp({}, status_code=200) + f.copy_to(dst_bucket="otherbucket") + mock_client.put.assert_called_with("/v1/buckets/foo/src/copy", params={"to": "otherbucket"}) + + +def test_dataproxy_file_copy_to_requires_argument(mock_client): + bucket = Bucket.from_json(mock_client, bucket_json) + f = DataproxyFile(mock_client, bucket, hash="h", last_modified="n", bytes=1, name="src", content_type="t") + with pytest.raises(ValueError): + f.copy_to() + + +# --------------------------- BucketFile URL helper ------------------- # + + +def test_bucket_file_get_file_by_url_bucket(): + from ebrains_drive.file import BucketFile + + client = MagicMock() + bucket = MagicMock() + client.buckets.get_bucket.return_value = bucket + helper = BucketFile(client) + helper.get_file_by_url("https://data-proxy.ebrains.eu/api/v1/buckets/my-bucket/path/to/file.txt") + client.buckets.get_bucket.assert_called_with("my-bucket") + bucket.get_file.assert_called_with("path/to/file.txt") + + +def test_bucket_file_get_file_by_url_dataset(): + from ebrains_drive.file import BucketFile + + client = MagicMock() + ds = MagicMock() + client.buckets.get_dataset.return_value = ds + helper = BucketFile(client) + helper.get_file_by_url("https://data-proxy.ebrains.eu/api/v1/datasets/ds-uuid/a/b.csv") + client.buckets.get_dataset.assert_called_with("ds-uuid") + ds.get_file.assert_called_with("a/b.csv") + + +def test_bucket_file_get_file_by_url_invalid(): + from ebrains_drive.file import BucketFile + + helper = BucketFile(MagicMock()) + with pytest.raises(ValueError): + helper.get_file_by_url("https://unrelated.example.com/foo") + + +# --------------------------- snake_case dirent aliases --------------- # + + +def test_seafdirent_snake_case_aliases_exist(): + from ebrains_drive.files import _SeafDirentBase + + assert _SeafDirentBase.move_to is _SeafDirentBase.moveTo + assert _SeafDirentBase.copy_to is _SeafDirentBase.copyTo From f1ca282348a3f22fa77ef8a10b1ffdb2ed41f846 Mon Sep 17 00:00:00 2001 From: Andrew Davison Date: Sat, 23 May 2026 18:07:04 +0200 Subject: [PATCH 2/2] Update doc.md for harmonised Drive/Bucket interface --- doc.md | 552 +++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 522 insertions(+), 30 deletions(-) diff --git a/doc.md b/doc.md index 6d6f323022..adf44a38f5 100644 --- a/doc.md +++ b/doc.md @@ -12,6 +12,7 @@
  • List all Libraries
  • Create Library
  • Delete Library
    • +
  • Upload (library shortcut)
  • @@ -31,6 +32,7 @@
  • Get Content
  • Create Empty File
  • Upload File
    • +
  • Move and Copy
  • Delete file
    • @@ -42,8 +44,18 @@
  • Bucket
    • +
  • Bucket Directory
    • +
  • Dataset
    • +
  • URL Helper
    • + + +
  • Cross-backend interface
    • + @@ -67,22 +87,30 @@ ## Get Client ## **Request Parameters** -* server * username * password +* token (optional, supplied instead of password) +* env (optional, one of `""` (default, production), `"int"`, `"dev"`) +* target (optional, `"drive"` (default) or `"bucket"` — selects which backend to talk to) **Sample Case** ```python import ebrains_drive - + + # Drive (Seafile) — default target client = ebrains_drive.connect('hbp_username', 'password') + + # Or talk to the Bucket (Data Proxy) backend via the same factory + bucket_client = ebrains_drive.connect('hbp_username', 'password', target="bucket") ``` **Return Type** -A Client Object +A `DriveApiClient` when `target="drive"` (default), a `BucketApiClient` +when `target="bucket"`. For anonymous read access to public buckets, +construct `BucketApiClient()` directly — `connect()` always authenticates. ## Library ## @@ -96,9 +124,10 @@ A Client Object ```python import ebrains_drive - + client = ebrains_drive.connect('hbp_username', 'password') repo = client.repos.get_repo('09c16e2a-ff1a-4207-99f3-1351c3f1e507') + # `client.repos.get(repo_id)` is an alias that mirrors the Bucket surface. ``` **Return Type** @@ -126,6 +155,10 @@ None is_readonly = repo.is_readonly() ``` +`Bucket.is_readonly()` exists with the same signature on the Bucket +backend, so cross-backend code can call `container.is_readonly()` +regardless of which container type it has. + **Return Type** Boolean @@ -160,6 +193,8 @@ None 'obj_test', 'fs_test', 'global'] + + # `client.repos.list()` is an alias for `list_repos()`. ``` **Return Type** @@ -178,9 +213,10 @@ A list of Libraries Object ```python import ebrains_drive - + client = ebrains_drive.connect('hbp_username', 'password') repo = client.repos.create_repo('test_repo') + # `client.repos.create('test_repo')` is an alias. ``` **Return Type** @@ -199,16 +235,46 @@ None ```python import ebrains_drive - + client = ebrains_drive.connect('hbp_username', 'password') repo = client.repos.get_repo('09c16e2a-ff1a-4207-99f3-1351c3f1e507') repo.delete() + # Equivalently: client.repos.delete('09c16e2a-...'). ``` **Return Type** None +### Upload (library shortcut) ### + +**Request Parameters** + +* filelike_or_path (file-like object, `bytes`, or path to a local file) +* filename (name to store the file under, at the root of the library) + +**Sample Case** + +```python + + import ebrains_drive + + client = ebrains_drive.connect('hbp_username', 'password') + repo = client.repos.get_repo('09c16e2a-ff1a-4207-99f3-1351c3f1e507') + + # Path or file-like — both work + repo.upload('/home/jovyan/notes.md', 'notes.md') + repo.upload_local_file('/home/jovyan/notes.md', overwrite=True) +``` + +These are convenience shortcuts for `repo.get_dir("/").upload(...)` and +`repo.get_dir("/").upload_local_file(...)` that mirror +`Bucket.upload` / `Bucket.upload_local_file`. + +**Return Type** + +A `SeafFile` of the newly uploaded file. + ## Directory ## ### Get Directory ### @@ -393,24 +459,29 @@ A File Object **Request Parameters** -None +* progress (keyword-only, default `False`) — when `True`, stream the + download and display a `tqdm` progress bar. **Sample Case** ```python import ebrains_drive - + client = ebrains_drive.connect('hbp_username', 'password') repo = client.repos.get_repo('09c16e2a-ff1a-4207-99f3-1351c3f1e507') seaffile = repo.get_file('/root/test.md') - + content = seaffile.get_content() + content = seaffile.get_content(progress=True) # with progress bar ``` +The `progress=` keyword mirrors `DataproxyFile.get_content(progress=...)` +on the Bucket backend. + **Return Type** -File Content +`bytes` (file content) ### Create Empty File ### **Request Parameters** @@ -438,29 +509,73 @@ A File Object of new empty file ### Upload File ### **Request Parameters** -* filepath -* name (default None, default use local file name) +`upload_local_file`: +* filepath (path to the local file) +* name (default `None`, defaults to the basename of `filepath`) +* overwrite (default `False`; when `True`, replace an existing file with the same name) + +`upload` (polymorphic): +* fileobj — a file-like object, a `bytes` value, or a path/`PathLike` + (the path overload mirrors `Bucket.upload`) +* filename — the name to store the file under **Sample Case** ```python import ebrains_drive - + client = ebrains_drive.connect('hbp_username', 'password') repo = client.repos.get_repo('09c16e2a-ff1a-4207-99f3-1351c3f1e507') seafdir = repo.get_dir('/root') - + + # original two-method form file = seafdir.upload_local_file('/home/ubuntu/env.md') + + # `upload(...)` now also accepts a path directly + file = seafdir.upload('/home/ubuntu/env.md', 'env.md') + + # ...and still accepts a file-like object / bytes + with open('/home/ubuntu/env.md', 'rb') as fp: + file = seafdir.upload(fp, 'env.md') ``` **Return Type** -A File Object of upload file +A File Object of the uploaded file **Exception** * Local file does not exist. +* `FileExistsError` — when `upload_local_file(..., overwrite=False)` + is called with a name that already exists in the directory. + +### Move and Copy ### + +**Request Parameters** + +* dst_dir (destination directory path) +* dst_repo_id (optional; copy/move into another library) + +**Sample Case** + +```python + + import ebrains_drive + + client = ebrains_drive.connect('hbp_username', 'password') + repo = client.repos.get_repo('09c16e2a-ff1a-4207-99f3-1351c3f1e507') + seaffile = repo.get_file('/root/test.md') + + # camelCase (legacy) and snake_case (new) are interchangeable + seaffile.copyTo('/backup') + seaffile.copy_to('/backup') + seaffile.move_to('/archive') +``` + +**Return Type** + +`bool` — `True` on success. ### Delete a file ### @@ -492,25 +607,39 @@ A Response Instance ## Get Client **Request Parameters** -* token +* username (optional) +* password (optional) +* token (optional) +* env (optional) **Sample Case** ```python + # Direct construction from ebrains_drive import BucketApiClient client = BucketApiClient(token="ey...") + + # Or via the shared factory with target="bucket" + import ebrains_drive + client = ebrains_drive.connect(token="ey...", target="bucket") + + # Anonymous read access to public buckets (no token) + anon = BucketApiClient() ``` **Return Type** -A Client Object +A `BucketApiClient`. `client.buckets` is the bucket manager (mirrors +`client.repos` on the Drive side); `client.file` exposes a URL-resolution +helper (see [URL Helper](#bucket_url_helper)). ## Bucket ## ### Get Bucket ### **Request Parameters** * existing_collab_name +* public (keyword-only, default `False`) **Sample Case** @@ -519,6 +648,7 @@ A Client Object from ebrains_drive import BucketApiClient client = BucketApiClient(token="ey...") bucket = client.buckets.get_bucket("existing_collab_name") + # `client.buckets.get("existing_collab_name")` is an alias. ``` **Return Type** @@ -529,10 +659,46 @@ A Bucket Object * Bucket does not exist or not authorized to use the specified bucket +### List Buckets ### + +**Request Parameters** + +* search (optional substring filter on bucket name) + +**Sample Case** + +```python + + from ebrains_drive import BucketApiClient + client = BucketApiClient(token="ey...") + + # all buckets the credentials can access + buckets = client.buckets.list_buckets() + print([b.name for b in buckets]) + + # filter by name substring + atlas_buckets = client.buckets.list_buckets(search="atlas") + # `client.buckets.list()` is an alias. +``` + +The returned `Bucket` objects have `objects_count` and `bytes` set to +`None`. Call `client.buckets.get_bucket(name)` (or `.stat`) to fetch the +full statistics for a specific bucket. + +**Return Type** + +A list of Bucket Objects (each carrying `name`, `role`, `is_public`). + +**Exceptions** + +* `Unauthorized` when the credentials cannot enumerate buckets. + ### Create Bucket ### **Request Parameters** -* new_collab_name +* bucket_name +* title (optional, defaults to `bucket_name`) +* description (optional) **Sample Case** @@ -540,6 +706,12 @@ A Bucket Object from ebrains_drive import BucketApiClient client = BucketApiClient(token="ey...") + + # Canonical entry point + bucket = client.buckets.create_bucket("new_collab_name") + # `client.buckets.create("new_collab_name")` is an alias. + + # Legacy form (still works, but emits a DeprecationWarning): bucket = client.create_new("new_collab_name") ``` @@ -551,7 +723,62 @@ A Bucket Object * Unauthorized to create new collab or bucket -### List Bucket Entries ### +### Delete Bucket ### + +**Request Parameters** + +* bucket_name +* delete_wiki (keyword-only, default `False`; when `True`, also delete the + associated wiki/collab) + +**Sample Case** + +```python + + from ebrains_drive import BucketApiClient + client = BucketApiClient(token="ey...") + + # Three equivalent ways to delete a bucket: + client.buckets.delete_bucket("name") # canonical, manager-level + client.buckets.delete("name", delete_wiki=True) # alias of delete_bucket + client.buckets.get_bucket("name").delete() # instance method on Bucket + # Legacy `client.delete_bucket("name")` still works but emits a + # DeprecationWarning. +``` + +**Return Type** + +None + +**Exceptions** + +* `Unauthorized` when the credentials cannot delete the bucket. + +### Check Bucket Permission ### + +**Request Parameters** + +None + +**Sample Case** + +```python + + bucket = client.buckets.get_bucket("existing_collab_name") + if bucket.is_readonly(): + ... +``` + +Returns `True` when `bucket.role` is `None` or `"viewer"`; `False` for +`"editor"` and `"administrator"`. Dataset buckets and anonymously +accessed public buckets conservatively report read-only. Mirrors +`Repo.is_readonly()` so cross-backend code can use a single check. + +**Return Type** + +Boolean + +### List Bucket Entries (flat) ### **Request Parameters** * prefix (optional) @@ -564,21 +791,137 @@ A Bucket Object client = BucketApiClient(token="ey...") bucket = client.buckets.get_bucket("existing_collab_name") - # shows all files + # shows all files (flat, no directory grouping) all_files = [f for f in bucket.ls()] - # shows all files that begins with path/to/my/files + # shows all files whose name begins with path/to/my/files my_files = [f for f in bucket.ls(prefix="path/to/my/files")] ``` +For a tree-style view that groups objects by `/` boundaries, use +[Bucket Directory](#bucket_dir) instead. + **Return Type** -An Iterator of File Objects +An Iterator of `DataproxyFile` Objects **Exceptions** * Unauthorized +## Bucket Directory ## + +The data-proxy backend is a flat object store, but `BucketDir` exposes +the same prefix-as-directory tree view that the DataProxy GUI uses, so +the same code that works against `SeafDir` can target buckets too. +`BucketDir.ls()` issues a request with the native `delimiter` parameter +so the server returns grouped sub-prefixes — efficient even for large +buckets. + +### Get Directory ### + +**Request Parameters** + +* path (`"/"` returns the bucket root) + +**Sample Case** + +```python + + bucket = client.buckets.get_bucket("existing_collab_name") + + root = bucket.get_dir("/") # bucket root + nested = bucket.get_dir("/path/inside") # arbitrary prefix + deeper = root.get_dir("path").get_dir("inside") # chained navigation +``` + +No HTTP round-trip is made — listing happens lazily. + +**Return Type** + +A `BucketDir` Object + +### List Directory Entries ### + +**Request Parameters** + +* recursive (keyword-only, default `False`) + * `False` — yield only the immediate children of this prefix. Each + entry is either a `BucketDir` (a sub-prefix) or a `DataproxyFile` + (an object directly under this prefix). + * `True` — yield every `DataproxyFile` under this prefix at any depth. + +**Sample Case** + +```python + + root = bucket.get_dir("/") + + # Tree view: subdirectories + files at this level + for entry in root.ls(): + print(entry) + # BucketDir[bucket=foo, path=/dir1] + # BucketDir[bucket=foo, path=/dir2] + # DataproxyFile[bucket=foo, path=top.txt, size=12] + + # Or flatten everything under this prefix: + all_files = list(root.ls(recursive=True)) +``` + +**Return Type** + +An iterator of `BucketDir` and `DataproxyFile` (or just `DataproxyFile` +when `recursive=True`). + +### Create New Directory ### + +**Request Parameters** + +* name + +**Sample Case** + +```python + + root = bucket.get_dir("/") + new_dir = root.mkdir("future") +``` + +**Note:** object-storage directories exist purely by convention — they +spring into existence when an object is uploaded under that prefix. +`mkdir` therefore performs **no HTTP request** and changes no +server-side state; it simply returns a new `BucketDir` for the named +sub-prefix. + +**Return Type** + +A `BucketDir` Object + +### Delete Directory ### + +**Request Parameters** + +* recursive (keyword-only, required; must be `True` to perform the deletion) + +**Sample Case** + +```python + + bucket.get_dir("/sub").delete(recursive=True) +``` + +This deletes every object under the prefix. Calling `delete()` without +`recursive=True` raises `OperationError` — the explicit opt-in is +deliberate because there is no undo. + +**Return Type** + +None + +**Exceptions** + +* `OperationError` — when called without `recursive=True`. + ## Dataset ## ### Get Dataset ### @@ -608,7 +951,10 @@ A Bucket Object ## File ## -Files in buckets are not typically organised in directories. Users may use the `/` in filename to construct a directory-like structure. +Files in buckets are objects in a flat namespace. Names commonly contain +`/` to encode a directory-like structure; see +[Bucket Directory](#bucket_dir) for a tree-style API over that +convention. ### Get File ### @@ -643,7 +989,8 @@ A File Object ### Get File Content ### **Request Parameters** -* filename +* progress (keyword-only, default `False`) — when `True`, stream the + download and display a `tqdm` progress bar. **Sample Case** @@ -658,9 +1005,12 @@ A File Object file_handle = bucket.get_file("filename") file_content = file_handle.get_content() - + file_content = file_handle.get_content(progress=True) # with progress bar ``` +`SeafFile.get_content(progress=...)` exists with the same signature on +the Drive backend. + **Return Type** bytes @@ -672,10 +1022,24 @@ bytes ### Upload File ### + +**`Bucket.upload`** is polymorphic — its first argument may be a path +(`str`/`PathLike`) or a file-like object. **`Bucket.upload_local_file`** +is the strict path-based variant (mirrors `SeafDir.upload_local_file`). + **Request Parameters** -* path_to_file -* dest_filename +`upload`: +* filelike — file-like object, `bytes`, or path to a local file +* filename — name to store the file under in the bucket +* `**kwargs` — passed through to the underlying request (e.g. `headers=`) + +`upload_local_file`: +* filepath — path to the local file +* name (optional) — destination object name; defaults to the basename + of `filepath` +* overwrite (default `False`) — when `False`, raise `FileExistsError` + if an object with that name already exists **Sample Case** @@ -685,8 +1049,16 @@ bytes client = BucketApiClient(token="ey...") bucket = client.buckets.get_bucket("existing_collab_name") - bucket.upload("path_to_file", "dest_filename") + # Path or file-like — both work + bucket.upload("/home/jovyan/test.txt", "test/foobar.txt") + from io import StringIO + fh = StringIO("hello world") + fh.seek(0) + bucket.upload(fh, "test/foobar.txt") + + # Path-only variant with overwrite control + bucket.upload_local_file("/home/jovyan/test.txt", "test/foobar.txt", overwrite=True) ``` **Return Type** @@ -695,12 +1067,51 @@ None **Exceptions** +* Unauthorized +* `FileExistsError` from `upload_local_file(..., overwrite=False)` when + the destination name is already used. + +### Copy File ### + +Server-side copy — no client-side data transfer. Uses the data-proxy's +native `PUT /v1/buckets/{name}/{object}/copy` endpoint. + +**Request Parameters** (at least one must be supplied) + +* dst_name — destination object name (within the same bucket, unless + `dst_bucket` is also set) +* dst_bucket (keyword-only) — destination bucket name + +**Sample Case** + +```python + + bucket = client.buckets.get_bucket("existing_collab_name") + f = bucket.get_file("report.csv") + + # Copy within the same bucket under a new name + f.copy_to(dst_name="report-backup.csv") + + # Copy to a different bucket (same name) + f.copy_to(dst_bucket="archive-bucket") + + # Copy to a different bucket under a new name + f.copy_to(dst_name="report-archived.csv", dst_bucket="archive-bucket") +``` + +**Return Type** + +`bool` — `True` on success. + +**Exceptions** + +* `ValueError` — when neither `dst_name` nor `dst_bucket` is supplied. * Unauthorized ### Delete File ### **Request Parameters** -* filename +None **Sample Case** @@ -724,3 +1135,84 @@ None * Unauthorized * DoesNotExist * AssertionError + +## URL Helper + +`BucketApiClient.file` exposes a URL-resolution helper analogous to +`DriveApiClient.file`. Pass a data-proxy URL and get back a +`DataproxyFile`: + +```python + + client = BucketApiClient(token="ey...") + + f = client.file.get_file_by_url( + "https://data-proxy.ebrains.eu/api/v1/buckets/my-bucket/path/to/file.txt" + ) + + # Dataset URLs are supported too + f = client.file.get_file_by_url( + "https://data-proxy.ebrains.eu/api/v1/datasets//path/to/file" + ) +``` + +**Return Type** + +A `DataproxyFile`. + +**Exceptions** + +* `ValueError` — when the URL does not match the expected data-proxy format. + +# Cross-backend interface + +The Drive (Seafile) and Bucket (Data-Proxy) clients share a harmonised +surface so the same code can target either backend. + +## Protocols + +`ebrains_drive.base` defines four runtime-checkable `typing.Protocol`s +that both backends satisfy structurally (no inheritance required): + +* `StorageClient` — `DriveApiClient`, `BucketApiClient` +* `ContainerManager` — `Repos`, `Buckets` +* `Container` — `Repo`, `Bucket` +* `StorageObject` — `SeafFile`, `DataproxyFile` + +```python + from ebrains_drive.base import Container, StorageObject + + def archive(container: Container, name: str) -> bytes: + obj: StorageObject = container.get_file(name) + return obj.get_content() + + # Works against either backend + archive(repo, "/notes.md") + archive(bucket, "notes.md") + + # isinstance() against the Protocol works at runtime + assert isinstance(bucket, Container) +``` + +## Method comparison table + +| concept | Drive | Bucket | +| --- | --- | --- | +| factory | `connect(target="drive")` | `connect(target="bucket")` | +| manager | `client.repos` | `client.buckets` | +| list | `client.repos.list()` / `list_repos()` | `client.buckets.list()` / `list_buckets()` | +| get | `client.repos.get(id)` / `get_repo(id)` | `client.buckets.get(name)` / `get_bucket(name)` | +| create | `client.repos.create(name)` / `create_repo(name)` | `client.buckets.create(name)` / `create_bucket(name)` | +| delete by name | `client.repos.delete(id)` | `client.buckets.delete(name)` / `delete_bucket(name)` | +| container delete | `repo.delete()` | `bucket.delete()` | +| readonly check | `repo.is_readonly()` | `bucket.is_readonly()` | +| upload (container shortcut) | `repo.upload(filelike_or_path, name)` / `repo.upload_local_file(path)` | `bucket.upload(filelike_or_path, name)` / `bucket.upload_local_file(path)` | +| directory view | `repo.get_dir("/")` → `SeafDir` | `bucket.get_dir("/")` → `BucketDir` | +| download w/ progress | `file.get_content(progress=True)` | `file.get_content(progress=True)` | +| server-side copy | `file.copy_to(dst_dir)` / `copyTo(...)` | `file.copy_to(dst_name=..., dst_bucket=...)` | +| URL helper | `client.file.get_file_by_url(url)` | `client.file.get_file_by_url(url)` | + +The legacy `BucketApiClient.create_new` / `BucketApiClient.delete_bucket` +methods still work but emit a `DeprecationWarning` pointing to the +canonical `client.buckets.create_bucket` / `client.buckets.delete_bucket` +calls.