ci: harden rust release publishing

.github/ISSUE_TEMPLATE/3-new-release.md

@@ -24,6 +24,7 @@ This issue is used to track tasks of the opendal ${opendal_version} release.
   - [ ] nodejs
 - [ ] Update docs
 - [ ] Generate dependencies list
+- [ ] Check Rust crates.io publish plan
 - [ ] Push release candidate tag to GitHub
 
 #### ASF Side
@@ -41,6 +42,7 @@ This issue is used to track tasks of the opendal ${opendal_version} release.
 - [ ] Push the release git tag
 - [ ] Publish artifacts to SVN RELEASE branch
 - [ ] Release Maven artifacts
+- [ ] Check Rust crates.io artifacts
 - [ ] Send the announcement
 
 For details of each step, please refer to: https://opendal.apache.org/community/release/

.github/scripts/release_rust/README.md

@@ -10,6 +10,7 @@ The release workflow needs to:
 - discover all publishable Rust crates under `core/` and `integrations/`
 - exclude crates with `publish = false`
 - publish them in dependency order so local path dependencies are already available on crates.io
+- package crates without repo-local dev-dependencies so same-version dev-dependency cycles don't block publishing
 
 Keeping this logic in a standalone script makes it testable and keeps the workflow YAML readable.
 
@@ -19,6 +20,7 @@ The planner scans:
 
 - `core/Cargo.toml`
 - `core/core/Cargo.toml`
+- `core/testkit/Cargo.toml`
 - `core/layers/*/Cargo.toml`
 - `core/services/*/Cargo.toml`
 - `integrations/*/Cargo.toml`
@@ -40,10 +42,25 @@ Write the same JSON to GitHub Actions output as `packages=<json>`:
 python3 .github/scripts/release_rust/plan.py --github-output
 ```
 
+Publish the planned crates:
+
+```bash
+PACKAGES="$(python3 .github/scripts/release_rust/plan.py)" \
+  python3 .github/scripts/release_rust/publish.py
+```
+
+`publish.py` wraps `cargo publish` with the release-specific behavior we need:
+
+- retries crates.io rate limits by using the server-provided retry time
+- uses `cargo publish --package <name>` so workspace packages publish the intended crate
+- temporarily removes repo-local `dev-dependencies` from the package manifest before publishing
+- restores every touched manifest and lockfile after each package
+
 ## Tests
 
 Run the unit tests with:
 
 ```bash
 python3 .github/scripts/release_rust/test_plan.py
+python3 .github/scripts/release_rust/test_publish.py
 ```

.github/scripts/release_rust/plan.py

@@ -30,6 +30,7 @@
 PUBLISH_GLOBS = (
     "core/Cargo.toml",
     "core/core/Cargo.toml",
+    "core/testkit/Cargo.toml",
     "core/layers/*/Cargo.toml",
     "core/services/*/Cargo.toml",
     "integrations/*/Cargo.toml",

.github/scripts/release_rust/publish.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+import tomllib
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).resolve()
+PROJECT_DIR = SCRIPT_PATH.parents[3]
+
+
+@dataclass(frozen=True)
+class Patch:
+    manifest_path: Path
+    original_manifest: str
+    lockfiles: dict[Path, str]
+
+
+def load_manifest(manifest_path: Path) -> dict:
+    with manifest_path.open("rb") as fp:
+        return tomllib.load(fp)
+
+
+def package_name(manifest_path: Path) -> str:
+    manifest = load_manifest(manifest_path)
+    return manifest["package"]["name"]
+
+
+def collect_lockfiles(project_dir: Path) -> dict[Path, str]:
+    lockfiles: dict[Path, str] = {}
+    for path in project_dir.rglob("Cargo.lock"):
+        if "target" in path.parts:
+            continue
+        lockfiles[path] = path.read_text()
+    return lockfiles
+
+
+def iter_dependency_names(table: dict | None, manifest_dir: Path) -> Iterable[str]:
+    if not isinstance(table, dict):
+        return
+
+    for name, dependency in table.items():
+        if not isinstance(dependency, dict):
+            continue
+        path = dependency.get("path")
+        if not isinstance(path, str):
+            continue
+        if (manifest_dir / path).resolve().is_dir():
+            yield name
+
+
+def local_dev_dependency_names(manifest: dict, manifest_dir: Path) -> set[str]:
+    names = set(iter_dependency_names(manifest.get("dev-dependencies"), manifest_dir))
+
+    for target in manifest.get("target", {}).values():
+        if not isinstance(target, dict):
+            continue
+        names.update(iter_dependency_names(target.get("dev-dependencies"), manifest_dir))
+
+    return names
+
+
+def section_header(line: str) -> str | None:
+    stripped = line.strip()
+    if stripped.startswith("[") and stripped.endswith("]"):
+        return stripped
+    return None
+
+
+def is_dev_dependency_section(header: str | None) -> bool:
+    return header == "[dev-dependencies]" or (
+        header is not None and header.startswith("[target.") and header.endswith(".dev-dependencies]")
+    )
+
+
+def remove_dependency_entry(lines: list[str], start: int) -> int:
+    brace_depth = lines[start].count("{") - lines[start].count("}")
+    bracket_depth = lines[start].count("[") - lines[start].count("]")
+    end = start + 1
+
+    while end < len(lines) and (brace_depth > 0 or bracket_depth > 0):
+        brace_depth += lines[end].count("{") - lines[end].count("}")
+        bracket_depth += lines[end].count("[") - lines[end].count("]")
+        end += 1
+
+    del lines[start:end]
+    return start
+
+
+def strip_local_dev_dependencies(manifest_path: Path, dependency_names: set[str]) -> bool:
+    if not dependency_names:
+        return False
+
+    lines = manifest_path.read_text().splitlines(keepends=True)
+    header = None
+    changed = False
+    index = 0
+
+    while index < len(lines):
+        current_header = section_header(lines[index])
+        if current_header is not None:
+            header = current_header
+            index += 1
+            continue
+
+        if is_dev_dependency_section(header):
+            match = re.match(r"^([A-Za-z0-9_-]+)\s*=", lines[index])
+            if match and match.group(1) in dependency_names:
+                index = remove_dependency_entry(lines, index)
+                changed = True
+                continue
+
+        index += 1
+
+    if changed:
+        manifest_path.write_text("".join(lines))
+    return changed
+
+
+def prepare_manifest(project_dir: Path, package_dir: Path) -> Patch | None:
+    manifest_path = package_dir / "Cargo.toml"
+    manifest = load_manifest(manifest_path)
+    dependency_names = local_dev_dependency_names(manifest, package_dir)
+    if not dependency_names:
+        return None
+
+    patch = Patch(
+        manifest_path=manifest_path,
+        original_manifest=manifest_path.read_text(),
+        lockfiles=collect_lockfiles(project_dir),
+    )
+    changed = strip_local_dev_dependencies(manifest_path, dependency_names)
+    return patch if changed else None
+
+
+def restore(patch: Patch | None) -> None:
+    if patch is None:
+        return
+
+    patch.manifest_path.write_text(patch.original_manifest)
+    for path, content in patch.lockfiles.items():
+        path.write_text(content)
+
+
+def parse_retry_after(output: str) -> int:
+    match = re.search(r"Please try again after ([^\n]+)", output)
+    if match:
+        value = match.group(1).strip().rstrip(".")
+        for parser in (
+            lambda text: parsedate_to_datetime(text),
+            lambda text: datetime.fromisoformat(text.replace("Z", "+00:00")),
+        ):
+            try:
+                retry_at = parser(value)
+                if retry_at.tzinfo is None:
+                    retry_at = retry_at.replace(tzinfo=timezone.utc)
+                return max(60, int((retry_at - datetime.now(timezone.utc)).total_seconds()) + 8)
+            except ValueError:
+                continue
+
+    return 610
+
+
+def should_retry(output: str) -> bool:
+    lowered = output.lower()
+    return (
+        "too many requests" in lowered
+        or "rate limit" in lowered
+        or "you have published too many crates" in lowered
+    )
+
+
+def already_published(output: str) -> bool:
+    lowered = output.lower()
+    return "already uploaded" in lowered or "already exists" in lowered or "is already uploaded" in lowered
+
+
+def publish_package(project_dir: Path, package: str, dry_run: bool) -> None:
+    package_dir = project_dir / package
+    name = package_name(package_dir / "Cargo.toml")
+
+    while True:
+        print(f"Publishing {name} from {package}", flush=True)
+        patch = prepare_manifest(project_dir, package_dir)
+        try:
+            cmd = ["cargo", "publish", "--package", name, "--no-verify"]
+            if dry_run:
+                cmd.append("--dry-run")
+            if patch is not None:
+                cmd.append("--allow-dirty")
+
+            proc = subprocess.run(
+                cmd,
+                cwd=package_dir,
+                check=False,
+                env=os.environ,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            )
+        finally:
+            restore(patch)
+
+        output = proc.stdout or ""
+        print(output, end="", flush=True)
+        if proc.returncode == 0:
+            return
+        if already_published(output):
+            print(f"Skipping {name}: already published", flush=True)
+            return
+        if should_retry(output):
+            sleep_for = parse_retry_after(output)
+            print(f"crates.io rate limited {name}; sleeping {sleep_for}s", flush=True)
+            time.sleep(sleep_for)
+            continue
+
+        raise subprocess.CalledProcessError(proc.returncode, cmd, output=output)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Publish Rust crates for an OpenDAL release.")
+    parser.add_argument(
+        "--project-dir",
+        type=Path,
+        default=PROJECT_DIR,
+        help="Path to the repository root.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Run cargo publish --dry-run for every package without uploading.",
+    )
+    args = parser.parse_args()
+
+    packages = json.loads(os.environ["PACKAGES"])
+    project_dir = args.project_dir.resolve()
+    for package in packages:
+        publish_package(project_dir, package, args.dry_run)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/release_rust/test_plan.py

@@ -73,11 +73,23 @@ def test_fixture_plan_orders_path_dependencies(self):
                 [dependencies]
                 opendal-core = { path = "core", version = "0.1.0" }
                 opendal-layer-retry = { path = "layers/retry", version = "0.1.0" }
+                opendal-testkit = { path = "testkit", version = "0.1.0", optional = true }
 
                 [target.'cfg(unix)'.build-dependencies]
                 opendal-service-fs = { path = "services/fs", version = "0.1.0" }
                 """,
             )
+            write_manifest(
+                root / "core" / "testkit" / "Cargo.toml",
+                """
+                [package]
+                name = "opendal-testkit"
+                version = "0.1.0"
+
+                [dependencies]
+                opendal-core = { path = "../core", version = "0.1.0" }
+                """,
+            )
             write_manifest(
                 root / "integrations" / "object_store" / "Cargo.toml",
                 """
@@ -106,6 +118,7 @@ def test_fixture_plan_orders_path_dependencies(self):
                     "core/core",
                     "core/layers/retry",
                     "core/services/fs",
+                    "core/testkit",
                     "core",
                     "integrations/object_store",
                 ],
@@ -115,17 +128,18 @@ def test_repository_plan_excludes_non_release_paths(self):
         result = plan()
 
         self.assertIn("core/core", result)
+        self.assertIn("core/testkit", result)
         self.assertIn("core", result)
         self.assertIn("integrations/object_store", result)
 
         self.assertNotIn("bindings/python", result)
-        self.assertNotIn("core/testkit", result)
         self.assertNotIn("core/examples/basic", result)
 
     def test_repository_plan_orders_core_before_root_and_integrations(self):
         result = plan()
 
         self.assertLess(result.index("core/core"), result.index("core"))
+        self.assertLess(result.index("core/testkit"), result.index("core"))
         self.assertLess(result.index("core"), result.index("integrations/object_store"))
         self.assertLess(result.index("core"), result.index("integrations/parquet"))
         self.assertLess(result.index("core"), result.index("integrations/unftp-sbe"))