From 7d39d012e1d52ffefc2a0f036001a2be5b6b52e1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 2 May 2026 06:00:26 +0800 Subject: [PATCH] ci: harden rust release publishing --- .github/ISSUE_TEMPLATE/3-new-release.md | 2 + .github/scripts/release_rust/README.md | 17 ++ .github/scripts/release_rust/plan.py | 1 + .github/scripts/release_rust/publish.py | 270 +++++++++++++++++++ .github/scripts/release_rust/test_plan.py | 16 +- .github/scripts/release_rust/test_publish.py | 86 ++++++ .github/workflows/release_rust.yml | 21 +- core/testkit/Cargo.toml | 1 - website/community/release/release.md | 33 +++ 9 files changed, 428 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/release_rust/publish.py create mode 100644 .github/scripts/release_rust/test_publish.py diff --git a/.github/ISSUE_TEMPLATE/3-new-release.md b/.github/ISSUE_TEMPLATE/3-new-release.md index f0784913e627..637e85801741 100644 --- a/.github/ISSUE_TEMPLATE/3-new-release.md +++ b/.github/ISSUE_TEMPLATE/3-new-release.md @@ -24,6 +24,7 @@ This issue is used to track tasks of the opendal ${opendal_version} release. - [ ] nodejs - [ ] Update docs - [ ] Generate dependencies list +- [ ] Check Rust crates.io publish plan - [ ] Push release candidate tag to GitHub #### ASF Side @@ -41,6 +42,7 @@ This issue is used to track tasks of the opendal ${opendal_version} release. - [ ] Push the release git tag - [ ] Publish artifacts to SVN RELEASE branch - [ ] Release Maven artifacts +- [ ] Check Rust crates.io artifacts - [ ] Send the announcement For details of each step, please refer to: https://opendal.apache.org/community/release/ diff --git a/.github/scripts/release_rust/README.md b/.github/scripts/release_rust/README.md index 8840d7abf723..882e42901a51 100644 --- a/.github/scripts/release_rust/README.md +++ b/.github/scripts/release_rust/README.md @@ -10,6 +10,7 @@ The release workflow needs to: - discover all publishable Rust crates under `core/` and `integrations/` - exclude crates with `publish = false` - publish them in dependency order so local path dependencies are already available on crates.io +- package crates without repo-local dev-dependencies so same-version dev-dependency cycles don't block publishing Keeping this logic in a standalone script makes it testable and keeps the workflow YAML readable. @@ -19,6 +20,7 @@ The planner scans: - `core/Cargo.toml` - `core/core/Cargo.toml` +- `core/testkit/Cargo.toml` - `core/layers/*/Cargo.toml` - `core/services/*/Cargo.toml` - `integrations/*/Cargo.toml` @@ -40,10 +42,25 @@ Write the same JSON to GitHub Actions output as `packages=`: python3 .github/scripts/release_rust/plan.py --github-output ``` +Publish the planned crates: + +```bash +PACKAGES="$(python3 .github/scripts/release_rust/plan.py)" \ + python3 .github/scripts/release_rust/publish.py +``` + +`publish.py` wraps `cargo publish` with the release-specific behavior we need: + +- retries crates.io rate limits by using the server-provided retry time +- uses `cargo publish --package ` so workspace packages publish the intended crate +- temporarily removes repo-local `dev-dependencies` from the package manifest before publishing +- restores every touched manifest and lockfile after each package + ## Tests Run the unit tests with: ```bash python3 .github/scripts/release_rust/test_plan.py +python3 .github/scripts/release_rust/test_publish.py ``` diff --git a/.github/scripts/release_rust/plan.py b/.github/scripts/release_rust/plan.py index c7c15c850fa6..2f748a2daaa2 100644 --- a/.github/scripts/release_rust/plan.py +++ b/.github/scripts/release_rust/plan.py @@ -30,6 +30,7 @@ PUBLISH_GLOBS = ( "core/Cargo.toml", "core/core/Cargo.toml", + "core/testkit/Cargo.toml", "core/layers/*/Cargo.toml", "core/services/*/Cargo.toml", "integrations/*/Cargo.toml", diff --git a/.github/scripts/release_rust/publish.py b/.github/scripts/release_rust/publish.py new file mode 100644 index 000000000000..9472457ebe25 --- /dev/null +++ b/.github/scripts/release_rust/publish.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import json +import os +import re +import subprocess +import time +import tomllib +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import datetime +from datetime import timezone +from email.utils import parsedate_to_datetime +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).resolve() +PROJECT_DIR = SCRIPT_PATH.parents[3] + + +@dataclass(frozen=True) +class Patch: + manifest_path: Path + original_manifest: str + lockfiles: dict[Path, str] + + +def load_manifest(manifest_path: Path) -> dict: + with manifest_path.open("rb") as fp: + return tomllib.load(fp) + + +def package_name(manifest_path: Path) -> str: + manifest = load_manifest(manifest_path) + return manifest["package"]["name"] + + +def collect_lockfiles(project_dir: Path) -> dict[Path, str]: + lockfiles: dict[Path, str] = {} + for path in project_dir.rglob("Cargo.lock"): + if "target" in path.parts: + continue + lockfiles[path] = path.read_text() + return lockfiles + + +def iter_dependency_names(table: dict | None, manifest_dir: Path) -> Iterable[str]: + if not isinstance(table, dict): + return + + for name, dependency in table.items(): + if not isinstance(dependency, dict): + continue + path = dependency.get("path") + if not isinstance(path, str): + continue + if (manifest_dir / path).resolve().is_dir(): + yield name + + +def local_dev_dependency_names(manifest: dict, manifest_dir: Path) -> set[str]: + names = set(iter_dependency_names(manifest.get("dev-dependencies"), manifest_dir)) + + for target in manifest.get("target", {}).values(): + if not isinstance(target, dict): + continue + names.update(iter_dependency_names(target.get("dev-dependencies"), manifest_dir)) + + return names + + +def section_header(line: str) -> str | None: + stripped = line.strip() + if stripped.startswith("[") and stripped.endswith("]"): + return stripped + return None + + +def is_dev_dependency_section(header: str | None) -> bool: + return header == "[dev-dependencies]" or ( + header is not None and header.startswith("[target.") and header.endswith(".dev-dependencies]") + ) + + +def remove_dependency_entry(lines: list[str], start: int) -> int: + brace_depth = lines[start].count("{") - lines[start].count("}") + bracket_depth = lines[start].count("[") - lines[start].count("]") + end = start + 1 + + while end < len(lines) and (brace_depth > 0 or bracket_depth > 0): + brace_depth += lines[end].count("{") - lines[end].count("}") + bracket_depth += lines[end].count("[") - lines[end].count("]") + end += 1 + + del lines[start:end] + return start + + +def strip_local_dev_dependencies(manifest_path: Path, dependency_names: set[str]) -> bool: + if not dependency_names: + return False + + lines = manifest_path.read_text().splitlines(keepends=True) + header = None + changed = False + index = 0 + + while index < len(lines): + current_header = section_header(lines[index]) + if current_header is not None: + header = current_header + index += 1 + continue + + if is_dev_dependency_section(header): + match = re.match(r"^([A-Za-z0-9_-]+)\s*=", lines[index]) + if match and match.group(1) in dependency_names: + index = remove_dependency_entry(lines, index) + changed = True + continue + + index += 1 + + if changed: + manifest_path.write_text("".join(lines)) + return changed + + +def prepare_manifest(project_dir: Path, package_dir: Path) -> Patch | None: + manifest_path = package_dir / "Cargo.toml" + manifest = load_manifest(manifest_path) + dependency_names = local_dev_dependency_names(manifest, package_dir) + if not dependency_names: + return None + + patch = Patch( + manifest_path=manifest_path, + original_manifest=manifest_path.read_text(), + lockfiles=collect_lockfiles(project_dir), + ) + changed = strip_local_dev_dependencies(manifest_path, dependency_names) + return patch if changed else None + + +def restore(patch: Patch | None) -> None: + if patch is None: + return + + patch.manifest_path.write_text(patch.original_manifest) + for path, content in patch.lockfiles.items(): + path.write_text(content) + + +def parse_retry_after(output: str) -> int: + match = re.search(r"Please try again after ([^\n]+)", output) + if match: + value = match.group(1).strip().rstrip(".") + for parser in ( + lambda text: parsedate_to_datetime(text), + lambda text: datetime.fromisoformat(text.replace("Z", "+00:00")), + ): + try: + retry_at = parser(value) + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + return max(60, int((retry_at - datetime.now(timezone.utc)).total_seconds()) + 8) + except ValueError: + continue + + return 610 + + +def should_retry(output: str) -> bool: + lowered = output.lower() + return ( + "too many requests" in lowered + or "rate limit" in lowered + or "you have published too many crates" in lowered + ) + + +def already_published(output: str) -> bool: + lowered = output.lower() + return "already uploaded" in lowered or "already exists" in lowered or "is already uploaded" in lowered + + +def publish_package(project_dir: Path, package: str, dry_run: bool) -> None: + package_dir = project_dir / package + name = package_name(package_dir / "Cargo.toml") + + while True: + print(f"Publishing {name} from {package}", flush=True) + patch = prepare_manifest(project_dir, package_dir) + try: + cmd = ["cargo", "publish", "--package", name, "--no-verify"] + if dry_run: + cmd.append("--dry-run") + if patch is not None: + cmd.append("--allow-dirty") + + proc = subprocess.run( + cmd, + cwd=package_dir, + check=False, + env=os.environ, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + finally: + restore(patch) + + output = proc.stdout or "" + print(output, end="", flush=True) + if proc.returncode == 0: + return + if already_published(output): + print(f"Skipping {name}: already published", flush=True) + return + if should_retry(output): + sleep_for = parse_retry_after(output) + print(f"crates.io rate limited {name}; sleeping {sleep_for}s", flush=True) + time.sleep(sleep_for) + continue + + raise subprocess.CalledProcessError(proc.returncode, cmd, output=output) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Publish Rust crates for an OpenDAL release.") + parser.add_argument( + "--project-dir", + type=Path, + default=PROJECT_DIR, + help="Path to the repository root.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Run cargo publish --dry-run for every package without uploading.", + ) + args = parser.parse_args() + + packages = json.loads(os.environ["PACKAGES"]) + project_dir = args.project_dir.resolve() + for package in packages: + publish_package(project_dir, package, args.dry_run) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/release_rust/test_plan.py b/.github/scripts/release_rust/test_plan.py index f7c227d09dad..d630c85e70ac 100644 --- a/.github/scripts/release_rust/test_plan.py +++ b/.github/scripts/release_rust/test_plan.py @@ -73,11 +73,23 @@ def test_fixture_plan_orders_path_dependencies(self): [dependencies] opendal-core = { path = "core", version = "0.1.0" } opendal-layer-retry = { path = "layers/retry", version = "0.1.0" } + opendal-testkit = { path = "testkit", version = "0.1.0", optional = true } [target.'cfg(unix)'.build-dependencies] opendal-service-fs = { path = "services/fs", version = "0.1.0" } """, ) + write_manifest( + root / "core" / "testkit" / "Cargo.toml", + """ + [package] + name = "opendal-testkit" + version = "0.1.0" + + [dependencies] + opendal-core = { path = "../core", version = "0.1.0" } + """, + ) write_manifest( root / "integrations" / "object_store" / "Cargo.toml", """ @@ -106,6 +118,7 @@ def test_fixture_plan_orders_path_dependencies(self): "core/core", "core/layers/retry", "core/services/fs", + "core/testkit", "core", "integrations/object_store", ], @@ -115,17 +128,18 @@ def test_repository_plan_excludes_non_release_paths(self): result = plan() self.assertIn("core/core", result) + self.assertIn("core/testkit", result) self.assertIn("core", result) self.assertIn("integrations/object_store", result) self.assertNotIn("bindings/python", result) - self.assertNotIn("core/testkit", result) self.assertNotIn("core/examples/basic", result) def test_repository_plan_orders_core_before_root_and_integrations(self): result = plan() self.assertLess(result.index("core/core"), result.index("core")) + self.assertLess(result.index("core/testkit"), result.index("core")) self.assertLess(result.index("core"), result.index("integrations/object_store")) self.assertLess(result.index("core"), result.index("integrations/parquet")) self.assertLess(result.index("core"), result.index("integrations/unftp-sbe")) diff --git a/.github/scripts/release_rust/test_publish.py b/.github/scripts/release_rust/test_publish.py new file mode 100644 index 000000000000..2da69b3817fa --- /dev/null +++ b/.github/scripts/release_rust/test_publish.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import tempfile +import textwrap +import unittest +from pathlib import Path + +from publish import local_dev_dependency_names +from publish import load_manifest +from publish import strip_local_dev_dependencies + + +def write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(content).strip() + "\n", encoding="utf-8") + + +class ReleaseRustPublishTest(unittest.TestCase): + def test_strip_local_dev_dependencies(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + package = root / "crate" + write( + package / "Cargo.toml", + """ + [package] + name = "crate" + version = "0.1.0" + + [dependencies] + local-normal = { path = "../local-normal", version = "0.1.0" } + + [dev-dependencies] + external-dev = "1" + local-dev = { path = "../local-dev", version = "0.1.0" } + local-dev-multiline = { path = "../local-dev-multiline", version = "0.1.0", features = [ + "test", + ] } + + [target.'cfg(unix)'.dev-dependencies] + local-target-dev = { path = "../local-target-dev", version = "0.1.0" } + """, + ) + for name in ( + "local-normal", + "local-dev", + "local-dev-multiline", + "local-target-dev", + ): + (root / name).mkdir() + + manifest = load_manifest(package / "Cargo.toml") + names = local_dev_dependency_names(manifest, package) + self.assertEqual( + names, + {"local-dev", "local-dev-multiline", "local-target-dev"}, + ) + + changed = strip_local_dev_dependencies(package / "Cargo.toml", names) + self.assertTrue(changed) + + stripped = (package / "Cargo.toml").read_text() + self.assertIn("local-normal", stripped) + self.assertIn("external-dev", stripped) + self.assertNotIn("local-dev =", stripped) + self.assertNotIn("local-dev-multiline", stripped) + self.assertNotIn("local-target-dev", stripped) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/release_rust.yml b/.github/workflows/release_rust.yml index 549e7d96317f..564058245ecf 100644 --- a/.github/workflows/release_rust.yml +++ b/.github/workflows/release_rust.yml @@ -28,6 +28,8 @@ on: - main paths: - ".github/workflows/release_rust.yml" + - ".github/scripts/release_rust/**" + - "core/testkit/Cargo.toml" workflow_dispatch: permissions: @@ -69,23 +71,8 @@ jobs: - uses: rust-lang/crates-io-auth-action@b7e9a28eded4986ec6b1fa40eeee8f8f165559ec id: auth - name: Publish Rust crates - shell: python env: PACKAGES: ${{ needs.plan.outputs.packages }} LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }} - CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} - run: | - import json - import os - import subprocess - - packages = json.loads(os.environ["PACKAGES"]) - - for package in packages: - print(f"Publishing {package}") - subprocess.run( - ["cargo", "publish", "--no-verify"], - cwd=package, - check=True, - env=os.environ, - ) + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN || steps.auth.outputs.token }} + run: python3 .github/scripts/release_rust/publish.py diff --git a/core/testkit/Cargo.toml b/core/testkit/Cargo.toml index bd719a05a381..537964a84633 100644 --- a/core/testkit/Cargo.toml +++ b/core/testkit/Cargo.toml @@ -18,7 +18,6 @@ [package] description = "Test harness and utilities for Apache OpenDAL" name = "opendal-testkit" -publish = false authors = { workspace = true } edition = { workspace = true } diff --git a/website/community/release/release.md b/website/community/release/release.md index ac2a2c1979d7..2a576a57ea93 100644 --- a/website/community/release/release.md +++ b/website/community/release/release.md @@ -130,6 +130,34 @@ After pushing the tag, check the GitHub action status to make sure the RC workfl In the most cases, it would be great to rerun the failed workflow directly when you find some failures. But if a new code patch is needed to fix the failure, you should create a new release candidate tag, increase the rc number and push it to GitHub. +### Check Rust crates.io readiness + +Before the official release tag is pushed, check the Rust package publish plan: + +```shell +python3 .github/scripts/release_rust/plan.py +``` + +The plan must include every Rust package that is referenced by released crates. +For example, `opendal-testkit` is referenced by the `opendal` crate's `tests` +feature, so it must be publishable and appear before `opendal` in the plan. + +The Rust release workflow uses `.github/scripts/release_rust/publish.py` instead +of calling `cargo publish` directly. The helper temporarily removes repo-local +`dev-dependencies` while packaging crates, because crates.io resolves +`dev-dependencies` even when `cargo publish --no-verify` is used. Without this +step, same-version dev dependencies and dev-only cycles can block the release +even though they are not needed by downstream users. + +:::caution + +crates.io trusted publishing tokens cannot create new crates. If this release +introduces a new crate name, make sure a crates.io token that is allowed to +create crates is available as `CARGO_REGISTRY_TOKEN`, or publish the new crate +manually before relying on trusted publishing. + +::: + ## ASF Side If any step in the ASF Release process fails and requires code changes, @@ -418,10 +446,15 @@ If the vote failed, click "Drop" to drop the staging Maven artifacts. We need to check the language binding artifacts in the language package repo to make sure they are released successfully. +- Rust: - Python: - Java: - Node.js: +For Rust crates, check both the top-level `opendal` crate and split crates that +were added or changed in the release, such as `opendal-service-*`, +`opendal-layer-*`, and integration crates. + For Java binding, if we cannot find the latest version of artifacts in the repo, we need to check the `orgapacheopendal-${maven_artifact_number}` artifact status in staging repo.