Publish #6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Tag-driven crates.io + npm publish for the basecrawl public surface. | |
| # | |
| # Triggers: | |
| # - push of tags matching v* → live publish (after dry-run gate) | |
| # - workflow_dispatch → optional dry-run (default true) | |
| # | |
| # Ordered cargo publish topology (VAL-CRATES-001): | |
| # basecrawl-headless-chrome → proof → fp → seal → render → core → ffi → thin basecrawl | |
| # | |
| # Secrets (names only; never print, never hard-code): | |
| # CARGO_REGISTRY_TOKEN — crates.io API token | |
| # NPM_TOKEN — npm access token for @basecrawl/sdk (NODE_AUTH_TOKEN) | |
| # | |
| # Quality: this workflow re-runs CI parity (fmt / clippy / workspace tests). | |
| # Existing .github/workflows/ci.yml remains the continuous quality gate and is not modified. | |
| # image.yml remains the separate GHCR path. | |
| name: Publish | |
| on: | |
| push: | |
| tags: | |
| - "v*" | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| description: "Dry-run only (no live crates.io / npm publish)" | |
| type: boolean | |
| default: true | |
| permissions: | |
| contents: read | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: "1" | |
| # Publish topology — single source of order for dry-run and live steps. | |
| PUBLISH_CRATES: >- | |
| basecrawl-headless-chrome | |
| basecrawl-proof | |
| basecrawl-fp | |
| basecrawl-seal | |
| basecrawl-render | |
| basecrawl-core | |
| basecrawl-ffi | |
| basecrawl | |
| jobs: | |
| # --------------------------------------------------------------------------- | |
| # Quality gate (CI parity with ci.yml: fmt, clippy -D warnings, workspace tests) | |
| # --------------------------------------------------------------------------- | |
| quality: | |
| name: Quality (CI parity) | |
| runs-on: ubuntu-latest | |
| services: | |
| httpbin: | |
| # Classic Python httpbin for hermetic HTTP-semantics tests (matches ci.yml). | |
| image: kennethreitz/httpbin:latest | |
| ports: | |
| - 8080:80 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: Cache cargo | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Cargo fmt | |
| run: cargo fmt --all -- --check | |
| - name: Cargo clippy | |
| run: cargo clippy --workspace --all-targets --all-features -- -D warnings | |
| - name: Wait for hermetic httpbin | |
| run: | | |
| for i in $(seq 1 60); do | |
| if curl -fsS -o /dev/null -w "%{http_code}" "http://127.0.0.1:8080/get" | grep -q '^200$'; then | |
| echo "httpbin ready" | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "hermetic httpbin did not become ready" >&2 | |
| exit 1 | |
| - name: Cargo test | |
| run: cargo test --workspace --all-features | |
| env: | |
| RUST_TEST_THREADS: "1" | |
| BASECRAWL_HTTPBIN_BASE: http://127.0.0.1:8080 | |
| # --------------------------------------------------------------------------- | |
| # Version match: tag vX.Y.Z == workspace package version == npm package.json | |
| # --------------------------------------------------------------------------- | |
| version-check: | |
| name: Version match | |
| runs-on: ubuntu-latest | |
| needs: quality | |
| outputs: | |
| version: ${{ steps.resolve.outputs.version }} | |
| dry_run: ${{ steps.resolve.outputs.dry_run }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Resolve version and dry-run mode | |
| id: resolve | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| WORKSPACE_VERSION="$( | |
| python3 - <<'PY' | |
| import re, pathlib | |
| text = pathlib.Path("Cargo.toml").read_text() | |
| # Prefer [workspace.package] version = "X.Y.Z" | |
| m = re.search( | |
| r'(?ms)^\[workspace\.package\]\s*.*?^version\s*=\s*"([^"]+)"', | |
| text, | |
| ) | |
| if not m: | |
| raise SystemExit("workspace.package.version not found in Cargo.toml") | |
| print(m.group(1)) | |
| PY | |
| )" | |
| NPM_VERSION="$( | |
| python3 - <<'PY' | |
| import json, pathlib | |
| pkg = json.loads(pathlib.Path("bindings/node/package.json").read_text()) | |
| print(pkg.get("version", "")) | |
| PY | |
| )" | |
| if [[ -z "${WORKSPACE_VERSION}" ]]; then | |
| echo "workspace version empty" >&2 | |
| exit 1 | |
| fi | |
| if [[ -z "${NPM_VERSION}" ]]; then | |
| echo "bindings/node/package.json version empty" >&2 | |
| exit 1 | |
| fi | |
| if [[ "${NPM_VERSION}" != "${WORKSPACE_VERSION}" ]]; then | |
| echo "version mismatch: workspace=${WORKSPACE_VERSION} npm=${NPM_VERSION}" >&2 | |
| exit 1 | |
| fi | |
| # Determine dry-run: workflow_dispatch defaults to dry-run; tag push is live. | |
| DRY_RUN="false" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| if [[ "${{ inputs.dry_run }}" == "true" ]]; then | |
| DRY_RUN="true" | |
| else | |
| DRY_RUN="false" | |
| fi | |
| EXPECTED="${WORKSPACE_VERSION}" | |
| echo "workflow_dispatch: using workspace version ${EXPECTED} (dry_run=${DRY_RUN})" | |
| else | |
| # Tag push: refs/tags/vX.Y.Z | |
| REF_NAME="${GITHUB_REF_NAME:-}" | |
| if [[ ! "${REF_NAME}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-].*)?$ ]]; then | |
| echo "tag '${REF_NAME}' does not match vX.Y.Z*" >&2 | |
| exit 1 | |
| fi | |
| TAG_VERSION="${REF_NAME#v}" | |
| if [[ "${TAG_VERSION}" != "${WORKSPACE_VERSION}" ]]; then | |
| echo "tag version ${TAG_VERSION} != workspace/npm version ${WORKSPACE_VERSION}" >&2 | |
| exit 1 | |
| fi | |
| EXPECTED="${TAG_VERSION}" | |
| DRY_RUN="false" | |
| echo "tag push: version ${EXPECTED} (live publish after dry-run gate)" | |
| fi | |
| # Also fail if any public workspace crate pin drifts from workspace version. | |
| for crate_toml in \ | |
| crates/basecrawl-proof/Cargo.toml \ | |
| crates/basecrawl-fp/Cargo.toml \ | |
| crates/basecrawl-seal/Cargo.toml \ | |
| crates/basecrawl-render/Cargo.toml \ | |
| crates/basecrawl-core/Cargo.toml \ | |
| crates/basecrawl-ffi/Cargo.toml \ | |
| crates/basecrawl/Cargo.toml | |
| do | |
| if ! grep -q 'version.workspace = true' "${crate_toml}"; then | |
| # bare version= allowed only if it matches EXPECTED | |
| ver="$(python3 -c "import re,pathlib,sys; t=pathlib.Path(sys.argv[1]).read_text(); m=re.search(r'^version\s*=\s*\"([^\"]+)\"', t, re.M); print(m.group(1) if m else '')" "${crate_toml}")" | |
| if [[ "${ver}" != "${EXPECTED}" ]]; then | |
| echo "crate ${crate_toml} version '${ver}' != expected ${EXPECTED}" >&2 | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| FORK_VERSION="$( | |
| python3 -c "import re,pathlib; t=pathlib.Path('crates/basecrawl-headless-chrome/Cargo.toml').read_text(); m=re.search(r'^version\s*=\s*\"([^\"]+)\"', t, re.M); print(m.group(1) if m else '')" | |
| )" | |
| if [[ "${FORK_VERSION}" != "${EXPECTED}" ]]; then | |
| echo "basecrawl-headless-chrome version ${FORK_VERSION} != expected ${EXPECTED}" >&2 | |
| exit 1 | |
| fi | |
| echo "version=${EXPECTED}" >> "${GITHUB_OUTPUT}" | |
| echo "dry_run=${DRY_RUN}" >> "${GITHUB_OUTPUT}" | |
| echo "Resolved publish version=${EXPECTED} dry_run=${DRY_RUN}" | |
| # --------------------------------------------------------------------------- | |
| # Ordered cargo publish (dry-run gate always; live only when not dry_run) | |
| # --------------------------------------------------------------------------- | |
| crates: | |
| name: crates.io ordered publish | |
| runs-on: ubuntu-latest | |
| needs: version-check | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache cargo | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Ordered dry-run then optional live publish | |
| shell: bash | |
| env: | |
| # Secret by name only when live publishing. Never echo raw value. | |
| # GitHub Actions masks secret-backed env values in logs. | |
| CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} | |
| DRY_RUN: ${{ needs.version-check.outputs.dry_run }} | |
| PUBLISH_VERSION: ${{ needs.version-check.outputs.version }} | |
| run: | | |
| set -euo pipefail | |
| is_dry="${DRY_RUN}" | |
| echo "Ordered crates publish for version=${PUBLISH_VERSION} dry_run=${is_dry}" | |
| echo "Topology: basecrawl-headless-chrome → proof → fp → seal → render → core → ffi → basecrawl" | |
| if [[ "${is_dry}" != "true" ]]; then | |
| if [[ -z "${CARGO_REGISTRY_TOKEN:-}" ]]; then | |
| echo "CARGO_REGISTRY_TOKEN secret is not set on this repository" >&2 | |
| exit 1 | |
| fi | |
| # Length-only presence check — never print the token value. | |
| echo "CARGO_REGISTRY_TOKEN is present (length=${#CARGO_REGISTRY_TOKEN})" | |
| else | |
| # Dry-run mode must not require a registry token. | |
| unset CARGO_REGISTRY_TOKEN || true | |
| echo "dry_run mode: CARGO_REGISTRY_TOKEN unset for tokenless dry packaging" | |
| fi | |
| # Per-crate: dry-run/verify first, then live publish (VAL-CRATES-001/002). | |
| # Dependents convert dual path+version → version-only on package, so they can only | |
| # dry-run once prior crates in the chain exist on crates.io. Live path therefore | |
| # interleaves: dry-run → publish → brief index pause → next crate. | |
| # | |
| # Pure dry-run workflow_dispatch (no prior registry presence): leaf crates get full | |
| # `cargo publish --dry-run`; dependents missing unreleased workspace deps fall back to | |
| # local `cargo check -p` plus dual-version metadata assert (still refuse reverse order). | |
| missing_registry_dep() { | |
| printf '%s\n' "$1" | grep -Eqi 'no matching package named|failed to select a version for the requirement' | |
| } | |
| already_uploaded() { | |
| printf '%s\n' "$1" | grep -Eqi 'already exists|already uploaded' | |
| } | |
| for crate in ${PUBLISH_CRATES}; do | |
| echo "::group::${crate}" | |
| dry_ok=0 | |
| attempt=1 | |
| max_attempts=12 | |
| while [[ ${attempt} -le ${max_attempts} ]]; do | |
| set +e | |
| dry_out="$(cargo publish --dry-run -p "${crate}" 2>&1)" | |
| dry_ec=$? | |
| set -e | |
| printf '%s\n' "${dry_out}" | |
| if [[ ${dry_ec} -eq 0 ]]; then | |
| dry_ok=1 | |
| break | |
| fi | |
| if missing_registry_dep "${dry_out}"; then | |
| if [[ "${is_dry}" == "true" ]]; then | |
| echo "NOTE: ${crate} dry-run blocked on crates.io absence of an a prior workspace dep (expected pre-first-release)." | |
| echo "Fallback: local cargo check -p ${crate} + dual path+version metadata assert." | |
| cargo check -p "${crate}" | |
| # Ensure dual path+version remains for monorepo packaging (not path-only). | |
| if [[ "${crate}" != "basecrawl-headless-chrome" && "${crate}" != "basecrawl-proof" && "${crate}" != "basecrawl-fp" && "${crate}" != "basecrawl-seal" ]]; then | |
| echo "dependent crate ${crate}: local check green; full publish --dry-run needs prior chain on registry (live tag path)." | |
| fi | |
| dry_ok=2 | |
| break | |
| fi | |
| # Live path: prior crate just published — wait for crates.io index to catch up. | |
| echo "waiting for crates.io index before dry-run of ${crate} (attempt ${attempt}/${max_attempts})" | |
| sleep 15 | |
| attempt=$((attempt + 1)) | |
| continue | |
| fi | |
| echo "cargo publish --dry-run -p ${crate} failed" >&2 | |
| exit ${dry_ec} | |
| done | |
| if [[ ${dry_ok} -eq 0 ]]; then | |
| echo "exhausted retries waiting for registry deps for ${crate}" >&2 | |
| exit 1 | |
| fi | |
| if [[ "${is_dry}" == "true" ]]; then | |
| echo "dry_run: skip live cargo publish for ${crate}" | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| # Live publish (token via CARGO_REGISTRY_TOKEN env; cargo never prints it). | |
| set +e | |
| live_out="$(cargo publish -p "${crate}" 2>&1)" | |
| live_ec=$? | |
| set -e | |
| printf '%s\n' "${live_out}" | |
| if [[ ${live_ec} -ne 0 ]]; then | |
| if already_uploaded "${live_out}"; then | |
| echo "crate ${crate} already on crates.io — continuing ordered chain" | |
| else | |
| echo "cargo publish -p ${crate} failed" >&2 | |
| exit ${live_ec} | |
| fi | |
| fi | |
| # Index pause between dependents so the next dry-run can resolve. | |
| echo "published ${crate}; pausing for crates.io index" | |
| sleep 12 | |
| echo "::endgroup::" | |
| done | |
| if [[ "${is_dry}" == "true" ]]; then | |
| echo "dry_run complete: ordered topology walked; live publish skipped" | |
| echo "Cut tag v${PUBLISH_VERSION} (or workflow_dispatch dry_run=false) for live registry upload" | |
| else | |
| echo "Live ordered crates.io publish chain complete" | |
| fi | |
| # --------------------------------------------------------------------------- | |
| # npm: @basecrawl/sdk (linux-x64 host). Multi-OS matrix is NOT required for M25. | |
| # --------------------------------------------------------------------------- | |
| npm: | |
| name: npm @basecrawl/sdk | |
| runs-on: ubuntu-latest | |
| needs: | |
| - version-check | |
| - crates | |
| defaults: | |
| run: | |
| working-directory: bindings/node | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache cargo | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| registry-url: "https://registry.npmjs.org" | |
| # Scope for @basecrawl/sdk publish. | |
| scope: "@basecrawl" | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 9 | |
| - name: Install Node build deps | |
| run: pnpm install | |
| - name: Assert linux-x64 honesty metadata (package.json) | |
| # VAL-NPM-003: single-arch ship must declare os/cpu; never multi-OS claim without artifacts. | |
| run: | | |
| set -euo pipefail | |
| node -e " | |
| const pkg = require('./package.json'); | |
| if (pkg.name !== '@basecrawl/sdk') { | |
| throw new Error('package name must be @basecrawl/sdk, got ' + pkg.name); | |
| } | |
| const os = pkg.os || []; | |
| const cpu = pkg.cpu || []; | |
| if (!(Array.isArray(os) && os.length === 1 && os[0] === 'linux')) { | |
| throw new Error('package.os must be [\"linux\"] for M25 single-arch honesty, got ' + JSON.stringify(os)); | |
| } | |
| if (!(Array.isArray(cpu) && cpu.length === 1 && cpu[0] === 'x64')) { | |
| throw new Error('package.cpu must be [\"x64\"] for M25 single-arch honesty, got ' + JSON.stringify(cpu)); | |
| } | |
| console.log('linux-x64 honesty metadata OK:', pkg.name, 'os=', os, 'cpu=', cpu); | |
| " | |
| - name: Build native addon (linux-x64 honesty / prepack) | |
| # M25 ships the Linux CI host artifact only. Multi-OS napi matrix is out of scope. | |
| # package.json os/cpu + README residual constrain consumers to linux-x64. | |
| run: | | |
| set -euo pipefail | |
| uname -sm | |
| node -p "process.platform+'-'+process.arch" | |
| # prepack is what npm pack / publish lifecycle runs; build produces basecrawl_sdk.node. | |
| pnpm run prepack | |
| test -f basecrawl_sdk.node | |
| file basecrawl_sdk.node || true | |
| - name: Smoke require / version (linux host) | |
| run: | | |
| set -euo pipefail | |
| pnpm run smoke:linux | |
| node -e " | |
| const sdk = require('./index.js'); | |
| const v = typeof sdk.version === 'function' ? sdk.version() : sdk.version; | |
| console.log('sdk.version=', v); | |
| if (!v) { | |
| throw new Error('missing sdk version after native load'); | |
| } | |
| if (typeof sdk.scrape !== 'function') { | |
| throw new Error('missing scrape export'); | |
| } | |
| console.log('linux smoke require OK'); | |
| " | |
| - name: npm pack dry-run (always) | |
| run: | | |
| set -euo pipefail | |
| pnpm pack --pack-destination /tmp | |
| ls -la /tmp/*.tgz | |
| # List packing content for linux honesty audit (no secrets). | |
| listing="$(tar -tzf /tmp/basecrawl-sdk-*.tgz 2>/dev/null || tar -tzf /tmp/*.tgz)" | |
| printf '%s\n' "${listing}" | sed -n '1,80p' | |
| printf '%s\n' "${listing}" | grep -E 'basecrawl_sdk\.node$' >/dev/null | |
| printf '%s\n' "${listing}" | grep -E 'README\.md$' >/dev/null | |
| # Refuse multi-OS artifact names in the shipping tarball. | |
| if printf '%s\n' "${listing}" | grep -Eiq 'darwin|win32|windows|msvc|apple-darwin'; then | |
| echo "pack listing must not claim multi-OS native artifacts" >&2 | |
| exit 1 | |
| fi | |
| - name: Live npm publish --access public | |
| if: needs.version-check.outputs.dry_run != 'true' | |
| id: npm_publish | |
| continue-on-error: true | |
| env: | |
| # NPM_TOKEN by secret name only → NODE_AUTH_TOKEN for setup-node registry-url. | |
| # Never echo, never log the raw value. | |
| NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -z "${NODE_AUTH_TOKEN:-}" ]]; then | |
| echo "NPM_TOKEN secret is not set on this repository" >&2 | |
| exit 1 | |
| fi | |
| echo "NPM_TOKEN is present (length=${#NODE_AUTH_TOKEN})" | |
| # Confirm package name before publish (must be @basecrawl/sdk). | |
| NAME="$(node -p "require('./package.json').name")" | |
| VER="$(node -p "require('./package.json').version")" | |
| echo "Publishing ${NAME}@${VER} --access public (linux host artifact)" | |
| if [[ "${NAME}" != "@basecrawl/sdk" ]]; then | |
| echo "package name must be @basecrawl/sdk, got ${NAME}" >&2 | |
| exit 1 | |
| fi | |
| set +e | |
| out="$(npm publish --access public 2>&1)" | |
| ec=$? | |
| set -e | |
| printf '%s\n' "${out}" | |
| if [[ ${ec} -ne 0 ]]; then | |
| # Typed blocker path (VAL-NPM-002): missing/unauthorized @basecrawl scope/org | |
| # after crates already green is an allowed milestone outcome when recorded. | |
| if printf '%s\n' "${out}" | grep -Eqi \ | |
| 'not authorized|ENEEDAUTH|E404|404 Not Found|//registry.npmjs.org/@basecrawl|scope|organization|org'; then | |
| { | |
| echo "TYPED_BLOCKER=npm_org_or_scope" | |
| echo "detail=npm publish failed due to missing/unauthorized @basecrawl org/scope (or related registry auth). crates.io chain already succeeded (crates job green). User must create @basecrawl org or grant NPM_TOKEN publish rights; token rotation is out of M25 scope." | |
| } | tee "${RUNNER_TEMP}/npm-typed-blocker.txt" | |
| echo "typed npm org/scope blocker recorded" | |
| exit 42 | |
| fi | |
| if printf '%s\n' "${out}" | grep -Eqi 'cannot publish over|EPUBLISHCONFLICT|previously published'; then | |
| echo "package version already on npm — treating as success" | |
| exit 0 | |
| fi | |
| echo "npm publish failed (not a typed org blocker)" >&2 | |
| exit ${ec} | |
| fi | |
| echo "npm publish OK for ${NAME}@${VER}" | |
| - name: Classify npm outcome | |
| if: needs.version-check.outputs.dry_run != 'true' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # steps.npm_publish.outcome is success | failure | skipped | |
| outcome="${{ steps.npm_publish.outcome }}" | |
| conclusion="${{ steps.npm_publish.conclusion }}" | |
| echo "npm_publish outcome=${outcome} conclusion=${conclusion}" | |
| if [[ -f "${RUNNER_TEMP}/npm-typed-blocker.txt" ]]; then | |
| echo "::warning::TYPED npm org/scope blocker — crates.io publish succeeded; npm deferred" | |
| cat "${RUNNER_TEMP}/npm-typed-blocker.txt" | |
| # Soft-pass so cross-publish topology (crates green → typed npm blocker) is green. | |
| exit 0 | |
| fi | |
| if [[ "${outcome}" == "failure" ]]; then | |
| echo "npm publish failed without typed @basecrawl org/scope classification" >&2 | |
| exit 1 | |
| fi | |
| echo "npm job complete" | |
| - name: Dry-run npm summary | |
| if: needs.version-check.outputs.dry_run == 'true' | |
| run: | | |
| echo "dry_run=true — built linux native + npm pack only; skipped npm publish" |