From 553b1f839eff6ae8fca2300fef8454357a2b33c5 Mon Sep 17 00:00:00 2001 From: Joe Lombrozo Date: Fri, 17 Apr 2026 17:10:24 -0700 Subject: [PATCH 1/3] clean up --- .github/workflows/release.yml | 23 ++++++++-- scripts/test_validate.py | 82 +++++++++++++---------------------- scripts/validate.py | 71 ++++-------------------------- 3 files changed, 56 insertions(+), 120 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0db99b577..804ed9d8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,6 @@ jobs: version_name: ${{ steps.validate.outputs.version_name }} commit_hash: ${{ steps.validate.outputs.commit_hash }} build_matrix: ${{ steps.validate.outputs.build_matrix }} - skip_build: ${{ steps.validate.outputs.skip_build }} steps: - uses: actions/checkout@v6 @@ -59,7 +58,6 @@ jobs: build: needs: validate - if: needs.validate.outputs.skip_build != 'true' strategy: fail-fast: false matrix: ${{ fromJson(needs.validate.outputs.build_matrix) }} @@ -67,10 +65,29 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Check if artifact exists in release + id: check_release + env: + GH_TOKEN: ${{ github.token }} + run: | + version_name="${{ needs.validate.outputs.version_name }}" + arch="${{ matrix.arch }}" + asset_name="firecracker-${arch}" + + if gh release view "$version_name" --json assets -q ".assets[].name" 2>/dev/null | grep -q "^${asset_name}$"; then + echo "Release: $arch artifact already exists, skipping build" + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "Release: $arch artifact missing, will build" + echo "skip=false" >> $GITHUB_OUTPUT + fi + - name: Build Firecracker ${{ needs.validate.outputs.version_name }} (${{ matrix.arch }}) + if: steps.check_release.outputs.skip != 'true' run: ./build.sh "${{ needs.validate.outputs.commit_hash }}" "${{ needs.validate.outputs.version_name }}" "${{ matrix.arch }}" - name: Upload build artifact + if: steps.check_release.outputs.skip != 'true' uses: actions/upload-artifact@v7 with: name: firecracker-${{ needs.validate.outputs.version_name }}-${{ matrix.arch }} @@ -79,7 +96,6 @@ jobs: publish: needs: [validate, build] - if: needs.validate.outputs.skip_build != 'true' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 @@ -165,7 +181,6 @@ jobs: deploy: needs: [validate, publish] - if: needs.validate.outputs.skip_build != 'true' strategy: fail-fast: false matrix: diff --git a/scripts/test_validate.py b/scripts/test_validate.py index d396d666a..449f7f910 100644 --- a/scripts/test_validate.py +++ b/scripts/test_validate.py @@ -14,7 +14,7 @@ find_tag_for_commit, resolve_tag_and_commit, check_ci_status, - check_existing_artifacts, + generate_build_matrix, gh_api, ) @@ -354,55 +354,31 @@ def test_ci_skipped_checks_count_as_success(self): assert success is True -class TestCheckExistingArtifacts: - """Tests for check_existing_artifacts function.""" - - def test_all_artifacts_exist_skip_build(self): - """Test when all artifacts exist in release.""" - with patch("validate.check_release_artifacts", return_value={"firecracker-amd64", "firecracker-arm64"}): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, True, "owner/repo" - ) - assert skip is True - assert matrix == {"include": []} - - def test_no_artifacts_exist_build_both(self): - """Test when no artifacts exist.""" - with patch("validate.check_release_artifacts", return_value=set()): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, True, "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 2 - archs = {item["arch"] for item in matrix["include"]} - assert archs == {"amd64", "arm64"} - - def test_only_amd64_missing(self): - """Test when only amd64 is missing.""" - with patch("validate.check_release_artifacts", return_value={"firecracker-arm64"}): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, True, "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 1 - assert matrix["include"][0]["arch"] == "amd64" - - def test_only_arm64_requested_and_missing(self): - """Test when only arm64 is requested and missing.""" - with patch("validate.check_release_artifacts", return_value=set()): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", False, True, "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 1 - assert matrix["include"][0]["arch"] == "arm64" - assert matrix["include"][0]["runner"] == "ubuntu-24.04-arm" - - def test_amd64_exists_in_release(self): - """Test when amd64 exists in release (skip build for amd64).""" - with patch("validate.check_release_artifacts", return_value={"firecracker-amd64"}): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, False, "owner/repo" - ) - assert skip is True - assert matrix == {"include": []} +class TestGenerateBuildMatrix: + """Tests for generate_build_matrix function.""" + + def test_build_both_architectures(self): + """Test generating matrix for both architectures.""" + matrix = generate_build_matrix(True, True) + assert len(matrix["include"]) == 2 + archs = {item["arch"] for item in matrix["include"]} + assert archs == {"amd64", "arm64"} + + def test_build_amd64_only(self): + """Test generating matrix for amd64 only.""" + matrix = generate_build_matrix(True, False) + assert len(matrix["include"]) == 1 + assert matrix["include"][0]["arch"] == "amd64" + assert matrix["include"][0]["runner"] == "ubuntu-24.04" + + def test_build_arm64_only(self): + """Test generating matrix for arm64 only.""" + matrix = generate_build_matrix(False, True) + assert len(matrix["include"]) == 1 + assert matrix["include"][0]["arch"] == "arm64" + assert matrix["include"][0]["runner"] == "ubuntu-24.04-arm" + + def test_build_neither_architecture(self): + """Test generating empty matrix when no architectures requested.""" + matrix = generate_build_matrix(False, False) + assert matrix == {"include": []} diff --git a/scripts/validate.py b/scripts/validate.py index c85ad8747..c993cce90 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -208,68 +208,19 @@ def check_ci_status(commit_hash: str, repo: str = "e2b-dev/firecracker") -> tupl return True, f"Could not definitively verify CI status (status={status}, check_conclusion={check_conclusion}) - proceeding anyway" -def check_release_artifacts(github_repo: str, version_name: str) -> set[str]: - """Get the set of artifact names in a GitHub release.""" - result = run_command([ - "gh", "release", "view", version_name, - "--repo", github_repo, - "--json", "assets", - "-q", ".assets[].name" - ], check=False) - - if result.returncode != 0: - return set() - - return set(result.stdout.strip().split("\n")) if result.stdout.strip() else set() - - -def check_existing_artifacts( - version_name: str, - build_amd64: bool, - build_arm64: bool, - github_repo: str -) -> tuple[dict, bool]: +def generate_build_matrix(build_amd64: bool, build_arm64: bool) -> dict: """ - Check existing artifacts and generate build matrix. + Generate build matrix for all requested architectures. - Returns (build_matrix, skip_build). + Build and deploy jobs always run; individual steps check for existing artifacts. """ - need_amd64 = False - need_arm64 = False - - release_assets = check_release_artifacts(github_repo, version_name) - - for arch, requested in [("amd64", build_amd64), ("arm64", build_arm64)]: - if not requested: - continue - - release_exists = f"firecracker-{arch}" in release_assets - - print(f"Release: {arch} artifact {'exists' if release_exists else 'missing'}", file=sys.stderr) - - if not release_exists: - if arch == "amd64": - need_amd64 = True - else: - need_arm64 = True - - if not need_amd64 and not need_arm64: - print("", file=sys.stderr) - print("==============================================", file=sys.stderr) - print("SKIPPING BUILD: All requested artifacts already exist", file=sys.stderr) - print("==============================================", file=sys.stderr) - print("", file=sys.stderr) - print("::notice::Skipped build - all requested artifacts already exist in GitHub release", file=sys.stderr) - return {"include": []}, True - - # Generate build matrix include = [] - if need_amd64: + if build_amd64: include.append({"arch": "amd64", "runner": "ubuntu-24.04"}) - if need_arm64: + if build_arm64: include.append({"arch": "arm64", "runner": "ubuntu-24.04-arm"}) - return {"include": include}, False + return {"include": include} def write_github_output(outputs: dict[str, str]) -> None: @@ -334,13 +285,8 @@ def main() -> int: return 1 print(ci_message, file=sys.stderr) - # Step 4: Check existing artifacts and generate build matrix - build_matrix, skip_build = check_existing_artifacts( - version_name, - args.build_amd64, - args.build_arm64, - args.github_repo - ) + # Step 4: Generate build matrix for all requested architectures + build_matrix = generate_build_matrix(args.build_amd64, args.build_arm64) print(f"Build matrix: {json.dumps(build_matrix)}", file=sys.stderr) @@ -349,7 +295,6 @@ def main() -> int: "commit_hash": commit_hash, "version_name": version_name, "build_matrix": json.dumps(build_matrix), - "skip_build": "true" if skip_build else "false" }) return 0 From 3da9fc54500c6bf5bbbfc32e2a8e808dc86031cd Mon Sep 17 00:00:00 2001 From: Joe Lombrozo Date: Fri, 17 Apr 2026 17:22:48 -0700 Subject: [PATCH 2/3] use vars, not secrets --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 804ed9d8c..7fea01c24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -191,8 +191,8 @@ jobs: - name: Setup GCS credentials uses: google-github-actions/auth@v3 with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + project_id: ${{ vars.GCP_PROJECT_ID }} + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - name: Download release assets env: From 2a638dfac28afef8f0455a61c78f660351b7d6d5 Mon Sep 17 00:00:00 2001 From: Joe Lombrozo Date: Fri, 17 Apr 2026 23:40:52 -0700 Subject: [PATCH 3/3] set service_account as well as the rest --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7fea01c24..50fe60761 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -193,6 +193,7 @@ jobs: with: project_id: ${{ vars.GCP_PROJECT_ID }} workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} - name: Download release assets env: