diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b2db4953..0db99b577 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,12 +37,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Setup GCS credentials - uses: google-github-actions/auth@v3 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - - name: Parse .tool-versions uses: wistia/parse-tool-versions@v2.1.1 @@ -55,7 +49,6 @@ jobs: id: validate env: GH_TOKEN: ${{ github.token }} - GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }} GITHUB_REPOSITORY: ${{ github.repository }} run: | python3 scripts/validate.py \ @@ -104,34 +97,6 @@ jobs: echo "Build artifacts:" find ./builds -type f | head -50 - - name: Setup Service Account - uses: google-github-actions/auth@v3 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - - - name: Upload to GCS (skip existing) - env: - GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }} - run: | - version_name="${{ needs.validate.outputs.version_name }}" - - for arch in amd64 arm64; do - local_path="./builds/$version_name/$arch/firecracker" - if [[ ! -f "$local_path" ]]; then - continue - fi - - gcs_path="gs://${GCP_BUCKET_NAME}/firecrackers/${version_name}/${arch}/firecracker" - - if gcloud storage ls "$gcs_path" >/dev/null 2>&1; then - echo "GCS: $arch artifact already exists, skipping" - else - echo "GCS: Uploading $arch artifact..." - gcloud storage cp "$local_path" "$gcs_path" - fi - done - - name: Create or update GitHub release (skip existing artifacts) env: GH_TOKEN: ${{ github.token }} @@ -197,3 +162,54 @@ jobs: done echo "Release URL: https://github.com/${{ github.repository }}/releases/tag/$version_name" + + deploy: + needs: [validate, publish] + if: needs.validate.outputs.skip_build != 'true' + strategy: + fail-fast: false + matrix: + environment: [staging, juliett, foxtrot] + runs-on: ubuntu-24.04 + environment: ${{ matrix.environment }} + steps: + - name: Setup GCS credentials + uses: google-github-actions/auth@v3 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + + - name: Download release assets + env: + GH_TOKEN: ${{ github.token }} + run: | + version_name="${{ needs.validate.outputs.version_name }}" + + for arch in amd64 arm64; do + asset_name="firecracker-${arch}" + mkdir -p ./builds/$version_name/$arch + gh release download "$version_name" \ + --repo "${{ github.repository }}" \ + --pattern "$asset_name" \ + --output "./builds/$version_name/$arch/firecracker" 2>/dev/null || true + done + + - name: Upload to GCS + env: + GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }} + run: | + version_name="${{ needs.validate.outputs.version_name }}" + + for arch in amd64 arm64; do + local_path="./builds/$version_name/$arch/firecracker" + [[ -f "$local_path" ]] || continue + + gcs_path="gs://${GCP_BUCKET_NAME}/firecrackers/${version_name}/${arch}/firecracker" + + if gcloud storage ls "$gcs_path" >/dev/null 2>&1; then + echo "GCS (${{ matrix.environment }}): $arch already exists, skipping" + else + echo "GCS (${{ matrix.environment }}): Uploading $arch..." + gcloud storage cp "$local_path" "$gcs_path" + fi + done diff --git a/scripts/test_validate.py b/scripts/test_validate.py index 3d26d270e..d396d666a 100644 --- a/scripts/test_validate.py +++ b/scripts/test_validate.py @@ -358,58 +358,51 @@ class TestCheckExistingArtifacts: """Tests for check_existing_artifacts function.""" def test_all_artifacts_exist_skip_build(self): - """Test when all artifacts exist in both GCS and release.""" + """Test when all artifacts exist in release.""" with patch("validate.check_release_artifacts", return_value={"firecracker-amd64", "firecracker-arm64"}): - with patch("validate.check_gcs_artifact", return_value=True): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, True, "my-bucket", "owner/repo" - ) - assert skip is True - assert matrix == {"include": []} + matrix, skip = check_existing_artifacts( + "v1.0.0_abc1234", True, True, "owner/repo" + ) + assert skip is True + assert matrix == {"include": []} def test_no_artifacts_exist_build_both(self): """Test when no artifacts exist.""" with patch("validate.check_release_artifacts", return_value=set()): - with patch("validate.check_gcs_artifact", return_value=False): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, True, "my-bucket", "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 2 - archs = {item["arch"] for item in matrix["include"]} - assert archs == {"amd64", "arm64"} + matrix, skip = check_existing_artifacts( + "v1.0.0_abc1234", True, True, "owner/repo" + ) + assert skip is False + assert len(matrix["include"]) == 2 + archs = {item["arch"] for item in matrix["include"]} + assert archs == {"amd64", "arm64"} def test_only_amd64_missing(self): """Test when only amd64 is missing.""" with patch("validate.check_release_artifacts", return_value={"firecracker-arm64"}): - with patch("validate.check_gcs_artifact") as mock_gcs: - mock_gcs.side_effect = lambda bucket, version, arch: arch == "arm64" - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, True, "my-bucket", "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 1 - assert matrix["include"][0]["arch"] == "amd64" + matrix, skip = check_existing_artifacts( + "v1.0.0_abc1234", True, True, "owner/repo" + ) + assert skip is False + assert len(matrix["include"]) == 1 + assert matrix["include"][0]["arch"] == "amd64" def test_only_arm64_requested_and_missing(self): """Test when only arm64 is requested and missing.""" with patch("validate.check_release_artifacts", return_value=set()): - with patch("validate.check_gcs_artifact", return_value=False): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", False, True, "my-bucket", "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 1 - assert matrix["include"][0]["arch"] == "arm64" - assert matrix["include"][0]["runner"] == "ubuntu-24.04-arm" - - def test_gcs_missing_but_release_exists(self): - """Test when GCS is missing but release exists (still needs build).""" + matrix, skip = check_existing_artifacts( + "v1.0.0_abc1234", False, True, "owner/repo" + ) + assert skip is False + assert len(matrix["include"]) == 1 + assert matrix["include"][0]["arch"] == "arm64" + assert matrix["include"][0]["runner"] == "ubuntu-24.04-arm" + + def test_amd64_exists_in_release(self): + """Test when amd64 exists in release (skip build for amd64).""" with patch("validate.check_release_artifacts", return_value={"firecracker-amd64"}): - with patch("validate.check_gcs_artifact", return_value=False): - matrix, skip = check_existing_artifacts( - "v1.0.0_abc1234", True, False, "my-bucket", "owner/repo" - ) - assert skip is False - assert len(matrix["include"]) == 1 - assert matrix["include"][0]["arch"] == "amd64" + matrix, skip = check_existing_artifacts( + "v1.0.0_abc1234", True, False, "owner/repo" + ) + assert skip is True + assert matrix == {"include": []} diff --git a/scripts/validate.py b/scripts/validate.py index fb30269ac..c85ad8747 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -208,13 +208,6 @@ def check_ci_status(commit_hash: str, repo: str = "e2b-dev/firecracker") -> tupl return True, f"Could not definitively verify CI status (status={status}, check_conclusion={check_conclusion}) - proceeding anyway" -def check_gcs_artifact(bucket: str, version_name: str, arch: str) -> bool: - """Check if an artifact exists in GCS.""" - gcs_path = f"gs://{bucket}/firecrackers/{version_name}/{arch}/firecracker" - result = run_command(["gcloud", "storage", "ls", gcs_path], check=False) - return result.returncode == 0 - - def check_release_artifacts(github_repo: str, version_name: str) -> set[str]: """Get the set of artifact names in a GitHub release.""" result = run_command([ @@ -234,7 +227,6 @@ def check_existing_artifacts( version_name: str, build_amd64: bool, build_arm64: bool, - gcp_bucket: str, github_repo: str ) -> tuple[dict, bool]: """ @@ -251,13 +243,11 @@ def check_existing_artifacts( if not requested: continue - gcs_exists = check_gcs_artifact(gcp_bucket, version_name, arch) release_exists = f"firecracker-{arch}" in release_assets - print(f"GCS: {arch} artifact {'exists' if gcs_exists else 'missing'}", file=sys.stderr) print(f"Release: {arch} artifact {'exists' if release_exists else 'missing'}", file=sys.stderr) - if not gcs_exists or not release_exists: + if not release_exists: if arch == "amd64": need_amd64 = True else: @@ -269,7 +259,7 @@ def check_existing_artifacts( print("SKIPPING BUILD: All requested artifacts already exist", file=sys.stderr) print("==============================================", file=sys.stderr) print("", file=sys.stderr) - print(f"::notice::Skipped build - all requested artifacts already exist in both GCS and GitHub release", file=sys.stderr) + print("::notice::Skipped build - all requested artifacts already exist in GitHub release", file=sys.stderr) return {"include": []}, True # Generate build matrix @@ -303,8 +293,6 @@ def main() -> int: help="Build for amd64 architecture") parser.add_argument("--build-arm64", type=lambda x: x.lower() == "true", default=True, help="Build for arm64 architecture") - parser.add_argument("--gcp-bucket", default=os.environ.get("GCP_BUCKET_NAME", ""), - help="GCP bucket name") parser.add_argument("--github-repo", default=os.environ.get("GITHUB_REPOSITORY", ""), help="GitHub repository (owner/repo)") @@ -351,7 +339,6 @@ def main() -> int: version_name, args.build_amd64, args.build_arm64, - args.gcp_bucket, args.github_repo )