fix(build_beacon.sh): add fallback logic when remote cache fails (#292)

Retry build without remote cache on first failure, then clean local
cache and retry once more to handle transient cache corruption issues.

Author: mattevans

prysm/build_beacon.sh

@@ -38,7 +38,17 @@ END
     ;;
   "bazel")
     echo "Building with Bazel..."
-    $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=release --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443
+    # Try with remote cache first
+    if ! $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=release --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443; then
+      echo "Build failed with remote cache, trying without remote cache..."
+      # Try without remote cache to avoid cache corruption issues
+      if ! $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=release --define pgo_enabled=0 --enable_bzlmod=false; then
+        echo "Build still failing, cleaning local Bazel cache and retrying..."
+        # Clean the local Bazel cache and try once more
+        $HOME/go/bin/bazelisk clean --expunge
+        $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=release --define pgo_enabled=0 --enable_bzlmod=false
+      fi
+    fi
     mv bazel-bin/cmd/beacon-chain/beacon-chain_/beacon-chain _beacon-chain
     ;;
   *)

prysm/build_beacon_minimal.sh

@@ -38,7 +38,17 @@ END
     ;;
   "bazel")
     echo "Building with Bazel..."
-    $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=minimal --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443
+    # Try with remote cache first
+    if ! $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=minimal --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443; then
+      echo "Build failed with remote cache, trying without remote cache..."
+      # Try without remote cache to avoid cache corruption issues
+      if ! $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=minimal --define pgo_enabled=0 --enable_bzlmod=false; then
+        echo "Build still failing, cleaning local Bazel cache and retrying..."
+        # Clean the local Bazel cache and try once more
+        $HOME/go/bin/bazelisk clean --expunge
+        $HOME/go/bin/bazelisk build //cmd/beacon-chain:beacon-chain --config=minimal --define pgo_enabled=0 --enable_bzlmod=false
+      fi
+    fi
     mv bazel-bin/cmd/beacon-chain/beacon-chain_/beacon-chain _beacon-chain
     ;;
   *)

prysm/build_validator.sh

@@ -38,7 +38,17 @@ END
     ;;
   "bazel")
     echo "Building with Bazel..."
-    $HOME/go/bin/bazelisk build //cmd/validator:validator --config=release --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443
+    # Try with remote cache first
+    if ! $HOME/go/bin/bazelisk build //cmd/validator:validator --config=release --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443; then
+      echo "Build failed with remote cache, trying without remote cache..."
+      # Try without remote cache to avoid cache corruption issues
+      if ! $HOME/go/bin/bazelisk build //cmd/validator:validator --config=release --define pgo_enabled=0 --enable_bzlmod=false; then
+        echo "Build still failing, cleaning local Bazel cache and retrying..."
+        # Clean the local Bazel cache and try once more
+        $HOME/go/bin/bazelisk clean --expunge
+        $HOME/go/bin/bazelisk build //cmd/validator:validator --config=release --define pgo_enabled=0 --enable_bzlmod=false
+      fi
+    fi
     mv bazel-bin/cmd/validator/validator_/validator _validator
     ;;
   *)

prysm/build_validator_minimal.sh

@@ -38,7 +38,17 @@ END
     ;;
   "bazel")
     echo "Building with Bazel..."
-    $HOME/go/bin/bazelisk build //cmd/validator:validator --config=minimal --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443
+    # Try with remote cache first
+    if ! $HOME/go/bin/bazelisk build //cmd/validator:validator --config=minimal --define pgo_enabled=0 --enable_bzlmod=false --remote_cache=grpcs://bazel-remote-cache-grpc.primary.production.platform.ethpandaops.io:443; then
+      echo "Build failed with remote cache, trying without remote cache..."
+      # Try without remote cache to avoid cache corruption issues
+      if ! $HOME/go/bin/bazelisk build //cmd/validator:validator --config=minimal --define pgo_enabled=0 --enable_bzlmod=false; then
+        echo "Build still failing, cleaning local Bazel cache and retrying..."
+        # Clean the local Bazel cache and try once more
+        $HOME/go/bin/bazelisk clean --expunge
+        $HOME/go/bin/bazelisk build //cmd/validator:validator --config=minimal --define pgo_enabled=0 --enable_bzlmod=false
+      fi
+    fi
     mv bazel-bin/cmd/validator/validator_/validator _validator
     ;;
   *)