Use P and Z GHA runners (#2215) (#2253)

* Use P and Z GHA runners (#2215)

* Use go cross-compilation to speed up test builds on CI (#2210)

After adding Arm runners on GHA, we made it so the Arm test image is
always built. Unfortunately, because we use QEMU to build the images,
the build has gotten quite slow.

In order to speed up the build, this change makes it so we cross-compile
the binaries for all our supported platforms locally and the image build
simply copies the binaries into the final image.

Author: Molter73

.github/workflows/collector-builder.yml

@@ -8,11 +8,6 @@ on:
         required: true
         description: |
           The tag used to build the collector image
-      architectures:
-        type: string
-        required: true
-        description: |
-          Space-separated list of architectures to build
 
     outputs:
       collector-builder-tag:
@@ -31,7 +26,6 @@ jobs:
     outputs:
       build-image: ${{ steps.builder-tag.outputs.build-image || false }}
       collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}}
-      local-exclude: ${{ steps.arch.outputs.local-exclude || '[]'}}
 
     env:
       DEFAULT_BUILDER_TAG: master
@@ -70,24 +64,23 @@ jobs:
           echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
           echo "build-image=true" >> "$GITHUB_OUTPUT"
 
-      - name: Check arches for local build
-        if: ${{ ! contains(inputs.architectures, 'ppc64le') }}
-        id: arch
-        run: echo 'local-exclude=[{"arch":"ppc64le"}]' >> "$GITHUB_OUTPUT"
-
   build-builder-image:
-    name: Local builder image
-    # Multiarch builds sometimes take for eeeeeeeeeever
-    timeout-minutes: 480
+    name: Build builder image
     needs:
     - builder-needs-rebuilding
     if: |
       needs.builder-needs-rebuilding.outputs.build-image == 'true'
     strategy:
       matrix:
-        arch: [amd64, arm64, ppc64le]
-        exclude: ${{ fromJSON(needs.builder-needs-rebuilding.outputs.local-exclude) }}
-    runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }}
+        arch:
+        - amd64
+        - arm64
+        - ppc64le
+        - s390x
+    runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') ||
+                 (matrix.arch == 'ppc64le' && 'ubuntu-24.04-ppc64le') ||
+                 (matrix.arch == 's390x' && 'ubuntu-24.04-s390x') ||
+                'ubuntu-24.04' }}
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
@@ -99,14 +92,14 @@ jobs:
         with:
           submodules: true
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          image: tonistiigi/binfmt:qemu-v8.1.5
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Install ansible on P&Z runners
+        if: matrix.arch == 'ppc64le' || matrix.arch == 's390x'
+        run: |
+          sudo apt-get install -y ansible
+
       - name: Create ansible vars
         run: |
           cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
@@ -120,7 +113,6 @@ jobs:
           EOF
 
       - name: Build images
-        timeout-minutes: 480
         run: |
           ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
@@ -131,98 +123,18 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
 
-  build-builder-image-remote-vm:
-    name: Remote builder image
-    # Multiarch builds sometimes take for eeeeeeeeeever
-    timeout-minutes: 480
-    needs:
-    - builder-needs-rebuilding
-    if: |
-      needs.builder-needs-rebuilding.outputs.build-image == 'true' &&
-      contains(inputs.architectures, 's390x')
-    strategy:
-      matrix:
-        arch: [s390x]
-    runs-on: ubuntu-24.04
-
-    env:
-      PLATFORM: linux/${{ matrix.arch }}
-      BUILD_TYPE: ci
-      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - uses: 'google-github-actions/auth@v2'
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
-
-      - uses: 'google-github-actions/setup-gcloud@v2'
-
-      - uses: ./.github/actions/setup-vm-creds
-        with:
-          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
-          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
-          s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
-          ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
-          ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }}
-          s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }}
-          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
-          redhat-username: ${{ secrets.REDHAT_USERNAME }}
-          redhat-password: ${{ secrets.REDHAT_PASSWORD }}
-          vm-type: all
-          job-tag: builder
-
-      - name: Create Build VMs
-        run: |
-          make -C "${{ github.workspace }}/ansible" create-build-vms
-
-      - name: Create ansible vars
-        run: |
-          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
-          ---
-          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
-          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
-          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
-          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
-          collector_git_ref: ${{ github.ref }}
-          collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
-          EOF
-
-      - name: Build images
-        timeout-minutes: 480
-        run: |
-          ansible-playbook \
-            -i ansible/ci \
-            -e build_hosts='job_id_${{ env.JOB_ID }}' \
-            -e arch='${{ matrix.arch }}' \
-            -e @'${{ github.workspace }}/ansible/secrets.yml' \
-            ansible/ci-build-builder.yml
-
-      - name: Destroy VMs
-        if: always()
-        run: |
-          make -C ansible destroy-vms
-
   create-multiarch-manifest:
     needs:
     - builder-needs-rebuilding
     - build-builder-image
-    - build-builder-image-remote-vm
     name: Create Multiarch manifest
     runs-on: ubuntu-24.04
     if: |
       always() && !contains(join(needs.*.result, ','), 'failure') &&
       needs.builder-needs-rebuilding.outputs.build-image == 'true'
     env:
       COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
-      ARCHS: ${{ inputs.architectures }}
+      ARCHS: amd64 arm64 ppc64le s390x
 
     steps:
       - uses: actions/checkout@v4
@@ -258,7 +170,6 @@ jobs:
     if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request'
     needs:
       - build-builder-image
-      - build-builder-image-remote-vm
       - create-multiarch-manifest
     steps:
       - name: Slack notification

.github/workflows/collector.yml

@@ -18,11 +18,6 @@ on:
         required: true
         description: |
           The builder tag to use in the build
-      architectures:
-        type: string
-        required: true
-        description: |
-          Space-separated list of architectures to build
 
 env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
@@ -33,26 +28,19 @@ env:
   ADDRESS_SANITIZER: ${{ contains(github.event.pull_request.labels.*.name, 'address-sanitizer') }}
 
 jobs:
-  prepare-build-collector:
-    name: Prepare builders for collector
-    runs-on: ubuntu-24.04
-    outputs:
-      local-exclude: ${{ steps.arch.outputs.local-exclude || '[]' }}
-
-    steps:
-      - name: Check arches for local build
-        if: ${{ ! contains(inputs.architectures, 'ppc64le') }}
-        id: arch
-        run: echo 'local-exclude=[{"arch":"ppc64le"}]' >> "$GITHUB_OUTPUT"
-
   build-collector-image:
-    name: Local collector image ${{ matrix.arch }}
-    needs: prepare-build-collector
+    name: Build collector image ${{ matrix.arch }}
     strategy:
       matrix:
-        arch: [amd64, arm64, ppc64le]
-        exclude: ${{ fromJSON(needs.prepare-build-collector.outputs.local-exclude) }}
-    runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }}
+        arch:
+        - amd64
+        - arm64
+        - ppc64le
+        - s390x
+    runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') ||
+                 (matrix.arch == 'ppc64le' && 'ubuntu-24.04-ppc64le') ||
+                 (matrix.arch == 's390x' && 'ubuntu-24.04-s390x') ||
+                'ubuntu-24.04' }}
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
@@ -63,14 +51,14 @@ jobs:
         with:
           submodules: true
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          image: tonistiigi/binfmt:qemu-v8.1.5
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Install ansible on P&Z runners
+        if: matrix.arch == 'ppc64le' || matrix.arch == 's390x'
+        run: |
+          sudo apt-get install -y ansible
+
       - name: Create ansible vars
         run: |
           cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
@@ -91,7 +79,6 @@ jobs:
           EOF
 
       - name: Build ${{ matrix.arch }} image locally
-        timeout-minutes: 480
         run: |
           ansible-playbook \
             --connection local \
@@ -101,90 +88,14 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 
-  build-collector-image-remote-vm:
-    name: Remote collector image
-    runs-on: ubuntu-24.04
-    if: contains(inputs.architectures, 's390x')
-    strategy:
-      matrix:
-        arch: [s390x]
-
-    env:
-      PLATFORM: linux/${{ matrix.arch }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - uses: 'google-github-actions/auth@v2'
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
-
-      - uses: 'google-github-actions/setup-gcloud@v2'
-
-      - uses: ./.github/actions/setup-vm-creds
-        with:
-          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
-          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
-          s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
-          ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
-          ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }}
-          s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }}
-          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
-          redhat-username: ${{ secrets.REDHAT_USERNAME }}
-          redhat-password: ${{ secrets.REDHAT_PASSWORD }}
-          vm-type: rhel-${{ matrix.arch }}
-          job-tag: builder
-
-      - name: Create Build VMs
-        run: |
-          make -C "${{ github.workspace }}/ansible" create-build-vms
-
-      - name: Create ansible vars
-        run: |
-          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
-          ---
-          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
-          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
-          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
-          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
-          collector_git_ref: ${{ github.ref }}
-          collector_git_sha: ${{ github.sha }}
-          collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
-          disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }}
-          rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }}
-          collector_image: ${{ inputs.collector-image }}
-          collector_tag: ${{ inputs.collector-tag }}
-          debug_mode: ${{ github.event_name == 'pull_request' }}
-          EOF
-
-      - name: Build ${{ matrix.arch }} image
-        timeout-minutes: 480
-        run: |
-          ansible-playbook \
-            -i ansible/ci \
-            -e arch='${{ matrix.arch }}' \
-            -e build_hosts='job_id_${{ env.JOB_ID }}' \
-            -e @'${{ github.workspace }}/ansible/secrets.yml' \
-            ansible/ci-build-collector.yml
-
-      - name: Destroy Build VMs
-        if: always()
-        run: |
-          make -C ansible destroy-vms
-
   create-multiarch-manifest:
     needs:
     - build-collector-image
-    - build-collector-image-remote-vm
     name: Create Multiarch manifest
     runs-on: ubuntu-24.04
     if: always() && !contains(join(needs.*.result, ','), 'failure')
     env:
-      ARCHS: ${{ inputs.architectures }}
+      ARCHS: amd64 arm64 ppc64le s390x
 
     steps:
       - uses: actions/checkout@v4
@@ -220,7 +131,6 @@ jobs:
     if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request'
     needs:
       - build-collector-image
-      - build-collector-image-remote-vm
       - create-multiarch-manifest
     steps:
       - name: Slack notification

.github/workflows/init.yml

@@ -58,13 +58,6 @@ on:
         description: |
           Trigger rebuild of QA containers
         value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }}
-      architectures:
-        description: |
-          Space-separated list of architectures to build
-        value: |
-          ${{ ((github.event_name != 'pull_request' ||
-            contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) &&
-            'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }}
 
 jobs:
   common-variables: