Fix kernel builds; Migrate ci runners (#291)

## Description


The `debian:stable` container image, combined with the packages we install for building the kernel, doesn't include Make anymore, so we explicitly add it. Also, move to CNCF GitHub Action runners as the self-hosted runners in Equinix Metal are going away with the Equinix Metal sunset.

## Why is this needed



Fixes: #

## How Has This Been Tested?





## How are existing users impacted? What migration steps/scripts do we need?





## Checklist:

I have:

- [ ] updated the documentation and/or roadmap (if required)
- [ ] added unit or e2e tests
- [ ] provided instructions on how to upgrade

Author: mergify[bot]

.github/workflows/build-all-matrix.yaml

@@ -25,10 +25,10 @@ env: # Global environment, passed to all jobs & all steps
   CI_TAGS: "standard armbian-sbc armbian-uefi lts" # 'dev' is not included
 
   # GHA runner configuration. See bash/json-matrix.sh for more details.
-  CI_RUNNER_LK_CONTAINERS_ARM64: "oracle-24cpu-384gb-arm64" # Use a self-hosted runner with the "ARM64" tag for the ARM64 builds of LK containers
-  CI_RUNNER_LK_CONTAINERS_AMD64: "oracle-24cpu-384gb-x86-64" # Use a self-hosted runner with the "X86" tag for the AMD64 builds of LK containers
-  CI_RUNNER_LK_ARM64: "oracle-24cpu-384gb-arm64" # Use a self-hosted runner with the "ARM64" tag for the ARM64 linuxkit builds
-  CI_RUNNER_LK_AMD64: "oracle-24cpu-384gb-x86-64" # Use a self-hosted runner with the "X86" tag for the AMD64 linuxkit builds
+  CI_RUNNER_LK_CONTAINERS_ARM64: "oracle-vm-32cpu-128gb-arm64" # Use a self-hosted runner with the "ARM64" tag for the ARM64 builds of LK containers
+  CI_RUNNER_LK_CONTAINERS_AMD64: "oracle-vm-32cpu-128gb-x86-64" # Use a self-hosted runner with the "X86" tag for the AMD64 builds of LK containers
+  CI_RUNNER_LK_ARM64: "oracle-vm-32cpu-128gb-arm64" # Use a self-hosted runner with the "ARM64" tag for the ARM64 linuxkit builds
+  CI_RUNNER_LK_AMD64: "oracle-vm-32cpu-128gb-x86-64" # Use a self-hosted runner with the "X86" tag for the AMD64 linuxkit builds
   CI_RUNNER_KERNEL_AMD64: "oracle-24cpu-384gb-x86-64" # Use a self-hosted runner with the "X86" tag for the AMD64 kernel builds
   CI_RUNNER_KERNEL_ARM64: "oracle-24cpu-384gb-arm64" # Use a self-hosted runner with the "ARM64" tag for the ARM64 kernel builds
 
@@ -45,7 +45,7 @@ jobs:
       lk_hooks_json: ${{ steps.prepare-matrix.outputs.lk_hooks_json }}
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Prepare release ID (current date) # This only used for the GitHub Release; not included in any way in the build process.
         id: date_prep
@@ -60,21 +60,27 @@ jobs:
 
   build-linuxkit-containers:
     needs: [ matrix_prep ]
-    runs-on: "${{ matrix.runner }}" # the runner to use is determined by the 'gha-matrix' code
+    runs-on:
+      group: Default
+      labels: "${{ matrix.runner }}" # the runner to use is determined by the 'gha-matrix' code
     strategy:
       fail-fast: true
       matrix:
         include: ${{ fromJSON(needs.matrix_prep.outputs.lkcontainers_json) }}
     name: "LinuxKit containers for ${{ matrix.docker_arch }}"
     steps:
       - name: Checkout build repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["mirror.gcr.io"]
 
       - name: Docker Login to quay.io
         if: ${{ env.REGISTRY == 'quay.io' && github.ref == 'refs/heads/main' }}
@@ -86,26 +92,41 @@ jobs:
         uses: docker/login-action@v3
         with: { registry: "ghcr.io", username: "${{ github.repository_owner }}", password: "${{ secrets.GITHUB_TOKEN }}" }
 
-      - name: Build and Push LinuxKit containers for ${{matrix.docker_arch}}
+      - name: Build and Push and Export LinuxKit containers for ${{matrix.docker_arch}}
         env:
           DOCKER_ARCH: "${{ matrix.docker_arch }}"
           DO_PUSH: "${{ github.ref == 'refs/heads/main' && 'yes' || 'no' }}"
+          EXPORT_LK_CONTAINERS: "${{ github.event_name == 'pull_request' && 'yes' || 'no' }}" # Builds on PRs don't push images to a registry so they need to be passed on through GitHub Artifacts.
+          EXPORT_LK_CONTAINERS_DIR: "${{ runner.temp }}"
         run: bash build.sh linuxkit-containers
-  
+
+      - name: Upload Linuxkit Docker images as GitHub Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: linuxkit-images-${{ matrix.docker_arch }}
+          path: ${{ runner.temp }}/*-${{ matrix.docker_arch }}.tar.gz
+          retention-days: 1
+
   build-kernels:
     needs: [ matrix_prep ] # depend on the previous job...
-    runs-on: "${{ matrix.runner }}" # the runner to use is determined by the 'gha-matrix' code
+    runs-on:
+      group: Default
+      labels: "${{ matrix.runner }}" # the runner to use is determined by the 'gha-matrix' code
     strategy:
       fail-fast: false # let other jobs try to complete if one fails, kernels might take long, and they'd be skipped on the next run
       matrix:
         include: ${{ fromJSON(needs.matrix_prep.outputs.kernels_json) }}
     name: "Kernel ${{ matrix.kernel }}"
     steps:
       - name: Checkout build repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Docker Buildx # nb: no need for qemu here, kernels are cross-compiled, instead of the compilation being emulated
         uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["mirror.gcr.io"]
 
       - name: Docker Login to quay.io
         if: ${{ env.REGISTRY == 'quay.io' && github.ref == 'refs/heads/main' }}
@@ -117,25 +138,40 @@ jobs:
         uses: docker/login-action@v3
         with: { registry: "ghcr.io", username: "${{ github.repository_owner }}", password: "${{ secrets.GITHUB_TOKEN }}" }
 
-      - name: Build and push Kernel ${{matrix.kernel}} (${{ matrix.arch }})
+      - name: Build and Push and Export Kernel ${{matrix.kernel}} (${{ matrix.arch }})
         env:
           DO_PUSH: "${{ github.ref == 'refs/heads/main' && 'yes' || 'no' }}"
+          EXPORT_KERNEL_IMAGE: "${{ github.event_name == 'pull_request' && 'yes' || 'no' }}" # Builds on PRs don't push images to a registry so they need to be passed on through GitHub Artifacts.
+          EXPORT_KERNEL_IMAGE_DIR: "${{ runner.temp }}"
         run: bash build.sh build-kernel "${{ matrix.kernel }}"
 
+      - name: Upload Kernel Docker images as GitHub Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: kernel-images-${{ matrix.kernel }}
+          path: ${{ runner.temp }}/hook-kernel-*.tar.gz
+          retention-days: 1
+
   build-hook-ensemble:
     needs: [ matrix_prep, build-linuxkit-containers, build-kernels ] # depend on the previous job...
-    runs-on: "${{ matrix.runner }}" # the runner to use is determined by the 'gha-matrix' code
+    runs-on:
+      group: Default
+      labels: "${{ matrix.runner }}" # the runner to use is determined by the 'gha-matrix' code
     strategy:
       fail-fast: false # let other jobs try to complete if one fails
       matrix:
         include: ${{ fromJSON(needs.matrix_prep.outputs.lk_hooks_json) }}
     name: "Hook ${{ matrix.kernel }}"
     steps:
       - name: Checkout build repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Docker Buildx # nb: no need for qemu here, kernels are cross-compiled, instead of the compilation being emulated
         uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["mirror.gcr.io"]
 
       - name: Docker Login to DockerHub # read-only token, required to be able to pull all the linuxkit pkgs without getting rate limited.
         if: ${{ env.LOGIN_TO_DOCKERHUB == 'yes' && github.ref == 'refs/heads/main' }}
@@ -163,6 +199,42 @@ jobs:
             lk-cache-${{ matrix.docker_arch }}
           save-always: true # always save the cache, even if build fails
 
+      - name: Download Linuxkit artifacts
+        uses: actions/download-artifact@v5
+        with:
+          name: linuxkit-images-${{ matrix.docker_arch }}
+          path: ${{ runner.temp }}
+
+      - name: Load Linuxkit Docker images into local Docker daemon
+        run: |
+          ls "${{ runner.temp }}"
+          imgs=$(ls "${{ runner.temp }}" | grep tar.gz | xargs)
+          echo "Found hook images: ${imgs}"
+          for img in ${imgs}; do
+            echo "extracting and loading image: ${{ runner.temp }}/${img}"
+            gunzip -d "${{ runner.temp }}/${img}"
+            docker load --input "${{ runner.temp }}/${img%.*}"
+          done
+          docker images
+
+      - name: Download Kernel artifacts
+        uses: actions/download-artifact@v5
+        with:
+          name: kernel-images-${{ matrix.kernel }}
+          path: ${{ runner.temp }}
+
+      - name: Load Kernel Docker images into local Docker daemon
+        run: |
+          ls "${{ runner.temp }}"
+          imgs=$(ls "${{ runner.temp }}" | grep tar.gz | xargs)
+          echo "Found kernel images: ${{ runner.temp }}/${imgs}"
+          for img in ${imgs}; do
+            echo "extracting and loading image: ${{ runner.temp }}/${img}"
+            gunzip -d "${{ runner.temp }}/${img}"
+            docker load --input "${{ runner.temp }}/${img%.*}"
+          done
+          docker images
+
       - name: "Build Hook with Kernel ${{matrix.kernel}} (${{ matrix.arch }}) - cache: ${{matrix.gha_cache}}"
         env:
           DO_BUILD_LK_CONTAINERS: "no" # already built them; this is only for hook/linuxkit.
@@ -181,6 +253,7 @@ jobs:
           path: |
             out/*.tar.gz
             out/*.iso
+          retention-days: 1
 
   release-latest:
     name: Publish all Hooks to GitHub Releases
@@ -190,10 +263,10 @@ jobs:
     steps:
 
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Download built Hook artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           pattern: "hook-tarball-*"
           merge-multiple: true
@@ -262,10 +335,10 @@ jobs:
     steps:
 
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Download built Hook artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           pattern: "hook-tarball-*"
           merge-multiple: true

bash/hook-lk-containers.sh

@@ -7,19 +7,21 @@ function build_all_hook_linuxkit_containers() {
 	# when adding new container builds here you'll also want to add them to the
 	# `linuxkit_build` function in the linuxkit.sh file.
 	# # NOTE: linuxkit containers must be in the images/ directory
-	build_hook_linuxkit_container hook-bootkit "HOOK_CONTAINER_BOOTKIT_IMAGE"
-	build_hook_linuxkit_container hook-docker "HOOK_CONTAINER_DOCKER_IMAGE"
-	build_hook_linuxkit_container hook-udev "HOOK_CONTAINER_UDEV_IMAGE"
-	build_hook_linuxkit_container hook-acpid "HOOK_CONTAINER_ACPID_IMAGE"
-	build_hook_linuxkit_container hook-containerd "HOOK_CONTAINER_CONTAINERD_IMAGE"
-	build_hook_linuxkit_container hook-runc "HOOK_CONTAINER_RUNC_IMAGE"
-	build_hook_linuxkit_container hook-embedded "HOOK_CONTAINER_EMBEDDED_IMAGE"
+	build_hook_linuxkit_container hook-bootkit "HOOK_CONTAINER_BOOTKIT_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
+	build_hook_linuxkit_container hook-docker "HOOK_CONTAINER_DOCKER_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
+	build_hook_linuxkit_container hook-udev "HOOK_CONTAINER_UDEV_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
+	build_hook_linuxkit_container hook-acpid "HOOK_CONTAINER_ACPID_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
+	build_hook_linuxkit_container hook-containerd "HOOK_CONTAINER_CONTAINERD_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
+	build_hook_linuxkit_container hook-runc "HOOK_CONTAINER_RUNC_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
+	build_hook_linuxkit_container hook-embedded "HOOK_CONTAINER_EMBEDDED_IMAGE" "${EXPORT_LK_CONTAINERS}" "${EXPORT_LK_CONTAINERS_DIR}"
 }
 
 function build_hook_linuxkit_container() {
 	declare container_dir="${1}"
 	declare template_var="${2}" # bash name reference, kind of an output var but weird
 	declare container_base_dir="images"
+	declare export_container_images="${3:-false}"
+	declare export_container_images_dir="${4:-/tmp}"
 
 	# Lets hash the contents of the directory and use that as a tag
 	declare container_files_hash
@@ -38,13 +40,24 @@ function build_hook_linuxkit_container() {
 		# we try to push here because a previous build may have created the image
 		# this is the case for GitHub Actions CI because we build PRs on the same self-hosted runner
 		push_hook_linuxkit_container "${container_oci_ref}"
+
+		# If export_container_images=yes then export images as tar.gzs to export_container_images_dir
+		# This is mainly for CI to be able to pass built images between jobs
+		if [[ "${export_container_images}" == "yes" ]]; then
+			save_docker_image_to_tar_gz "${container_oci_ref}" "${export_container_images_dir}"
+		fi
 		return 0
 	fi
 
 	# Check if we can pull the image from registry; if so, skip the build.
 	log debug "Checking if image ${container_oci_ref} can be pulled from remote registry"
 	if docker pull "${container_oci_ref}"; then
 		log info "Image ${container_oci_ref} pulled from remote registry, skipping build"
+		# If export_container_images=yes then export images as tar.gzs to export_container_images_dir
+		# This is mainly for CI to be able to pass built images between jobs
+		if [[ "${export_container_images}" == "yes" ]]; then
+			save_docker_image_to_tar_gz "${container_oci_ref}" "${export_container_images_dir}"
+		fi
 		return 0
 	fi
 
@@ -64,9 +77,27 @@ function build_hook_linuxkit_container() {
 
 	push_hook_linuxkit_container "${container_oci_ref}"
 
+	# If export_container_images=yes then export images as tar.gzs to export_container_images_dir
+	# This is mainly for CI to be able to pass built images between jobs
+	if [[ "${export_container_images}" == "yes" ]]; then
+		save_docker_image_to_tar_gz "${container_oci_ref}" "${export_container_images_dir}"
+	fi
+
 	return 0
 }
 
+function save_docker_image_to_tar_gz() {
+	declare container_oci_ref="${1}"
+	declare export_dir="${2:-/tmp}"
+
+	# Create the export directory if it doesn't exist
+	mkdir -p "${export_dir}"
+
+	# Save the Docker image as a tar.gz file
+	docker save "${container_oci_ref}" | gzip > "${export_dir}/$(basename "${container_oci_ref}" | sed 's/:/-/g').tar.gz"
+	log info "Saved Docker image ${container_oci_ref} to ${export_dir}/$(basename "${container_oci_ref}" | sed 's/:/-/g').tar.gz"
+}
+
 function push_hook_linuxkit_container() {
 	declare container_oci_ref="${1}"
 

bash/json-matrix.sh

@@ -210,9 +210,6 @@ function json_matrix_find_runner() {
 	declare -a json_items_bare=(${runner})
 	# wrap each json_items array item in double quotes
 	declare -a json_items=()
-	if [[ "${runner}" != "ubuntu-latest" ]]; then # if not using a GH-hosted runner, auto-add the "self-hosted" member
-		json_items+=("\"self-hosted\"")
-	fi
 	for item in "${json_items_bare[@]}"; do
 		json_items+=("\"${item}\"")
 	done

bash/kernel.sh

@@ -48,6 +48,11 @@ function kernel_build() {
 	else
 		log info "DO_PUSH not 'yes', not pushing."
 	fi
+
+	if [[ "${EXPORT_KERNEL_IMAGE}" == "yes" ]]; then
+		log info "Exporting kernel image ${kernel_oci_image} to ${EXPORT_KERNEL_IMAGE_DIR}"
+		save_docker_image_to_tar_gz "${kernel_oci_image}" "${EXPORT_KERNEL_IMAGE_DIR}"
+	fi
 }
 
 function kernel_configure_interactive() {

kernel/Dockerfile

@@ -3,7 +3,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # crossbuild-essentials are pretty heavy; here we install for both architecures to maximize Docker layer hit cache rate during development, but only one will be used
 RUN set -x && apt -o "Dpkg::Use-Pty=0" -y update && \
-      apt -o "Dpkg::Use-Pty=0" -y install curl xz-utils gnupg2 flex bison libssl-dev libelf-dev bc libncurses-dev kmod \
+      apt -o "Dpkg::Use-Pty=0" -y install curl xz-utils gnupg2 flex bison libssl-dev libelf-dev bc libncurses-dev kmod make \
                      crossbuild-essential-amd64 crossbuild-essential-arm64 && \
       apt -o "Dpkg::Use-Pty=0" -y clean