feat: docker buildx bake action

.github/workflows/build-push.yml

@@ -45,7 +45,7 @@ env:
   GH_PUSH: 1
 
   ## Number of jobs to start during container builds
-  JOBS: 4
+  jobs: 4
 
   ## Internal tag used for the CI
   INTERNAL_TAG: pipeline-${{ github.run_id }}
@@ -93,20 +93,14 @@ jobs:
           registry: ${{ env.GH_REGISTRY }}
           username: ${{ secrets.GHCR_REGISTRY_USER }}
           password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v5
+      - name: Buildx Bake
+        uses: docker/bake-action@v5
         with:
-          file: containers/debian/base.Dockerfile
-          context: containers/debian
-          platforms: ${{ matrix.PLATFORM }}
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            BASE_IMAGE=${{ matrix.BASE_IMAGE }}
-            BUILD_IMAGE=${{ matrix.BUILD_IMAGE }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
+          provenance: false
+          files: |
+            containers/debian/base.hcl
+            ${{ steps.meta.outputs.bake-file }}
 
   dev:
     runs-on: ubuntu-latest
@@ -194,106 +188,11 @@ jobs:
           registry: ${{ env.GH_REGISTRY }}
           username: ${{ secrets.GHCR_REGISTRY_USER }}
           password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v5
+      - name: Buildx Bake
+        uses: docker/bake-action@v5
         with:
-          file: containers/jug/dev.Dockerfile
-          context: containers/jug
-          build-contexts: |
-            spack-environment=spack-environment
-          secret-files: |
-            mirrors=mirrors.yaml
-          platforms: ${{ matrix.PLATFORM }}
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/
-            BUILDER_IMAGE=${{ matrix.BUILDER_IMAGE }}
-            RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }}
-            BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }}
-            INTERNAL_TAG=${{ env.INTERNAL_TAG }}
-            SPACK_ORGREPO=${{ steps.spack.outputs.orgrepo }}
-            SPACK_VERSION=${{ steps.spack.outputs.version }}
-            SPACK_CHERRYPICKS=${{ steps.spack.outputs.cherrypicks }}
-            SPACK_CHERRYPICKS_FILES=${{ steps.spack.outputs.cherrypicks_files }}
-            KEY4HEPSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }}
-            KEY4HEPSPACK_VERSION=${{ steps.eic-spack.outputs.version }}
-            EICSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }}
-            EICSPACK_VERSION=${{ steps.eic-spack.outputs.version }}
-            KEY4HEPSPACK_ORGREPO=${{ steps.key4hep-spack.outputs.orgrepo }}
-            KEY4HEPSPACK_VERSION=${{ steps.key4hep-spack.outputs.version }}
-            S3_ACCESS_KEY=${{ secrets.S3_ACCESS_KEY }}
-            S3_SECRET_KEY=${{ secrets.S3_SECRET_KEY }}
-            jobs=${{ env.JOBS }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
-
-  xl:
-    runs-on: ubuntu-latest
-    needs: dev
-    strategy:
-      matrix:
-        include:
-        - BASE_IMAGE: dev
-          BUILD_IMAGE: xl
-          PLATFORM: linux/amd64
-    steps:
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/[email protected]
-        with:
-          android: true
-          dotnet: true
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: linux/amd64
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.DH_PUSH != 0 }}
-            name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.GH_PUSH != 0 }}
-          tags: |
-            ${{ env.INTERNAL_TAG }}
-            type=schedule,pattern={{date 'YYYY-MM-DD'}}
-            type=ref,prefix=unstable-pr-,event=pr
-            type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: ${{ env.DH_PUSH == '1' }}
-        with:
-          registry: ${{ env.DH_REGISTRY }}
-          username: ${{ env.DH_REGISTRY_USER }}
-          password: ${{ secrets.DH_EICWEB_TOKEN }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        if: ${{ env.GH_PUSH == '1' }}
-        with:
-          registry: ${{ env.GH_REGISTRY }}
-          username: ${{ secrets.GHCR_REGISTRY_USER }}
-          password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          file: containers/jug/xl.Dockerfile
-          context: containers/jug
-          build-contexts: |
-            detectors=.
-          platforms: ${{ matrix.PLATFORM }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/
-            BASE_IMAGE=eic_${{ matrix.BASE_IMAGE }}
-            BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }}
-            INTERNAL_TAG=${{ env.INTERNAL_TAG }}
-            jobs=${{ env.JOBS }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
+          provenance: false
+          files: |
+            containers/jug/dev.hcl
+            ${{ steps.meta.outputs.bake-file }}

.gitlab-ci.yml

@@ -270,35 +270,24 @@ base:
   needs:
     - version
   script:
-    - source spack.sh ; 
-      source key4hep-spack.sh ;
-      source eic-spack.sh ;
     - attempts=0
     - nocache=""
+    - docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --print
+        --file spack.hcl
+        --file eic-spack.hcl
+        --file key4hep-spack.hcl
+        --file containers/variables.hcl
+        --file containers/debian/base.hcl
     - while !
-      docker buildx build --push ${BUILD_OPTIONS} ${nocache}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
-                   ${EXPORT_TAG:+
-                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
-                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
-                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
-                   }
-                   --file containers/debian/base.Dockerfile
-                   --platform ${PLATFORM}
-                   --build-arg BASE_IMAGE=${BASE_IMAGE}
-                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
-                   --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
-                   --build-arg SPACK_VERSION=${SPACK_VERSION}
-                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
-                   --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
-                   --build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}
-                   --build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}
-                   --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
-                   --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
-                   --build-arg jobs=${JOBS}
-                   --provenance false
-                   containers/debian
-                   2>&1 | tee build.log
+      docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --push
+        --file spack.hcl
+        --file eic-spack.hcl
+        --file key4hep-spack.hcl
+        --file containers/variables.hcl
+        --file containers/debian/base.hcl
+        2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
@@ -365,92 +354,20 @@ eic:
       if [ "$CI_COMMIT_BRANCH" == "master" ]; then
         PUSH_NIGHTLY_WITH_DATE="1"
       fi
-    - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY= ; fi
-    - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT= ; fi
-    - apk add envsubst git
-    - source spack.sh ; 
-      source key4hep-spack.sh ;
-      source eic-spack.sh ;
-      export SPACK_VERSION ;
-      cat mirrors.yaml.in | envsubst > mirrors.yaml
+    - apk add envsubst
+    - cat mirrors.yaml.in | envsubst > mirrors.yaml
     - attempts=0
     - nocache=""
+    - docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --print
+        --file containers/variables.hcl
+        --file containers/jug/dev.hcl
     - while !
-      docker buildx build --push ${BUILD_OPTIONS} ${nocache}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
-                   ${EXPORT_TAG:+
-                     ${IF_BUILD_DEFAULT+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
-                     }
-                     ${IF_BUILD_DEFAULT-
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                     }
-                   }
-                   ${IF_BUILD_NIGHTLY+
-                     ${NIGHTLY:+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
-                       ${PUSH_NIGHTLY_WITH_DATE:+
-                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                       }
-                     }
-                   }
-                   ${EXPORT_TAG:+
-                     ${IF_BUILD_DEFAULT+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}}
-                     }
-                     ${IF_BUILD_DEFAULT-
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                     }
-                   }
-                   ${IF_BUILD_NIGHTLY+
-                     ${NIGHTLY:+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}}
-                     }
-                       ${PUSH_NIGHTLY_WITH_DATE:+
-                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                       }
-                   }
-                   --file containers/jug/dev.Dockerfile
-                   --platform ${PLATFORM}
-                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
-                   --build-arg BUILDER_IMAGE=${BUILDER_IMAGE}
-                   --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE}
-                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
-                   --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
-                   ${IF_BUILD_DEFAULT+
-                     ${EDM4EIC_VERSION:+--build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION}}
-                     ${EICRECON_VERSION:+--build-arg EICRECON_VERSION=${EICRECON_VERSION}}
-                     ${EPIC_VERSION:+--build-arg EPIC_VERSION=${EPIC_VERSION}}
-                     ${JUGGLER_VERSION:+--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}}
-                   }
-                   ${IF_BUILD_NIGHTLY+
-                     --build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION:-main}
-                     --build-arg EICRECON_VERSION=${EICRECON_VERSION:-main}
-                     --build-arg EPIC_VERSION=${EPIC_VERSION:-main}
-                     --build-arg JUGGLER_VERSION=${JUGGLER_VERSION:-main}
-                   }
-                   --build-arg ENV=${ENV}
-                   --build-arg jobs=${JOBS}
-                   --build-context spack-environment=spack-environment
-                   --secret id=mirrors,src=mirrors.yaml
-                   --provenance false
-                   containers/jug
-                   2>&1 | tee build.log
+      docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --push
+        --file containers/variables.hcl
+        --file containers/jug/dev.hcl
+        2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;

containers/debian/base.Dockerfile

@@ -1,7 +1,7 @@
 #syntax=docker/dockerfile:1.8
 #check
 ARG BASE_IMAGE="amd64/debian:stable-slim"
-ARG BUILD_IMAGE="debian_stable_base"
+ARG BUILD_IMAGE="debian_base"
 
 # Minimal container based on Debian base systems for up-to-date packages. 
 FROM  ${BASE_IMAGE}

containers/debian/base.hcl

@@ -0,0 +1,45 @@
+# Variables which are required to be defined
+variable "jobs" { default = 1 }
+variable "BUILD_IMAGE" { default = "debian_base" }
+
+# Variables whose defaults can be overridden on build
+variable "BASE_IMAGE" { default = null }
+
+variable "S3_ACCESS_KEY" { default = null }
+variable "S3_SECRET_KEY" { default = null }
+
+# docker/metadata-action overrides the following target with tags
+# but we implement it for use outside docker/metadata-action
+target "docker-metadata-action" {
+  tags = compact(flatten([
+    join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${BUILD_IMAGE}:${INTERNAL_TAG}"]) ),
+    EXPORT_TAG != null && EXPORT_TAG != "" ? [
+      for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}"
+    ] : [ null ]
+  ]))
+}
+
+target "default" {
+  inherits = ["docker-metadata-action"]
+  attest = [
+    "type=provenance,disabled=true"
+  ]
+  context = "containers/debian"
+  dockerfile = "base.Dockerfile"
+  platforms = [ "linux/amd64" ]
+  args = {
+    jobs = jobs
+    BASE_IMAGE = BASE_IMAGE
+    BUILD_IMAGE = BUILD_IMAGE
+    SPACK_ORGREPO = try(SPACK_ORGREPO, null)
+    SPACK_VERSION = try(SPACK_VERSION, null)
+    SPACK_CHERRYPICKS = join("\n", try(SPACK_CHERRYPICKS, []))
+    SPACK_CHERRYPICKS_FILES = join("\n", try(SPACK_CHERRYPICKS_FILES, []))
+    KEY4HEPSPACK_ORGREPO = try(KEY4HEPSPACK_ORGREPO, null)
+    KEY4HEPSPACK_VERSION = try(KEY4HEPSPACK_VERSION, null)
+    EICSPACK_ORGREPO = try(EICSPACK_ORGREPO, null)
+    EICSPACK_VERSION = try(EICSPACK_VERSION, null)
+    S3_ACCESS_KEY = S3_ACCESS_KEY
+    S3_SECRET_KEY = S3_SECRET_KEY
+  }
+}

containers/jug/dev.Dockerfile

@@ -1,8 +1,8 @@
 #syntax=docker/dockerfile:1.8
 #check
 ARG DOCKER_REGISTRY="eicweb/"
-ARG BUILDER_IMAGE="debian_stable_base"
-ARG RUNTIME_IMAGE="debian_stable_base"
+ARG BUILDER_IMAGE="debian_base"
+ARG RUNTIME_IMAGE="debian_base"
 ARG INTERNAL_TAG="testing"
 
 ##
@@ -39,7 +39,7 @@ ARG TARGETPLATFORM
 
 ## Copy our default environment
 COPY --from=spack-environment . /opt/spack-environment/
-ARG ENV=dev
+ARG ENV=xl
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
 
 # Concretization (default environment)