ubuntu24.04 ci pipeline fix

Signed-off-by: shiva kumar <[email protected]>

Author: tariq1890

.common-ci.yml

@@ -88,6 +88,13 @@ trigger-pipeline:
       - DRIVER_BRANCH: [535, 550]
         KERNEL_FLAVOR: [aws, azure, generic, nvidia, oracle]
 
+# Define the matrix of precompiled jobs that can be run in parallel for ubuntu24.04
+.driver-versions-precompiled-ubuntu24.04:
+  parallel:
+    matrix:
+      - DRIVER_BRANCH: [535, 550]
+        KERNEL_FLAVOR: [aws, azure, generic, nvidia, oracle]
+
 # Define the distribution targets
 .dist-ubuntu20.04:
   variables:
@@ -172,6 +179,14 @@ trigger-pipeline:
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule"
 
+.release-ubuntu24.04:
+  # Perform for each DRIVER_VERSION
+  extends:
+    - .release-generic
+    - .driver-versions
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+
 .release-rhel9:
   # Perform for each DRIVER_VERSION
   extends:
@@ -304,3 +319,14 @@ release:staging-precompiled-ubuntu22.04:
     - .release:staging-precompiled
   needs:
     - image-precompiled-ubuntu22.04
+
+# Precompiled Ubuntu24.04 release
+release:staging-precompiled-ubuntu24.04:
+  variables:
+    DIST: signed_ubuntu24.04
+    BASE_TARGET: noble
+  extends:
+    - .driver-versions-precompiled-ubuntu24.04
+    - .release:staging-precompiled
+  needs:
+    - image-precompiled-ubuntu24.04

.github/workflows/ci.yaml

@@ -40,7 +40,7 @@ jobs:
         aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
         aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
-        holodeck_config: "tests/holodeck.yaml"
+        holodeck_config: "tests/holodeck_ubuntu22.04.yaml"
 
     - name: Get public dns name
       id: get_public_dns_name

.github/workflows/precompiled.yaml

@@ -15,9 +15,20 @@
 # Run this workflow on a schedule
 name: Precompiled images
 
+# on:
+#   schedule:
+#     - cron: '00 09 * * *' # scheduled job
+
 on:
-  schedule:
-    - cron: '00 09 * * *'
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - ci-precompile-ubuntu24.04
+  push:
+    branches:
+      - ci-precompile-ubuntu24.04
 
 jobs:
   set-driver-version-matrix:
@@ -43,7 +54,8 @@ jobs:
           echo "kernel_flavors=$kernel_flavors_json" >> $GITHUB_OUTPUT
 
           # get ubuntu distributions
-          DIST=("ubuntu22.04")
+          # DIST=("ubuntu22.04" "ubuntu24.04")
+          DIST=("ubuntu24.04")
           dist_json=$(printf '%s\n' "${DIST[@]}" | jq -R . | jq -cs .)
           echo "dist=$dist_json" >> $GITHUB_OUTPUT
 
@@ -60,6 +72,8 @@ jobs:
         name: Check out code
       - name: Calculate build vars
         id: vars
+        env:
+          DIST: ${{ matrix.dist }}
         run: |
           echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
           echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV
@@ -70,6 +84,14 @@ jobs:
           echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
           echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
 
+          if [ "$DIST" == "ubuntu22.04" ]; then
+            echo  "BASE_TARGET=jammy" >> $GITHUB_OUTPUT
+            echo "LTS_KERNEL=5.15" >> $GITHUB_OUTPUT
+          elif [ "$DIST" == "ubuntu24.04" ]; then
+            echo  "BASE_TARGET=noble" >> $GITHUB_OUTPUT
+            echo "LTS_KERNEL=6.8" >> $GITHUB_OUTPUT
+          fi
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -84,9 +106,10 @@ jobs:
         env:
           IMAGE_NAME: ghcr.io/nvidia/driver
           VERSION: ${COMMIT_SHORT_SHA}
-          BASE_TARGET: jammy
+          BASE_TARGET: ${{ steps.vars.outputs.BASE_TARGET }}
+          LTS_KERNEL: ${{ steps.vars.outputs.LTS_KERNEL }}
         run: |
-          make DRIVER_BRANCH=${{ matrix.driver_branch }} KERNEL_FLAVOR=${{ matrix.flavor }} build-base-${BASE_TARGET}
+          make DRIVER_BRANCH=${{ matrix.driver_branch }} KERNEL_FLAVOR=${{ matrix.flavor }} LTS_KERNEL=${LTS_KERNEL} build-base-${BASE_TARGET}
 
           trap "docker rm -f base-${BASE_TARGET}-${{ matrix.flavor }}" EXIT
           docker run -d --name base-${BASE_TARGET}-${{ matrix.flavor }} ghcr.io/nvidia/driver:base-${BASE_TARGET}-${{ matrix.flavor }}-${{ matrix.driver_branch }}
@@ -105,12 +128,13 @@ jobs:
           source kernel_version.txt && \
           make DRIVER_VERSIONS=${DRIVER_VERSIONS} DRIVER_BRANCH=${{ matrix.driver_branch }} build-${DIST}-${DRIVER_VERSION}
 
-      - name: Save build image as a tar
+      - name: Save build image and kernel version file
         env:
           DIST: ${{ matrix.dist }}
           PRIVATE_REGISTRY: "ghcr.io"
         run: |
           source kernel_version.txt
+          tar -cvf kernel-version-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar kernel_version.txt
           docker save "${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}" \
             -o  ./driver-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar
           # set env for artifacts upload
@@ -123,7 +147,14 @@ jobs:
            name: driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}
            path: ./driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar
            retention-days: 1
-
+      
+      - name: Upload kernel version as an artifact
+        uses: actions/upload-artifact@v4
+        with:
+           name: kernel-version-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}
+           path: ./kernel-version-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar
+           retention-days: 1
+  
   determine-e2e-test-matrix:
     runs-on: linux-amd64-cpu4
     strategy:
@@ -149,11 +180,19 @@ jobs:
       - name: Set kernel version
         id: set_kernel_version
         env:
-          BASE_TARGET: "jammy"
           DIST: ${{ matrix.dist }}
+          GH_TOKEN:  ${{ secrets.GITHUB_TOKEN }}
         run: |
           echo "matrix_values_not_empty=0" >> $GITHUB_OUTPUT
 
+          if [ "$DIST" == "ubuntu22.04" ]; then
+            export BASE_TARGET="jammy"
+            export LTS_KERNEL="5.15"
+          elif [ "$DIST" == "ubuntu24.04" ]; then
+            export BASE_TARGET="noble"
+            export LTS_KERNEL="6.8"
+          fi
+
           kernel_flavors_json='${{ needs.set-driver-version-matrix.outputs.kernel_flavors }}'
           KERNEL_FLAVORS=($(echo "$kernel_flavors_json" | jq -r '.[]'))
           driver_branch_json='${{ needs.set-driver-version-matrix.outputs.driver_branch }}'
@@ -201,7 +240,7 @@ jobs:
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
-          holodeck_config: "tests/holodeck.yaml"
+          holodeck_config: "tests/holodeck_${{ matrix.dist }}.yaml"
 
       - name: Get public dns name
         id: get_public_dns_name
@@ -222,6 +261,15 @@ jobs:
           DRIVER_BRANCHES=($(echo "$driver_branch_json" | jq -r '.[]'))
           echo "DRIVER_BRANCHES=${DRIVER_BRANCHES[*]}" >> $GITHUB_ENV
 
+      # step added to skip azure e2e tests and publish the image
+      # FIXME -- remove step once azure kernel upgrade starts working
+      - name: Skip azure e2e
+        run: |
+          if [[ "${KERNEL_VERSION}" == *-azure ]]; then
+              echo "e2e test for azure flavor skipped, as kernel upgrade AWS => azure is not supported"
+              exit 0
+          fi
+
       - name: Install GitHub CLI
         run: |
           sudo apt-get update
@@ -296,7 +344,7 @@ jobs:
       matrix:
         driver_branch: ${{ fromJson(needs.set-driver-version-matrix.outputs.driver_branch) }}
         kernel_version: ${{ fromJson(needs.determine-e2e-test-matrix.outputs.matrix_values) }}
-        dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }}     
+        dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4

.gitlab-ci.yml

@@ -53,7 +53,7 @@ include:
 .image-build-ubuntu24.04:
   # Perform for each DRIVER_VERSION
   extends:
-    - .driver-versions-ubuntu24.04
+    - .driver-versions
     - .image-build-generic
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule"
@@ -120,3 +120,12 @@ image-precompiled-ubuntu22.04:
   extends:
     - .driver-versions-precompiled-ubuntu22.04
     - .image-build-precompiled
+
+image-precompiled-ubuntu24.04:
+  variables:
+    DIST: signed_ubuntu24.04
+    BASE_TARGET: noble
+    CVE_UPDATES: "curl libc6"
+  extends:
+    - .driver-versions-precompiled-ubuntu24.04
+    - .image-build-precompiled

.nvidia-ci.yml

@@ -77,6 +77,30 @@ variables:
     - !reference [.image-pull-rules, rules]
 
 
+.image-pull-ubuntu24.04:
+  # Perform for each DRIVER_VERSION
+  extends:
+    - .driver-versions
+    - .image-pull-generic
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - !reference [.image-pull-rules, rules]
+
+image-precompiled-ubuntu24.04:
+  variables:
+    DIST: signed_ubuntu24.04
+    BASE_TARGET: noble
+    PRECOMPILED: "true"
+    CVE_UPDATES: "curl libc6"
+  rules:
+    - when: delayed
+      start_in: 30 minutes
+  extends:
+    - .driver-versions-precompiled-ubuntu24.04
+    - .image-pull-generic
+
+
 .image-pull-ubuntu22.04:
   # Perform for each DRIVER_VERSION
   extends:
@@ -187,7 +211,7 @@ image-rhel8:
 .scan-ubuntu24.04:
   # Repeat for each DRIVER_VERSION
   extends:
-    - .driver-versions-ubuntu24.04
+    - .driver-versions
     - .scan-generic
   rules:
     - !reference [.scan-rules-common, rules]
@@ -196,6 +220,18 @@ image-rhel8:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     - !reference [.pipeline-trigger-rules, rules]
 
+.scan-precompiled-ubuntu24.04:
+  variables:
+    DIST: signed_ubuntu24.04
+    BASE_TARGET: noble
+    PRECOMPILED: "true"
+  extends:
+    - .driver-versions-precompiled-ubuntu24.04
+    - .scan-generic
+  rules:
+    - !reference [.scan-rules-common, rules]
+    - when: always
+
 .scan-precompiled-ubuntu22.04:
   variables:
     DIST: signed_ubuntu22.04
@@ -306,6 +342,25 @@ release:ngc-ubuntu22.04:
     - .dist-ubuntu22.04
     - .driver-versions
 
+release:ngc-ubuntu24.04:
+  extends:
+    - .release:ngc
+    - .dist-ubuntu24.04
+    - .driver-versions
+
+release:ngc-precompiled-ubuntu24.04:
+  variables:
+    DIST: signed_ubuntu24.04
+    BASE_TARGET: noble
+    PRECOMPILED: "true"
+  extends:
+    - .driver-versions-precompiled-ubuntu24.04
+    - .release-generic
+    - .release:ngc-variables
+  rules:
+    # Only run NGC release job on scheduled pipelines
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+
 release:ngc-precompiled-ubuntu22.04:
   variables:
     DIST: signed_ubuntu22.04
@@ -433,6 +488,23 @@ release:ngc-rhel8.10:
     - 'echo "Signing the image ${IMAGE_NAME}:${IMAGE_TAG}"'
     - ngc-cli/ngc registry image publish --source ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${IMAGE_TAG} --public --discoverable --allow-guest --sign --org nvidia
 
+sign:ngc-precompiled-ubuntu24.04:
+  extends:
+    - .driver-versions-precompiled-ubuntu24.04
+    - .dist-ubuntu22.04
+    - .release-generic
+    - .release:ngc-variables
+    - .sign:ngc
+  variables:
+    DIST: signed_ubuntu24.04
+    BASE_TARGET: noble
+    PRECOMPILED: "true"
+  needs:
+    - release:ngc-precompiled-ubuntu24.04
+  rules:
+    # Only run NGC release job on scheduled pipelines
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+
 sign:ngc-precompiled-ubuntu22.04:
   extends:
     - .driver-versions-precompiled-ubuntu22.04
@@ -455,6 +527,9 @@ sign:ngc-ubuntu-rhel-rhcos:
     - .sign:ngc
   parallel:
     matrix:
+    - SIGN_JOB_NAME: ["ubuntu"]
+      VERSION: ["24.04"]
+      DRIVER_VERSION: ["535.216.03", "550.127.08"]
     - SIGN_JOB_NAME: ["ubuntu"]
       VERSION: ["22.04"]
       DRIVER_VERSION: ["535.216.03", "550.127.08"]

Makefile

@@ -54,9 +54,9 @@ OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(OUT_DIST)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
-DISTRIBUTIONS := ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 signed_ubuntu20.04 signed_ubuntu22.04 signed_ubuntu24.04 rhel8 rhel9 flatcar fedora36 sles15.3 precompiled_rhcos
+DISTRIBUTIONS := ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 signed_ubuntu24.04 signed_ubuntu20.04 signed_ubuntu22.04 rhel8 rhel9 flatcar fedora36 sles15.3 precompiled_rhcos
 PUSH_TARGETS := $(patsubst %, push-%, $(DISTRIBUTIONS))
-BASE_FROM := jammy focal
+BASE_FROM := jammy focal noble
 PUSH_TARGETS := $(patsubst %, push-%, $(DISTRIBUTIONS))
 DRIVER_PUSH_TARGETS := $(foreach push_target, $(PUSH_TARGETS), $(addprefix $(push_target)-, $(DRIVER_VERSIONS)))
 BUILD_TARGETS := $(patsubst %, build-%, $(DISTRIBUTIONS))
@@ -210,6 +210,7 @@ $(BASE_BUILD_TARGETS):
 				--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 				--build-arg DRIVER_BRANCH="$(DRIVER_BRANCH)" \
 				--build-arg KERNEL_FLAVOR="$(KERNEL_FLAVOR)" \
+				--build-arg LTS_KERNEL="$(LTS_KERNEL)" \
 				--file $(DOCKERFILE) \
 				$(CURDIR)/base