Test Not for review holodeck kernel version change

Author: shivakunv

.github/workflows/image.yaml

@@ -21,12 +21,12 @@ on:
       - opened
       - synchronize
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
   push:
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
 
 jobs:
   image:

.github/workflows/precompiled.yaml

@@ -16,8 +16,15 @@
 name: Precompiled images
 
 on:
-  schedule:
-    - cron: '00 09 * * *'
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - test-holodeck
+  push:
+    branches:
+      - test-holodeck
 
 jobs:
   set-driver-version-matrix:
@@ -40,16 +47,19 @@ jobs:
 
           # get kernel flavors
           KERNEL_FLAVORS=("aws" "azure" "generic" "nvidia" "oracle")
+          # KERNEL_FLAVORS=("generic")
           kernel_flavors_json=$(printf '%s\n' "${KERNEL_FLAVORS[@]}" | jq -R . | jq -cs .)
           echo "kernel_flavors=$kernel_flavors_json" >> $GITHUB_OUTPUT
 
           # get ubuntu distributions
           DIST=("ubuntu22.04" "ubuntu24.04")
+          # DIST=("ubuntu22.04")
           dist_json=$(printf '%s\n' "${DIST[@]}" | jq -R . | jq -cs .)
           echo "dist=$dist_json" >> $GITHUB_OUTPUT
 
           # LTS_KERNEL setup
-          LTS_KERNEL=("5.15" "6.8")
+          # LTS_KERNEL=("5.15" "6.8")
+          LTS_KERNEL=("6.8")
           lts_kernel_json=$(printf '%s\n' "${LTS_KERNEL[@]}" | jq -R . | jq -cs .)
           echo "lts_kernel=$lts_kernel_json" >> $GITHUB_OUTPUT
 
@@ -273,18 +283,18 @@ jobs:
 
       # slack notification for new kernel release before e2e tests starts
       # as e2e tests may fail for new kernel release
-      - name: Slack notification
-        if: steps.set_kernel_version.outputs.matrix_values_not_empty == '1'
-        uses: slackapi/[email protected]
-        with:
-          token: ${{ secrets.SLACK_BOT_TOKEN }}
-          method: chat.postMessage
-          payload: |
-            {
-              "channel": "${{ secrets.SLACK_CHANNEL_ID }}",
-              "text": "${{ secrets.DETECTED_PRECOMPILED_KERNEL_MESSAGE }} ${{ steps.set_kernel_version.outputs.published_kernels }} \n
-              Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            }
+      # - name: Slack notification
+      #   if: steps.set_kernel_version.outputs.matrix_values_not_empty == '1'
+      #   uses: slackapi/[email protected]
+      #   with:
+      #     token: ${{ secrets.SLACK_BOT_TOKEN }}
+      #     method: chat.postMessage
+      #     payload: |
+      #       {
+      #         "channel": "${{ secrets.SLACK_CHANNEL_ID }}",
+      #         "text": "${{ secrets.DETECTED_PRECOMPILED_KERNEL_MESSAGE }} ${{ steps.set_kernel_version.outputs.published_kernels }} \n
+      #         Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+      #       }
 
   e2e-tests-nvidiadriver:
     runs-on: linux-amd64-cpu4
@@ -319,9 +329,12 @@ jobs:
           driver_branch_json="${{ needs.set-driver-version-matrix.outputs.driver_branch }}"
           DRIVER_BRANCHES=($(echo "$driver_branch_json" | jq -r '.[]'))
           echo "DRIVER_BRANCHES=${DRIVER_BRANCHES[*]}" >> $GITHUB_ENV
+      - name: Set kernel version in holodeck_${{ env.DIST }}.yaml
+        run: |
+          yq e '.spec.kernel.version = "${{ env.KERNEL_VERSION }}"' -i tests/holodeck_${{ env.DIST }}.yaml
 
       - name: Set up Holodeck
-        uses: NVIDIA/[email protected].6
+        uses: NVIDIA/[email protected].15
         env:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -346,27 +359,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y gh
 
-      - name: Upgrade the kernel for Precompiled e2e test
-        env:
-          UPGRADE_KERNEL_SCRIPT: "./tests/scripts/upgrade-kernel.sh"
-        run: |
-          status=0
-          ./tests/ci-remote-exec.sh "${UPGRADE_KERNEL_SCRIPT}" "${KERNEL_VERSION}" || status=$?
-          # On the target system, all scripts/test-case exit with code 1 for error handling.
-          # However, since reboot-related disconnections break the SSH connection
-          # and can cause the entire job to exit, we should ignore all errors except
-          # exit code 1. During a reboot, exit code 1 will not be thrown, so handling
-          # other errors as code 1 will ensure proper management of reboot scenarios
-          if [ $status -eq 1 ]; then
-            echo "Kernel version $KERNEL_VERSION upgrade failed"
-            exit 1
-          fi
-          ./tests/scripts/remote_retry.sh || status=$?
-          if [ $status -ne 0 ]; then
-              echo "Failed to connect to remote instance"
-              exit $status
-          fi
-
       - name: Precompiled e2e test gpu driver validation
         env:
           TEST_CASE: "./tests/cases/nvidia-driver.sh"
@@ -455,8 +447,8 @@ jobs:
           fi
           image_path="./base-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}.tar"
           echo "uploading  $image_path"
-          docker load -i $image_path
-          docker push ${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${KERNEL_FLAVOR}-${{ matrix.driver_branch }}
+          # docker load -i $image_path
+          # docker push ${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${KERNEL_FLAVOR}-${{ matrix.driver_branch }}
 
       - name: Download built image artifact
         if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
@@ -470,17 +462,17 @@ jobs:
         run: |
           image_path="./driver-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}.tar"
           echo "uploading  $image_path"
-          docker load -i $image_path
-          docker push ${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}
-
-      - name: Slack notification
-        if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
-        uses: slackapi/[email protected]
-        with:
-          token: ${{ secrets.SLACK_BOT_TOKEN }}
-          method: chat.postMessage
-          payload: |
-            {
-              "channel": "${{ secrets.SLACK_CHANNEL_ID }}",
-              "text": "${{ secrets.PUBLISHED_PRECOMPILED_IMAGE_MESSAGE }} ${{ env.PRIVATE_REGISTRY }}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}"
-            }
+          # docker load -i $image_path
+          # docker push ${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}
+
+      # - name: Slack notification
+      #   if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }}
+      #   uses: slackapi/[email protected]
+      #   with:
+      #     token: ${{ secrets.SLACK_BOT_TOKEN }}
+      #     method: chat.postMessage
+      #     payload: |
+      #       {
+      #         "channel": "${{ secrets.SLACK_CHANNEL_ID }}",
+      #         "text": "${{ secrets.PUBLISHED_PRECOMPILED_IMAGE_MESSAGE }} ${{ env.PRIVATE_REGISTRY }}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}"
+      #       }

tests/holodeck_ubuntu22.04.yaml

@@ -30,5 +30,7 @@ spec:
   kubernetes:
     install: true
     installer: kubeadm
-    version: v1.28.5
-    crictlVersion: v1.28.0
+    version: v1.31.1
+    crictlVersion: v1.31.1
+  kernel:
+    version: ${{ env.KERNEL_VERSION }}

tests/holodeck_ubuntu24.04.yaml

@@ -24,6 +24,10 @@ spec:
     image:
       architecture: amd64
       imageId: ami-0da424eb883458071
+  containerdConfigPatches:
+  - |-
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "/etc/containerd/certs.d"
   containerRuntime:
     install: true
     name: containerd
@@ -33,3 +37,5 @@ spec:
     installer: kubeadm
     version: v1.30.0
     crictlVersion: v1.30.0
+  kernel:
+    version:${{ env.KERNEL_VERSION }}

tests/scripts/findkernelversion.sh

@@ -42,3 +42,4 @@ if [[ $status_nvcr -eq 0 || $status_ghcr -eq 0 ]]; then
 else
     export should_continue=true
 fi
+export should_continue=true

tests/scripts/install-operator.sh

@@ -22,6 +22,7 @@ kubectl create namespace "${TEST_NAMESPACE}"
 
 # Run the helm install command
 echo "OPERATOR_OPTIONS: ${OPERATOR_OPTIONS}"
+
 eval ${HELM} install gpu-operator  nvidia/gpu-operator \
 	-n "${TEST_NAMESPACE}" \
 	"${OPERATOR_OPTIONS}" \