Add init container to plugin pod (GPU driver dependency check)

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

Author: jgehrcke

deployments/container/Dockerfile

@@ -94,4 +94,5 @@ COPY --from=build   /artifacts/compute-domain-controller     /usr/bin/compute-do
 COPY --from=build   /artifacts/compute-domain-kubelet-plugin /usr/bin/compute-domain-kubelet-plugin
 COPY --from=build   /artifacts/compute-domain-daemon         /usr/bin/compute-domain-daemon
 COPY --from=build   /artifacts/gpu-kubelet-plugin            /usr/bin/gpu-kubelet-plugin
+COPY --from=build   /build/hack/kubelet-plugin-prestart.sh   /usr/bin/kubelet-plugin-prestart.sh
 COPY --from=build   /build/templates                         /templates

deployments/helm/nvidia-dra-driver-gpu/templates/kubeletplugin.yaml

@@ -49,6 +49,24 @@ spec:
       serviceAccountName: {{ include "nvidia-dra-driver-gpu.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.kubeletPlugin.podSecurityContext | nindent 8 }}
+      initContainers:
+      - name: init-container
+        image: {{ include "nvidia-dra-driver-gpu.fullimage" . }}
+        securityContext:
+          privileged: true
+        # Bashisms are used in kubelet-plugin-prestart.sh. Feed logic from
+        # stdin: script path does not exist in chroot environment.
+        command: [sh, "-c", "chroot /driver-root bash -s < /usr/bin/kubelet-plugin-prestart.sh"]
+        env:
+        - name: NVIDIA_DRIVER_ROOT
+          value: "{{ .Values.nvidiaDriverRoot }}"
+        # Explicit "void"; otherwise we inherit "all".
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: void
+        volumeMounts:
+        - name: driver-root
+          mountPath: /driver-root
+          readOnly: true
       containers:
       {{- if .Values.resources.computeDomains.enabled }}
       - name: compute-domains

hack/kubelet-plugin-prestart.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Main intent: help users to self-troubleshoot when the GPU driver is not set up
+# properly before installing this DRA driver. In that case, the log of the init
+# container running this script is meant to yield an actionable error message.
+#
+# Crash-loop (retry periodically) and actively wait for the GPU driver to come
+# online (unblock the DRA driver plugin daemonset pods from progressing right
+# after the GPU driver setup has been fixed).
+
+if [ -z "$NVIDIA_DRIVER_ROOT" ]; then
+    # Not set, or set to empty string (not distinguishable).
+    # Normalize to "/" (treated as such elsewhere).
+    export NVIDIA_DRIVER_ROOT="/"
+fi
+
+echo "NVIDIA_DRIVER_ROOT (path on host): ${NVIDIA_DRIVER_ROOT}"
+
+while true
+do
+if ! command -v nvidia-smi &>/dev/null
+then
+    printf '%b' \
+    "Not in PATH: 'nvidia-smi'. " \
+    "Has the NVIDIA GPU driver been set up? " \
+    "The GPU driver is expected to be installed under " \
+    "NVIDIA_DRIVER_ROOT ('${NVIDIA_DRIVER_ROOT}') in the host filesystem. " \
+    "If NVIDIA_DRIVER_ROOT appears to be unexpected: " \
+    "review and adjust the 'nvidiaDriverRoot' Helm chart variable. " \
+    "If the value is expected: review if the GPU driver has " \
+    "actually been installed under NVIDIA_DRIVER_ROOT. " \
+    "If you chose the NVIDIA GPU Operator to manage the GPU driver " \
+    "(NVIDIA_DRIVER_ROOT set to /run/nvidia/driver): "\
+    "make sure that Operator is deployed and healthy.\n"
+
+    # Provide hint for a specific, common mistake.
+    if   [ "${NVIDIA_DRIVER_ROOT}" == "/" ] && [ -f /run/nvidia/driver/usr/bin/nvidia-smi ]; then
+        printf '%b' \
+        "Note: /run/nvidia/driver/usr/bin/nvidia-smi exists on the host, you " \
+        "may want to re-install the DRA driver Helm chart with " \
+        "--set nvidiaDriverRoot=/run/nvidia/driver\n"
+    fi
+else
+    # Note: the following path is in-container, after chroot to /driver-root (and
+    # /driver-root is where NVIDIA_DRIVER_ROOT is mounted from the host filesystem).
+    # This typically outputs /usr/bin/nvidia-smi.
+    echo "command -v nvidia-smi: $(command -v nvidia-smi)"
+
+    # This may be slow or hang, in a bad setup.
+    echo "invoking nvidia-smi"
+    nvidia-smi
+    RCODE="$?"
+
+    # For checking GPU driver health: for now, rely on nvidia-smi's exit code.
+    # Rely on code 0 meaning that the driver is properly set up. For example,
+    # code 9 means that the GPU driver is not loaded; see section 'RETURN VALUE'
+    # in the man page.
+    if [ ${RCODE} -eq 0 ]
+    then
+        echo "nvidia-smi returned with code 0: success, leave"
+        exit 0
+    else
+        # Do we worry about false-negatives here (where nvidia-smi fails
+        # e.g. because of one bad GPU but where we could still proceed)?
+        printf '%b' \
+        "nvidia-smi returned an error (code ${RCODE}). " \
+        "Verify that the GPU driver is set up correctly\n"
+    fi
+fi
+echo "retry in 30 s"
+sleep 30
+done