feat: Implement log collector mock mode and KWOK failure injection for testing

Author: rupalis-nv

distros/kubernetes/nvsentinel/charts/fault-remediation/files/log-collector-job.yaml

@@ -36,7 +36,7 @@ spec:
       {{- end }}
       containers:
         - name: log-collector
-          image: {{ .Values.logCollector.image.repository }}:{{ .Values.global.image.tag | default "latest"  }}
+          image: {{ .Values.logCollector.image.repository }}:{{ .Values.logCollector.image.tag | default .Values.global.image.tag | default "latest"  }}
           imagePullPolicy: {{ .Values.logCollector.image.pullPolicy }}          
           securityContext:
             privileged: true
@@ -61,6 +61,10 @@ spec:
               value: {{ .Values.logCollector.enableGcpSosCollection | quote }}
             - name: ENABLE_AWS_SOS_COLLECTION
               value: {{ .Values.logCollector.enableAwsSosCollection | quote }}
+            {{- range $key, $value := .Values.logCollector.env }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
           volumeMounts:
             - name: artifacts
               mountPath: /artifacts

distros/kubernetes/nvsentinel/charts/fault-remediation/values.yaml

@@ -96,6 +96,7 @@ logCollector:
   enabled: false
   image:
     repository: ghcr.io/nvidia/nvsentinel/log-collector
+    # tag: latest  # Optional: Override global.image.tag for log-collector
     pullPolicy: IfNotPresent
   # HTTP endpoint where collected logs will be uploaded
   uploadURL: "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local/upload"

distros/kubernetes/nvsentinel/kwok-stages/log-collector-failure-stage.yaml

@@ -0,0 +1,44 @@
+##
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+# KWOK Stage to inject failure into log-collector jobs on KWOK nodes
+# This stage makes jobs with label 'test-scenario=log-collector-failure' fail
+apiVersion: kwok.x-k8s.io/v1alpha1
+kind: Stage
+metadata:
+  name: job-log-collector-failure
+spec:
+  resourceRef:
+    apiGroup: batch
+    kind: Job
+  selector:
+    matchLabels:
+      test-scenario: log-collector-failure
+  delay:
+    durationMilliseconds: 1000
+  next:
+    statusTemplate: |
+      conditions:
+      - type: Failed
+        status: "True"
+        reason: BackoffLimitExceeded
+        message: "Job has reached the specified backoff limit"
+        lastProbeTime: {{ now }}
+        lastTransitionTime: {{ now }}
+      failed: 1
+      startTime: {{ now }}
+      completionTime: {{ now }}
+

distros/kubernetes/nvsentinel/values-tilt.yaml

@@ -121,6 +121,17 @@ fault-quarantine:
 fault-remediation:
   logLevel: debug
 
+  logCollector:
+    enabled: true
+    image:
+      repository: localhost:5001/ghcr.io_nvidia_nvsentinel_log-collector
+      tag: latest  # Use latest tag built by Tilt
+      pullPolicy: Always  # Always pull latest Tilt-built image
+    env:
+      MOCK_MODE: "false"
+      MOCK_EXIT_CODE: "0"
+      MOCK_SLEEP_DURATION: "2"
+
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:

fault-remediation/pkg/reconciler/remediation.go

@@ -300,6 +300,10 @@ func (c *FaultRemediationClient) RunLogCollectorJob(ctx context.Context, nodeNam
 	// Set target node
 	job.Spec.Template.Spec.NodeName = nodeName
 
+	// Check for test-scenario annotation on the node and propagate to job and pod labels
+	log.Printf("[DEBUG] About to check test-scenario annotation for node %s", nodeName)
+	c.propagateTestScenarioLabel(ctx, nodeName, job)
+
 	// Create Job using typed client
 	created, err := c.kubeClient.BatchV1().Jobs(job.Namespace).Create(ctx, job, metav1.CreateOptions{})
 	if err != nil {
@@ -415,3 +419,44 @@ func (c *FaultRemediationClient) RunLogCollectorJob(ctx context.Context, nodeNam
 		return result
 	}
 }
+
+// propagateTestScenarioLabel checks for test-scenario annotation on node and propagates to job/pod labels.
+func (c *FaultRemediationClient) propagateTestScenarioLabel(ctx context.Context, nodeName string, job *batchv1.Job) {
+	log.Printf("[DEBUG] propagateTestScenarioLabel called for node %s", nodeName)
+
+	node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+	if err != nil {
+		log.Printf("Warning: failed to get node %s for annotation check: %v", nodeName, err)
+		return
+	}
+
+	log.Printf("[DEBUG] Node %s annotations: %v", nodeName, node.Annotations)
+
+	testScenario, ok := node.Annotations["nvsentinel.nvidia.com/test-scenario"]
+	if !ok {
+		log.Printf("[DEBUG] No test-scenario annotation found on node %s", nodeName)
+		return
+	}
+
+	log.Printf("[DEBUG] Found test-scenario annotation: %s", testScenario)
+
+	// Set label on Job metadata
+	if job.Labels == nil {
+		job.Labels = make(map[string]string)
+	}
+
+	job.Labels["test-scenario"] = testScenario
+
+	// Set label on Pod template metadata (so pods inherit it)
+	if job.Spec.Template.Labels == nil {
+		job.Spec.Template.Labels = make(map[string]string)
+	}
+
+	job.Spec.Template.Labels["test-scenario"] = testScenario
+
+	log.Printf(
+		"Applied test-scenario label '%s' to log collector job and pod template for node %s",
+		testScenario,
+		nodeName,
+	)
+}

log-collector/Dockerfile

@@ -42,7 +42,9 @@ RUN useradd -u 10001 -m nvsentinel
 WORKDIR /opt/log-collector
 
 COPY log-collector/entrypoint.sh /opt/log-collector/entrypoint.sh
-RUN chmod +x /opt/log-collector/entrypoint.sh
+COPY log-collector/mock-nvidia-bug-report.sh /opt/log-collector/mock-nvidia-bug-report.sh
+COPY log-collector/mock-must-gather.sh /opt/log-collector/mock-must-gather.sh
+RUN chmod +x /opt/log-collector/entrypoint.sh /opt/log-collector/mock-nvidia-bug-report.sh /opt/log-collector/mock-must-gather.sh
 
 ENV PATH="/opt/log-collector:${PATH}"
 

log-collector/Tiltfile

@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-docker_build(
-    "ghcr.io/nvidia/nvsentinel/log-collector",
-    context=".",
-    dockerfile="./Dockerfile"
+# Build docker image and push to local registry with fixed tag
+# Using local_resource since the image is used in dynamically created jobs
+# Build from repo root since Dockerfile expects that context
+local_resource(
+    "log-collector",
+    "cd .. && docker build -t localhost:5001/ghcr.io_nvidia_nvsentinel_log-collector:latest -f log-collector/Dockerfile . && docker push localhost:5001/ghcr.io_nvidia_nvsentinel_log-collector:latest",
+    deps=["./"],
+    ignore=["./Tiltfile"],
+    labels=["images"]
 )

log-collector/entrypoint.sh

@@ -30,6 +30,28 @@ MUST_GATHER_SCRIPT_URL="${MUST_GATHER_SCRIPT_URL:-https://raw.githubusercontent.
 ENABLE_GCP_SOS_COLLECTION="${ENABLE_GCP_SOS_COLLECTION:-false}"
 ENABLE_AWS_SOS_COLLECTION="${ENABLE_AWS_SOS_COLLECTION:-false}"
 
+# Mock mode for testing - prepends mock scripts to PATH
+MOCK_MODE="${MOCK_MODE:-false}"
+MOCK_EXIT_CODE="${MOCK_EXIT_CODE:-0}"
+
+if [ "${MOCK_MODE}" = "true" ]; then
+  echo "[MOCK] Enabling mock mode - using mock nvidia-bug-report.sh and must-gather.sh"
+  MOCK_SCRIPTS_DIR="/opt/log-collector"
+  
+  # Copy mock scripts and make them executable
+  cp "${MOCK_SCRIPTS_DIR}/mock-nvidia-bug-report.sh" "${MOCK_SCRIPTS_DIR}/nvidia-bug-report.sh"
+  cp "${MOCK_SCRIPTS_DIR}/mock-must-gather.sh" "${MOCK_SCRIPTS_DIR}/must-gather.sh"
+  chmod +x "${MOCK_SCRIPTS_DIR}/nvidia-bug-report.sh" "${MOCK_SCRIPTS_DIR}/must-gather.sh"
+  
+  # Prepend to PATH so mocks are used instead of real tools
+  export PATH="${MOCK_SCRIPTS_DIR}:${PATH}"
+  
+  # Override MUST_GATHER_SCRIPT_URL to use local mock instead of downloading
+  MUST_GATHER_SCRIPT_URL="file://${MOCK_SCRIPTS_DIR}/must-gather.sh"
+  
+  echo "[MOCK] Mock mode enabled. nvidia-bug-report.sh and must-gather.sh will use mock versions."
+fi
+
 mkdir -p "${ARTIFACTS_DIR}"
 echo "[INFO] Target node: ${NODE_NAME} | GPU Operator namespace: ${GPU_OPERATOR_NAMESPACE} | Driver container: ${DRIVER_CONTAINER_NAME}"
 
@@ -77,8 +99,16 @@ AWS_SOS_REPORT=""
 GCP_NVIDIA_BUG_REPORT="/host/home/kubernetes/bin/nvidia/bin/nvidia-bug-report.sh"
 
 # 1) Collect nvidia-bug-report - auto-detect approach
+# In mock mode, use the local mock script directly
+if [ "${MOCK_MODE}" = "true" ]; then
+  echo "[MOCK] Using local mock nvidia-bug-report.sh"
+  BUG_REPORT_LOCAL_BASE="${ARTIFACTS_DIR}/nvidia-bug-report-${NODE_NAME}-${TIMESTAMP}"
+  BUG_REPORT_LOCAL="${BUG_REPORT_LOCAL_BASE}.log.gz"
+  nvidia-bug-report.sh --output-file "${BUG_REPORT_LOCAL_BASE}.log"
+  echo "[MOCK] Bug report saved to ${BUG_REPORT_LOCAL}"
+
 # Check if GCP COS nvidia-bug-report exists on the host filesystem (accessed via privileged container)
-if [ -f "${GCP_NVIDIA_BUG_REPORT}" ]; then
+elif [ -f "${GCP_NVIDIA_BUG_REPORT}" ]; then
   echo "[INFO] Found nvidia-bug-report at GCP COS location: ${GCP_NVIDIA_BUG_REPORT}"
 
   # Use GCP COS approach - write directly to container filesystem

log-collector/mock-must-gather.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Mock GPU Operator must-gather for testing
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+
+set -e
+
+echo "[MOCK] GPU Operator must-gather called"
+echo "[MOCK] Collecting mock diagnostic data..."
+
+# Create mock must-gather directory structure
+mkdir -p ./namespaces ./logs ./cluster-info
+
+# Generate simple mock files
+cat > ./cluster-info/info.txt <<EOF
+Mock GPU Operator Must-Gather
+Generated: $(date)
+Node: ${NODE_NAME:-unknown}
+Mock Mode: Enabled
+EOF
+
+echo "Mock pod logs - $(date)" > ./logs/mock-pod.log
+echo "Mock namespace data" > ./namespaces/mock-ns.yaml
+
+echo "[MOCK] Must-gather complete"
+exit 0
+

log-collector/mock-nvidia-bug-report.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Mock nvidia-bug-report.sh for testing
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+
+set -e
+
+echo "[MOCK] nvidia-bug-report.sh called"
+
+# Parse arguments to find output file
+OUTPUT_FILE=""
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --output-file)
+      OUTPUT_FILE="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+
+if [ -z "$OUTPUT_FILE" ]; then
+  echo "[MOCK ERROR] No output file specified" >&2
+  exit 1
+fi
+
+echo "[MOCK] Generating mock nvidia-bug-report to: ${OUTPUT_FILE}"
+
+# Create a simple mock bug report and gzip it
+# Real nvidia-bug-report.sh automatically appends .gz to the output file
+# We mimic this behavior
+echo "Mock NVIDIA Bug Report - Node: ${NODE_NAME:-unknown}, Date: $(date)" | gzip > "${OUTPUT_FILE}.gz"
+
+echo "[MOCK] Mock nvidia-bug-report created successfully at ${OUTPUT_FILE}.gz"
+exit 0
+