feat: Add mock mode for log collector with Tilt tests for HIPPO-2120

Author: rupalis-nv

distros/kubernetes/nvsentinel/charts/fault-remediation/files/log-collector-job.yaml

@@ -61,6 +61,12 @@ spec:
               value: {{ .Values.logCollector.enableGcpSosCollection | quote }}
             - name: ENABLE_AWS_SOS_COLLECTION
               value: {{ .Values.logCollector.enableAwsSosCollection | quote }}
+            - name: MOCK_MODE
+              value: {{ .Values.logCollector.mockMode | quote }}
+            - name: MOCK_EXIT_CODE
+              value: {{ .Values.logCollector.mockExitCode | quote }}
+            - name: MOCK_SLEEP_DURATION
+              value: {{ .Values.logCollector.mockSleepDuration | quote }}
           volumeMounts:
             - name: artifacts
               mountPath: /artifacts

distros/kubernetes/nvsentinel/charts/fault-remediation/values.yaml

@@ -105,3 +105,10 @@ logCollector:
   enableGcpSosCollection: false
   # Enable AWS-specific SOS report collection
   enableAwsSosCollection: false
+  # Mock mode for testing
+  # Values: "true" (enable mock mode), "false" (disable mock mode)
+  mockMode: false
+  # Exit code to use in mock mode (0 for success, non-zero for failure scenarios)
+  mockExitCode: 0
+  # Sleep duration in seconds during mock mode to simulate collection time
+  mockSleepDuration: 5

distros/kubernetes/nvsentinel/values-tilt.yaml

@@ -121,6 +121,12 @@ fault-quarantine:
 fault-remediation:
   logLevel: debug
 
+  logCollector:
+    enabled: true
+    mockMode: "true"
+    mockExitCode: "0"
+    mockSleepDuration: "5"
+
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:

fault-remediation/pkg/reconciler/remediation.go

@@ -270,6 +270,38 @@ func (c *FaultRemediationClient) handleCreateCRError(
 	return false, ""
 }
 
+// updateJobEnvVar updates an environment variable in the job spec.
+func updateJobEnvVar(job *batchv1.Job, envName, envValue string) {
+	for i := range job.Spec.Template.Spec.Containers {
+		for j := range job.Spec.Template.Spec.Containers[i].Env {
+			if job.Spec.Template.Spec.Containers[i].Env[j].Name == envName {
+				job.Spec.Template.Spec.Containers[i].Env[j].Value = envValue
+				return
+			}
+		}
+	}
+}
+
+// applyNodeAnnotationsToJob applies test overrides from node annotations to the job spec.
+func (c *FaultRemediationClient) applyNodeAnnotationsToJob(ctx context.Context, job *batchv1.Job, nodeName string) {
+	node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+	if err != nil || node.Annotations == nil {
+		return
+	}
+
+	// Override MOCK_EXIT_CODE from annotation (for test scenarios)
+	if exitCodeStr, ok := node.Annotations["nvsentinel.nvidia.com/log-collector-mock-exit-code"]; ok {
+		log.Printf("Overriding log collector mock exit code from node annotation: %s", exitCodeStr)
+		updateJobEnvVar(job, "MOCK_EXIT_CODE", exitCodeStr)
+	}
+
+	// Override MOCK_SLEEP_DURATION from annotation (for test scenarios)
+	if sleepDurationStr, ok := node.Annotations["nvsentinel.nvidia.com/log-collector-mock-sleep"]; ok {
+		log.Printf("Overriding log collector mock sleep duration from node annotation: %s", sleepDurationStr)
+		updateJobEnvVar(job, "MOCK_SLEEP_DURATION", sleepDurationStr)
+	}
+}
+
 // RunLogCollectorJob creates a log collector Job and waits for completion.
 // nolint: cyclop // todo
 func (c *FaultRemediationClient) RunLogCollectorJob(ctx context.Context, nodeName string) error {
@@ -297,6 +329,9 @@ func (c *FaultRemediationClient) RunLogCollectorJob(ctx context.Context, nodeNam
 		return fmt.Errorf("failed to unmarshal Job manifest: %w", err)
 	}
 
+	// Apply test overrides from node annotations (for test scenarios)
+	c.applyNodeAnnotationsToJob(ctx, job, nodeName)
+
 	// Set target node
 	job.Spec.Template.Spec.NodeName = nodeName
 
@@ -311,7 +346,7 @@ func (c *FaultRemediationClient) RunLogCollectorJob(ctx context.Context, nodeNam
 	log.Printf("Waiting for log collector job %s to complete", created.Name)
 
 	// Use a context with timeout for the watch
-	watchCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	watchCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer cancel()
 
 	// Use SharedInformerFactory for efficient job status monitoring with filtering

log-collector/Dockerfile

@@ -44,6 +44,11 @@ WORKDIR /opt/log-collector
 COPY log-collector/entrypoint.sh /opt/log-collector/entrypoint.sh
 RUN chmod +x /opt/log-collector/entrypoint.sh
 
+# Copy mock scripts for testing
+COPY log-collector/mock-nvidia-bug-report.sh /mock-nvidia-bug-report.sh
+COPY log-collector/mock-must-gather.sh /mock-must-gather.sh
+RUN chmod +x /mock-nvidia-bug-report.sh /mock-must-gather.sh
+
 ENV PATH="/opt/log-collector:${PATH}"
 
 ENTRYPOINT ["/opt/log-collector/entrypoint.sh"]

log-collector/entrypoint.sh

@@ -30,6 +30,32 @@ MUST_GATHER_SCRIPT_URL="${MUST_GATHER_SCRIPT_URL:-https://raw.githubusercontent.
 ENABLE_GCP_SOS_COLLECTION="${ENABLE_GCP_SOS_COLLECTION:-false}"
 ENABLE_AWS_SOS_COLLECTION="${ENABLE_AWS_SOS_COLLECTION:-false}"
 
+# Mock mode for testing
+MOCK_MODE="${MOCK_MODE:-false}"
+MOCK_EXIT_CODE="${MOCK_EXIT_CODE:-0}"
+MOCK_SLEEP_DURATION="${MOCK_SLEEP_DURATION:-5}"
+
+# If mock mode is enabled, set up mock tools directory and prepend to PATH
+# This approach allows the real production flow to run while using fake tools
+if [ "${MOCK_MODE}" = "true" ]; then
+    echo "[INFO] Mock mode enabled - setting up mock tools"
+    
+    MOCK_TOOLS_DIR="/tmp/mock-tools"
+    mkdir -p "${MOCK_TOOLS_DIR}"
+    
+    # Symlink mock nvidia-bug-report.sh
+    if [ -f "/mock-nvidia-bug-report.sh" ]; then
+        ln -sf /mock-nvidia-bug-report.sh "${MOCK_TOOLS_DIR}/nvidia-bug-report.sh"
+        chmod +x "${MOCK_TOOLS_DIR}/nvidia-bug-report.sh"
+    fi
+    
+    # Prepend mock tools directory to PATH so our mocks are used first
+    export PATH="${MOCK_TOOLS_DIR}:${PATH}"
+    
+    echo "[INFO] Mock tools setup complete. PATH: ${PATH}"
+    echo "[INFO] Mock exit code will be: ${MOCK_EXIT_CODE}"
+fi
+
 mkdir -p "${ARTIFACTS_DIR}"
 echo "[INFO] Target node: ${NODE_NAME} | GPU Operator namespace: ${GPU_OPERATOR_NAMESPACE} | Driver container: ${DRIVER_CONTAINER_NAME}"
 
@@ -197,8 +223,17 @@ fi
 GPU_MG_DIR="${ARTIFACTS_DIR}/gpu-operator-must-gather"
 mkdir -p "${GPU_MG_DIR}"
 echo "[INFO] Running GPU Operator must-gather..."
-curl -fsSL "${MUST_GATHER_SCRIPT_URL}" -o "${GPU_MG_DIR}/must-gather.sh"
-chmod +x "${GPU_MG_DIR}/must-gather.sh"
+
+# In mock mode, use the mock script directly; otherwise download from URL
+if [ "${MOCK_MODE}" = "true" ] && [ -f "/mock-must-gather.sh" ]; then
+    cp /mock-must-gather.sh "${GPU_MG_DIR}/must-gather.sh"
+    chmod +x "${GPU_MG_DIR}/must-gather.sh"
+    echo "[INFO] Using mock must-gather script"
+else
+    curl -fsSL "${MUST_GATHER_SCRIPT_URL}" -o "${GPU_MG_DIR}/must-gather.sh"
+    chmod +x "${GPU_MG_DIR}/must-gather.sh"
+fi
+
 bash "${GPU_MG_DIR}/must-gather.sh"
 
 GPU_MG_TARBALL="${ARTIFACTS_DIR}/gpu-operator-must-gather-${NODE_NAME}-${TIMESTAMP}.tar.gz"
@@ -250,4 +285,10 @@ if [ -n "${UPLOAD_URL_BASE:-}" ]; then
   fi
 fi
 
-echo "[INFO] Done. Artifacts under ${ARTIFACTS_DIR}"
+echo "[INFO] Done. Artifacts under ${ARTIFACTS_DIR}"
+
+# If mock mode is enabled, exit with the configured mock exit code
+if [ "${MOCK_MODE}" = "true" ]; then
+    echo "[INFO] Mock mode: Exiting with code ${MOCK_EXIT_CODE}"
+    exit "${MOCK_EXIT_CODE}"
+fi

log-collector/mock-must-gather.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Mock must-gather.sh script for testing
+# This script simulates the behavior of GPU Operator must-gather without requiring actual GPU Operator
+
+set -e
+
+OUTPUT_DIR="${PWD}"
+OUTPUT_FILE="must-gather.tar.gz"
+
+# Parse command-line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -o|--output)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+echo "[MOCK] Generating GPU Operator must-gather"
+
+# Simulate collection time
+if [ -n "${MOCK_SLEEP_DURATION:-}" ] && [ "${MOCK_SLEEP_DURATION}" -gt 0 ]; then
+    sleep "${MOCK_SLEEP_DURATION}"
+fi
+
+# Create mock must-gather content
+TEMP_DIR=$(mktemp -d)
+mkdir -p "${TEMP_DIR}/gpu-operator-must-gather"
+
+cat > "${TEMP_DIR}/gpu-operator-must-gather/summary.txt" <<EOF
+Mock GPU Operator Must-Gather Report
+=====================================
+Generated: $(date)
+Node: ${NODE_NAME:-unknown}
+Timestamp: ${TIMESTAMP:-$(date +%s)}
+Namespace: ${GPU_OPERATOR_NAMESPACE:-gpu-operator}
+
+GPU Operator Pods:
+------------------
+- nvidia-driver-daemonset-xxxxx (Mock)
+- nvidia-device-plugin-daemonset-xxxxx (Mock)
+- gpu-feature-discovery-xxxxx (Mock)
+
+GPU Operator Version: v23.9.0 (Mock)
+
+Mock must-gather completed successfully
+EOF
+
+# Create mock pod logs
+mkdir -p "${TEMP_DIR}/gpu-operator-must-gather/logs"
+echo "Mock driver pod logs - Node: ${NODE_NAME:-unknown}" > "${TEMP_DIR}/gpu-operator-must-gather/logs/nvidia-driver.log"
+echo "Mock device plugin logs - Node: ${NODE_NAME:-unknown}" > "${TEMP_DIR}/gpu-operator-must-gather/logs/device-plugin.log"
+
+# Create tarball
+cd "${TEMP_DIR}"
+tar czf "${OUTPUT_FILE}" gpu-operator-must-gather/
+
+# Move to output directory
+mv "${OUTPUT_FILE}" "${OUTPUT_DIR}/"
+
+# Cleanup
+cd - > /dev/null
+rm -rf "${TEMP_DIR}"
+
+echo "[MOCK] GPU Operator must-gather completed: ${OUTPUT_DIR}/${OUTPUT_FILE}"
+exit 0
+

log-collector/mock-nvidia-bug-report.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Mock nvidia-bug-report.sh script for testing
+# This script simulates the behavior of nvidia-bug-report.sh without requiring actual NVIDIA drivers
+
+set -e
+
+OUTPUT_FILE="nvidia-bug-report.log.gz"
+
+# Parse command-line arguments to find output file
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --output-file)
+            OUTPUT_FILE="$2"
+            shift 2
+            ;;
+        --output-file=*)
+            OUTPUT_FILE="${1#*=}"
+            shift
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+echo "[MOCK] Generating nvidia-bug-report at ${OUTPUT_FILE}"
+
+# Simulate collection time
+if [ -n "${MOCK_SLEEP_DURATION:-}" ] && [ "${MOCK_SLEEP_DURATION}" -gt 0 ]; then
+    sleep "${MOCK_SLEEP_DURATION}"
+fi
+
+# Create mock content with realistic structure
+cat > /tmp/mock-nvidia-bug-report.log <<EOF
+Mock NVIDIA Bug Report
+============================
+Generated: $(date)
+Node: ${NODE_NAME:-unknown}
+Timestamp: ${TIMESTAMP:-$(date +%s)}
+
+System Information:
+-------------------
+Hostname: ${NODE_NAME:-mock-node}
+Kernel: $(uname -r)
+OS: Mock Linux Distribution
+
+NVIDIA Driver Information:
+--------------------------
+Driver Version: 550.54.15 (Mock)
+CUDA Version: 12.4 (Mock)
+
+GPU Information:
+----------------
+GPU 0: NVIDIA A100-SXM4-80GB (Mock)
+  UUID: GPU-12345678-1234-1234-1234-123456789012
+  Bus ID: 0000:00:04.0
+
+Mock nvidia-bug-report generated successfully
+EOF
+
+# Compress the mock report
+gzip -c /tmp/mock-nvidia-bug-report.log > "${OUTPUT_FILE}"
+rm -f /tmp/mock-nvidia-bug-report.log
+
+echo "[MOCK] nvidia-bug-report completed successfully"
+exit 0
+

tests/helpers/kube.go

@@ -1398,6 +1398,24 @@ func PortForwardPod(
 	return stopChan, readyChan
 }
 
+// GetFileServerPodName returns the name of the file-server pod in the NVSentinel namespace
+func GetFileServerPodName(ctx context.Context, client klient.Client) (string, error) {
+	podList := &v1.PodList{}
+
+	err := client.Resources(NVSentinelNamespace).List(ctx, podList)
+	if err != nil {
+		return "", fmt.Errorf("failed to list pods: %w", err)
+	}
+
+	for _, pod := range podList.Items {
+		if strings.Contains(pod.Name, "file-server") && IsPodReady(pod) {
+			return pod.Name, nil
+		}
+	}
+
+	return "", fmt.Errorf("file-server pod not found")
+}
+
 // WaitForNodeConditionWithCheckName waits for the node to have a condition with the reason as checkName.
 func WaitForNodeConditionWithCheckName(
 	ctx context.Context, t *testing.T, c klient.Client, nodeName, checkName, message string,