Skip to content

Commit 49f792f

Browse files
rupalis-nvrupalis-nv
committed
feat: Implement log collector mock mode and KWOK failure injection for testing
1 parent 3f3c256 commit 49f792f

File tree

15 files changed

+542
-73
lines changed

15 files changed

+542
-73
lines changed

‎distros/kubernetes/nvsentinel/charts/fault-remediation/files/log-collector-job.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ spec:
3636
{{- end }}
3737
containers:
3838
- name: log-collector
39-
image: {{ .Values.logCollector.image.repository }}:{{ .Values.global.image.tag | default "latest" }}
39+
image: {{ .Values.logCollector.image.repository }}:{{ .Values.logCollector.image.tag | default .Values.global.image.tag | default "latest" }}
4040
imagePullPolicy: {{ .Values.logCollector.image.pullPolicy }}
4141
securityContext:
4242
privileged: true
@@ -61,6 +61,10 @@ spec:
6161
value: {{ .Values.logCollector.enableGcpSosCollection | quote }}
6262
- name: ENABLE_AWS_SOS_COLLECTION
6363
value: {{ .Values.logCollector.enableAwsSosCollection | quote }}
64+
{{- range $key, $value := .Values.logCollector.env }}
65+
- name: {{ $key }}
66+
value: {{ $value | quote }}
67+
{{- end }}
6468
volumeMounts:
6569
- name: artifacts
6670
mountPath: /artifacts

‎distros/kubernetes/nvsentinel/charts/fault-remediation/values.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ logCollector:
9696
enabled: false
9797
image:
9898
repository: ghcr.io/nvidia/nvsentinel/log-collector
99+
# tag: latest # Optional: Override global.image.tag for log-collector
99100
pullPolicy: IfNotPresent
100101
# HTTP endpoint where collected logs will be uploaded
101102
uploadURL: "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local/upload"

‎distros/kubernetes/nvsentinel/kwok-stages/log-collector-failure-stage.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
##
2+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
##
16+
17+
# KWOK Stage to inject failure into log-collector jobs on KWOK nodes
18+
# This stage makes jobs with label 'test-scenario=log-collector-failure' fail
19+
apiVersion: kwok.x-k8s.io/v1alpha1
20+
kind: Stage
21+
metadata:
22+
name: job-log-collector-failure
23+
spec:
24+
resourceRef:
25+
apiGroup: batch
26+
kind: Job
27+
selector:
28+
matchLabels:
29+
test-scenario: log-collector-failure
30+
delay:
31+
durationMilliseconds: 1000
32+
next:
33+
statusTemplate: |
34+
conditions:
35+
- type: Failed
36+
status: "True"
37+
reason: BackoffLimitExceeded
38+
message: "Job has reached the specified backoff limit"
39+
lastProbeTime: {{ now }}
40+
lastTransitionTime: {{ now }}
41+
failed: 1
42+
startTime: {{ now }}
43+
completionTime: {{ now }}
44+

‎distros/kubernetes/nvsentinel/values-tilt.yaml‎

100644100755
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,17 @@ fault-quarantine:
121121
fault-remediation:
122122
logLevel: debug
123123

124+
logCollector:
125+
enabled: true
126+
image:
127+
repository: localhost:5001/ghcr.io_nvidia_nvsentinel_log-collector
128+
tag: latest # Use latest tag built by Tilt
129+
pullPolicy: Always # Always pull latest Tilt-built image
130+
env:
131+
MOCK_MODE: "false"
132+
MOCK_EXIT_CODE: "0"
133+
MOCK_SLEEP_DURATION: "2"
134+
124135
affinity:
125136
podAntiAffinity:
126137
requiredDuringSchedulingIgnoredDuringExecution:

‎fault-remediation/pkg/reconciler/remediation.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,19 @@ func (c *FaultRemediationClient) RunLogCollectorJob(ctx context.Context, nodeNam
300300
// Set target node
301301
job.Spec.Template.Spec.NodeName = nodeName
302302

303+
// Check for test-scenario annotation on the node and propagate to job label
304+
node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
305+
if err != nil {
306+
log.Printf("Warning: failed to get node %s for annotation check: %v", nodeName, err)
307+
} else if testScenario, ok := node.Annotations["nvsentinel.nvidia.com/test-scenario"]; ok {
308+
if job.Labels == nil {
309+
job.Labels = make(map[string]string)
310+
}
311+
312+
job.Labels["test-scenario"] = testScenario
313+
log.Printf("Applied test-scenario label '%s' to log collector job for node %s", testScenario, nodeName)
314+
}
315+
303316
// Create Job using typed client
304317
created, err := c.kubeClient.BatchV1().Jobs(job.Namespace).Create(ctx, job, metav1.CreateOptions{})
305318
if err != nil {

‎fault-remediation/pkg/reconciler/remediation_test.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,70 @@ func TestLogCollectorJobErrorHandling(t *testing.T) {
438438
}
439439
}
440440

441+
func TestLogCollectorJob_TestScenarioLabelPropagation(t *testing.T) {
442+
tests := []struct {
443+
name string
444+
nodeName string
445+
nodeAnnotations map[string]string
446+
expectedJobLabels map[string]string
447+
description string
448+
}{
449+
{
450+
name: "Node with test-scenario annotation",
451+
nodeName: "test-node-with-annotation",
452+
nodeAnnotations: map[string]string{
453+
"nvsentinel.nvidia.com/test-scenario": "log-collector-failure",
454+
},
455+
expectedJobLabels: map[string]string{
456+
"test-scenario": "log-collector-failure",
457+
},
458+
description: "Should propagate test-scenario annotation from node to job label",
459+
},
460+
{
461+
name: "Node without test-scenario annotation",
462+
nodeName: "test-node-without-annotation",
463+
nodeAnnotations: map[string]string{},
464+
expectedJobLabels: nil,
465+
description: "Should not add test-scenario label when node has no annotation",
466+
},
467+
{
468+
name: "Node with other annotations",
469+
nodeName: "test-node-with-other-annotations",
470+
nodeAnnotations: map[string]string{
471+
"some-other-annotation": "some-value",
472+
"another-annotation": "another-value",
473+
},
474+
expectedJobLabels: nil,
475+
description: "Should not add test-scenario label when node has other annotations",
476+
},
477+
}
478+
479+
for _, tt := range tests {
480+
t.Run(tt.name, func(t *testing.T) {
481+
// This test validates the label propagation logic conceptually
482+
// The actual implementation reads the node and adds labels to the job
483+
// Since we can't easily test the full RunLogCollectorJob flow (requires manifest file),
484+
// this test documents the expected behavior
485+
486+
// Expected behavior:
487+
// 1. Controller reads node annotations
488+
// 2. If "nvsentinel.nvidia.com/test-scenario" exists, propagate to job label "test-scenario"
489+
// 3. Job labels are used by KWOK Stage for failure injection
490+
491+
if tt.expectedJobLabels != nil {
492+
// When node has test-scenario annotation, job should have the label
493+
assert.NotNil(t, tt.expectedJobLabels, tt.description)
494+
assert.Equal(t, tt.nodeAnnotations["nvsentinel.nvidia.com/test-scenario"],
495+
tt.expectedJobLabels["test-scenario"],
496+
"Job label should match node annotation value")
497+
} else {
498+
// When node doesn't have test-scenario annotation, job shouldn't have the label
499+
assert.Nil(t, tt.expectedJobLabels, tt.description)
500+
}
501+
})
502+
}
503+
}
504+
441505
func TestRunLogCollectorJobDryRun(t *testing.T) {
442506
// Create a fake Kubernetes client
443507
fakeClient := fake.NewSimpleClientset()

‎log-collector/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ RUN useradd -u 10001 -m nvsentinel
4242
WORKDIR /opt/log-collector
4343

4444
COPY log-collector/entrypoint.sh /opt/log-collector/entrypoint.sh
45-
RUN chmod +x /opt/log-collector/entrypoint.sh
45+
COPY log-collector/mock-nvidia-bug-report.sh /opt/log-collector/mock-nvidia-bug-report.sh
46+
COPY log-collector/mock-must-gather.sh /opt/log-collector/mock-must-gather.sh
47+
RUN chmod +x /opt/log-collector/entrypoint.sh /opt/log-collector/mock-nvidia-bug-report.sh /opt/log-collector/mock-must-gather.sh
4648

4749
ENV PATH="/opt/log-collector:${PATH}"
4850

‎log-collector/Tiltfile‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,13 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
docker_build(
16-
"ghcr.io/nvidia/nvsentinel/log-collector",
17-
context=".",
18-
dockerfile="./Dockerfile"
15+
# Build docker image and push to local registry with fixed tag
16+
# Using local_resource since the image is used in dynamically created jobs
17+
# Build from repo root since Dockerfile expects that context
18+
local_resource(
19+
"log-collector",
20+
"cd .. && docker build -t localhost:5001/ghcr.io_nvidia_nvsentinel_log-collector:latest -f log-collector/Dockerfile . && docker push localhost:5001/ghcr.io_nvidia_nvsentinel_log-collector:latest",
21+
deps=["./"],
22+
ignore=["./Tiltfile"],
23+
labels=["images"]
1924
)

‎log-collector/entrypoint.sh‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,28 @@ MUST_GATHER_SCRIPT_URL="${MUST_GATHER_SCRIPT_URL:-https://raw.githubusercontent.
3030
ENABLE_GCP_SOS_COLLECTION="${ENABLE_GCP_SOS_COLLECTION:-false}"
3131
ENABLE_AWS_SOS_COLLECTION="${ENABLE_AWS_SOS_COLLECTION:-false}"
3232

33+
# Mock mode for testing - prepends mock scripts to PATH
34+
MOCK_MODE="${MOCK_MODE:-false}"
35+
MOCK_EXIT_CODE="${MOCK_EXIT_CODE:-0}"
36+
37+
if [ "${MOCK_MODE}" = "true" ]; then
38+
echo "[MOCK] Enabling mock mode - using mock nvidia-bug-report.sh and must-gather.sh"
39+
MOCK_SCRIPTS_DIR="/opt/log-collector"
40+
41+
# Copy mock scripts and make them executable
42+
cp "${MOCK_SCRIPTS_DIR}/mock-nvidia-bug-report.sh" "${MOCK_SCRIPTS_DIR}/nvidia-bug-report.sh"
43+
cp "${MOCK_SCRIPTS_DIR}/mock-must-gather.sh" "${MOCK_SCRIPTS_DIR}/must-gather.sh"
44+
chmod +x "${MOCK_SCRIPTS_DIR}/nvidia-bug-report.sh" "${MOCK_SCRIPTS_DIR}/must-gather.sh"
45+
46+
# Prepend to PATH so mocks are used instead of real tools
47+
export PATH="${MOCK_SCRIPTS_DIR}:${PATH}"
48+
49+
# Override MUST_GATHER_SCRIPT_URL to use local mock instead of downloading
50+
MUST_GATHER_SCRIPT_URL="file://${MOCK_SCRIPTS_DIR}/must-gather.sh"
51+
52+
echo "[MOCK] Mock mode enabled. nvidia-bug-report.sh and must-gather.sh will use mock versions."
53+
fi
54+
3355
mkdir -p "${ARTIFACTS_DIR}"
3456
echo "[INFO] Target node: ${NODE_NAME} | GPU Operator namespace: ${GPU_OPERATOR_NAMESPACE} | Driver container: ${DRIVER_CONTAINER_NAME}"
3557

@@ -77,15 +99,20 @@ AWS_SOS_REPORT=""
7799
GCP_NVIDIA_BUG_REPORT="/host/home/kubernetes/bin/nvidia/bin/nvidia-bug-report.sh"
78100

79101
# 1) Collect nvidia-bug-report - auto-detect approach
102+
BUG_REPORT_LOCAL_BASE="${ARTIFACTS_DIR}/nvidia-bug-report-${NODE_NAME}-${TIMESTAMP}"
103+
BUG_REPORT_LOCAL="${BUG_REPORT_LOCAL_BASE}.log.gz"
104+
105+
# In mock mode, use the local mock script directly
106+
if [ "${MOCK_MODE}" = "true" ]; then
107+
echo "[MOCK] Using local mock nvidia-bug-report.sh"
108+
nvidia-bug-report.sh --output-file "${BUG_REPORT_LOCAL_BASE}.log"
109+
echo "[MOCK] Bug report saved to ${BUG_REPORT_LOCAL}"
110+
80111
# Check if GCP COS nvidia-bug-report exists on the host filesystem (accessed via privileged container)
81-
if [ -f "${GCP_NVIDIA_BUG_REPORT}" ]; then
112+
elif [ -f "${GCP_NVIDIA_BUG_REPORT}" ]; then
82113
echo "[INFO] Found nvidia-bug-report at GCP COS location: ${GCP_NVIDIA_BUG_REPORT}"
83114

84115
# Use GCP COS approach - write directly to container filesystem
85-
BUG_REPORT_LOCAL_BASE="${ARTIFACTS_DIR}/nvidia-bug-report-${NODE_NAME}-${TIMESTAMP}"
86-
BUG_REPORT_LOCAL="${BUG_REPORT_LOCAL_BASE}.log.gz"
87-
88-
# Run nvidia-bug-report and output directly to artifacts directory
89116
"${GCP_NVIDIA_BUG_REPORT}" --output-file "${BUG_REPORT_LOCAL_BASE}.log"
90117
echo "[INFO] Bug report saved to ${BUG_REPORT_LOCAL}"
91118

@@ -104,14 +131,12 @@ else
104131

105132
# Collect bug report from driver container
106133
BUG_REPORT_REMOTE_BASE="/var/tmp/nvidia-bug-report-${NODE_NAME}-${TIMESTAMP}"
107-
BUG_REPORT_LOCAL_BASE="${ARTIFACTS_DIR}/nvidia-bug-report-${NODE_NAME}-${TIMESTAMP}"
108134
BUG_REPORT_REMOTE_PATH="${BUG_REPORT_REMOTE_BASE}.log.gz"
109135

110136
kubectl -n "${GPU_OPERATOR_NAMESPACE}" exec -c "${DRIVER_CONTAINER_NAME}" "${DRIVER_POD_NAME}" -- \
111137
nvidia-bug-report.sh --output-file "${BUG_REPORT_REMOTE_BASE}.log"
112138

113139
# Copy the bug report with retry
114-
BUG_REPORT_LOCAL="${BUG_REPORT_LOCAL_BASE}.log.gz"
115140
if ! kubectl -n "${GPU_OPERATOR_NAMESPACE}" cp "${DRIVER_POD_NAME}:${BUG_REPORT_REMOTE_PATH}" "${BUG_REPORT_LOCAL}"; then
116141
sleep 2
117142
kubectl -n "${GPU_OPERATOR_NAMESPACE}" cp "${DRIVER_POD_NAME}:${BUG_REPORT_REMOTE_PATH}" "${BUG_REPORT_LOCAL}"

‎log-collector/mock-must-gather.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
# Mock GPU Operator must-gather for testing
3+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
4+
5+
set -e
6+
7+
echo "[MOCK] GPU Operator must-gather called"
8+
echo "[MOCK] Collecting mock diagnostic data..."
9+
10+
# Create mock must-gather directory structure
11+
mkdir -p ./namespaces ./logs ./cluster-info
12+
13+
# Generate simple mock files
14+
cat > ./cluster-info/info.txt <<EOF
15+
Mock GPU Operator Must-Gather
16+
Generated: $(date)
17+
Node: ${NODE_NAME:-unknown}
18+
Mock Mode: Enabled
19+
EOF
20+
21+
echo "Mock pod logs - $(date)" > ./logs/mock-pod.log
22+
echo "Mock namespace data" > ./namespaces/mock-ns.yaml
23+
24+
echo "[MOCK] Must-gather complete"
25+
exit 0
26+

0 commit comments

Comments
 (0)