feat: integrate syslog health monitor with metadata-collector (#288)

Signed-off-by: Ajay Mishra <[email protected]>

Author: XRFXLP

distros/kubernetes/nvsentinel/charts/metadata-collector/templates/daemonset.yaml

@@ -50,7 +50,7 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default ((.Values.global).image).tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
-            - --output-path={{ .Values.outputPath }}
+            - --output-path={{ .Values.global.metadataPath }}
           env:
             - name: NODE_NAME
               valueFrom:

distros/kubernetes/nvsentinel/charts/metadata-collector/values.yaml

@@ -31,5 +31,3 @@ resources:
     cpu: 100m
     memory: 128Mi
 
-outputPath: /var/lib/nvsentinel/gpu_metadata.json
-

distros/kubernetes/nvsentinel/charts/syslog-health-monitor/templates/_helpers.tpl

@@ -98,6 +98,8 @@ spec:
             {{- end }}
             - "--checks"
             - "{{ join "," $root.Values.enabledChecks }}"
+            - "--metadata-path"
+            - "{{ $root.Values.global.metadataPath }}"
           resources:
             {{- toYaml $root.Values.resources | nindent 12 }}
           ports:
@@ -130,6 +132,9 @@ spec:
               mountPath: /var/run/
             - name: syslog-state-vol
               mountPath: /var/run/syslog_health_monitor
+            - name: metadata-vol
+              mountPath: /var/lib/nvsentinel
+              readOnly: true
             {{- if $kataMode }}
             # Kata mode: Mount systemd journal for accessing host logs
             - name: host-journal
@@ -181,6 +186,10 @@ spec:
           hostPath:
             path: /var/run/syslog_health_monitor
             type: DirectoryOrCreate
+        - name: metadata-vol
+          hostPath:
+            path: /var/lib/nvsentinel
+            type: DirectoryOrCreate
         {{- if $kataMode }}
         # Kata mode: Systemd journal volumes for host log access
         - name: host-journal

distros/kubernetes/nvsentinel/values.yaml

@@ -18,6 +18,9 @@ global:
     tag: "main"
   metricsPort: 2112
 
+  # Shared metadata path used by metadata-collector and syslog-health-monitor
+  metadataPath: /var/lib/nvsentinel/gpu_metadata.json
+
   nodeSelector: {}
   tolerations: []
   affinity: {}

health-monitors/syslog-health-monitor/Dockerfile

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG CUDA_VERSION="12.1.1-base-ubuntu22.04"
-
 FROM public.ecr.aws/docker/library/golang:1.25-trixie AS builder
 
 ARG BUILD_TAGS="systemd"
@@ -47,16 +45,17 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 RUN cd health-monitors/syslog-health-monitor && \
     CGO_ENABLED=1 go build -tags "${BUILD_TAGS}" -ldflags="-s -w" -o syslog-health-monitor main.go
 
-FROM nvcr.io/nvidia/cuda:${CUDA_VERSION} AS runtime
+# Runtime stage - using Debian slim for systemd journal support
+FROM public.ecr.aws/docker/library/debian:bookworm-slim AS runtime
 
-RUN apt-get update && apt-get install -y \
-    systemd-sysv \
-    --no-install-recommends \
+# Install required system libraries for systemd journal
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libsystemd0 \
+    liblz4-1 \
+    libzstd1 \
+    ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /go/src/nvsentinel/health-monitors/syslog-health-monitor/syslog-health-monitor /app/syslog-health-monitor
 
-RUN apt-get update && \
-    apt-get install -y libsystemd0 liblz4-1 libzstd1
-
 ENTRYPOINT ["/app/syslog-health-monitor"] 

health-monitors/syslog-health-monitor/main.go

@@ -65,6 +65,8 @@ var (
 		"Endpoint to the XID analyser service.")
 	kataEnabled = flag.String("kata-enabled", "false",
 		"Indicates if this monitor is running in Kata Containers mode (set by DaemonSet variant).")
+	metadataPath = flag.String("metadata-path", "/var/lib/nvsentinel/gpu_metadata.json",
+		"Path to GPU metadata JSON file.")
 )
 
 var checks []fd.CheckDefinition
@@ -166,6 +168,7 @@ func run() error {
 		*pollingIntervalFlag,
 		*stateFileFlag,
 		*xidAnalyserEndpoint,
+		*metadataPath,
 	)
 	if err != nil {
 		return fmt.Errorf("error creating syslog health monitor: %w", err)

health-monitors/syslog-health-monitor/pkg/metadata/reader.go

@@ -0,0 +1,169 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadata
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/nvidia/nvsentinel/data-models/pkg/model"
+)
+
+type Reader struct {
+	path string
+
+	once    sync.Once
+	loadErr error
+
+	metadata *model.GPUMetadata
+
+	pciToGPU      map[string]*model.GPUInfo
+	nvswitchLinks map[string]map[int]*gpuLinkInfo
+}
+
+type gpuLinkInfo struct {
+	GPU         *model.GPUInfo
+	LocalLinkID int
+}
+
+func NewReader(path string) *Reader {
+	return &Reader{
+		path: path,
+	}
+}
+
+func (r *Reader) ensureLoaded() error {
+	r.once.Do(func() {
+		r.loadErr = r.load()
+	})
+
+	return r.loadErr
+}
+
+func (r *Reader) load() error {
+	data, err := os.ReadFile(r.path)
+	if err != nil {
+		return fmt.Errorf("failed to read metadata file: %w", err)
+	}
+
+	var metadata model.GPUMetadata
+	if err := json.Unmarshal(data, &metadata); err != nil {
+		return fmt.Errorf("failed to parse metadata JSON: %w", err)
+	}
+
+	r.metadata = &metadata
+	r.buildMaps()
+
+	slog.Info("GPU metadata loaded",
+		"gpus", len(metadata.GPUs),
+		"nvswitches", len(metadata.NVSwitches),
+		"chassis_serial", metadata.ChassisSerial != nil)
+
+	return nil
+}
+
+func (r *Reader) buildMaps() {
+	r.pciToGPU = make(map[string]*model.GPUInfo)
+	r.nvswitchLinks = make(map[string]map[int]*gpuLinkInfo)
+
+	for i := range r.metadata.GPUs {
+		gpu := &r.metadata.GPUs[i]
+		normPCI := normalizePCI(gpu.PCIAddress)
+		r.pciToGPU[normPCI] = gpu
+
+		for _, link := range gpu.NVLinks {
+			remotePCI := normalizePCI(link.RemotePCIAddress)
+
+			if r.nvswitchLinks[remotePCI] == nil {
+				r.nvswitchLinks[remotePCI] = make(map[int]*gpuLinkInfo)
+			}
+
+			r.nvswitchLinks[remotePCI][link.RemoteLinkID] = &gpuLinkInfo{
+				GPU:         gpu,
+				LocalLinkID: link.LinkID,
+			}
+		}
+	}
+}
+
+func (r *Reader) GetGPUByPCI(pci string) (*model.GPUInfo, error) {
+	if err := r.ensureLoaded(); err != nil {
+		return nil, fmt.Errorf("failed to load metadata for PCI lookup %s: %w", pci, err)
+	}
+
+	normPCI := normalizePCI(pci)
+	gpu, ok := r.pciToGPU[normPCI]
+
+	if !ok {
+		return nil, fmt.Errorf("GPU not found for PCI address: %s", pci)
+	}
+
+	return gpu, nil
+}
+
+func (r *Reader) GetGPUByNVSwitchLink(nvswitchPCI string, linkID int) (*model.GPUInfo, int, error) {
+	if err := r.ensureLoaded(); err != nil {
+		return nil, -1, fmt.Errorf("failed to load metadata for NVSwitch lookup %s link %d: %w", nvswitchPCI, linkID, err)
+	}
+
+	normPCI := normalizePCI(nvswitchPCI)
+	links, ok := r.nvswitchLinks[normPCI]
+
+	if !ok {
+		return nil, -1, fmt.Errorf("NVSwitch not found: %s", nvswitchPCI)
+	}
+
+	info, ok := links[linkID]
+
+	if !ok {
+		return nil, -1, fmt.Errorf("link %d not found on NVSwitch %s", linkID, nvswitchPCI)
+	}
+
+	return info.GPU, info.LocalLinkID, nil
+}
+
+func (r *Reader) GetChassisSerial() *string {
+	if err := r.ensureLoaded(); err != nil {
+		return nil
+	}
+
+	return r.metadata.ChassisSerial
+}
+
+func normalizePCI(pci string) string {
+	parts := strings.Split(pci, ":")
+	if len(parts) != 3 {
+		return strings.ToLower(pci)
+	}
+
+	domain := parts[0]
+	if len(domain) > 4 {
+		domain = domain[len(domain)-4:]
+	}
+
+	busDeviceFunc := parts[2]
+	if idx := strings.Index(busDeviceFunc, "."); idx != -1 {
+		busDeviceFunc = busDeviceFunc[:idx]
+	}
+
+	return fmt.Sprintf("%s:%s:%s",
+		strings.ToLower(domain),
+		strings.ToLower(parts[1]),
+		strings.ToLower(busDeviceFunc))
+}