feat: Integrate GPU health monitor with metadata collector (#290)

Signed-off-by: Ajay Mishra <[email protected]>

Author: XRFXLP

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-3.x.yaml

@@ -56,6 +56,8 @@ spec:
             - "/var/run/statefile"
             - --dcgm-k8s-service-enabled
             - {{ .Values.dcgm.dcgmK8sServiceEnabled | quote }}
+            - --metadata-path
+            - {{ .Values.global.metadataPath | quote }}
           securityContext:
             runAsUser: 0
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default ((.Values.global).image).tag | default .Chart.AppVersion }}-dcgm-3.x"
@@ -89,6 +91,9 @@ spec:
               mountPath: /etc/dcgmhealth
             - name: var-run-vol
               mountPath: /var/run/
+            - name: gpu-metadata
+              mountPath: /var/lib/nvsentinel
+              readOnly: true
           env: 
             - name: NODE_NAME
               valueFrom:
@@ -104,6 +109,10 @@ spec:
           hostPath:
             path: /var/run/nvsentinel
             type: Directory
+        - name: gpu-metadata
+          hostPath:
+            path: /var/lib/nvsentinel
+            type: DirectoryOrCreate
         {{- if .Values.additionalHostVolumes }}
         {{ toYaml .Values.additionalHostVolumes | nindent 8 }}
         {{- end }}

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-4.x.yaml

@@ -56,6 +56,8 @@ spec:
             - "/var/run/statefile"
             - --dcgm-k8s-service-enabled
             - {{ .Values.dcgm.dcgmK8sServiceEnabled | quote }}
+            - --metadata-path
+            - {{ .Values.global.metadataPath | quote }}
           securityContext:
             runAsUser: 0
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default ((.Values.global).image).tag | default .Chart.AppVersion }}-dcgm-4.x"
@@ -89,6 +91,9 @@ spec:
               mountPath: /etc/dcgmhealth
             - name: var-run-vol
               mountPath: /var/run/
+            - name: gpu-metadata
+              mountPath: /var/lib/nvsentinel
+              readOnly: true
           env: 
             - name: NODE_NAME
               valueFrom:
@@ -104,6 +109,10 @@ spec:
           hostPath:
             path: /var/run/nvsentinel
             type: Directory
+        - name: gpu-metadata
+          hostPath:
+            path: /var/lib/nvsentinel
+            type: DirectoryOrCreate
         {{- if .Values.additionalHostVolumes }}
         {{ toYaml .Values.additionalHostVolumes | nindent 8 }}
         {{- end }}

health-monitors/gpu-health-monitor/gpu_health_monitor/cli.py

@@ -31,6 +31,7 @@ def _init_event_processor(
     dcgm_errors_info_dict: dict[str, str],
     state_file_path: str,
     dcgm_health_conditions_categorization_mapping_config: dict[str, str],
+    metadata_path: str,
 ):
     platform_connector_config = config["eventprocessors.platformconnector"]
     match event_processor_name:
@@ -42,6 +43,7 @@ def _init_event_processor(
                 dcgm_errors_info_dict=dcgm_errors_info_dict,
                 state_file_path=state_file_path,
                 dcgm_health_conditions_categorization_mapping_config=dcgm_health_conditions_categorization_mapping_config,
+                metadata_path=metadata_path,
             )
         case _:
             log.fatal(f"Unknown event processor {event_processor_name}")
@@ -58,6 +60,13 @@ def _init_event_processor(
 @click.option("--verbose", type=bool, default=False, help="Enable debug logging", required=False)
 @click.option("--state-file", type=click.Path(), help="gpu health monitor state file path", required=True)
 @click.option("--dcgm-k8s-service-enabled", type=bool, help="Is DCGM K8s service Enabled", required=True)
+@click.option(
+    "--metadata-path",
+    type=click.Path(),
+    default="/var/lib/nvsentinel/gpu_metadata.json",
+    help="Path to GPU metadata JSON file",
+    required=False,
+)
 def cli(
     dcgm_addr,
     dcgm_error_mapping_config_file,
@@ -66,6 +75,7 @@ def cli(
     verbose,
     state_file,
     dcgm_k8s_service_enabled,
+    metadata_path,
 ):
     exit = Event()
     config = configparser.ConfigParser()
@@ -112,6 +122,7 @@ def cli(
                 dcgm_errors_info_dict,
                 state_file_path,
                 dcgm_health_conditions_categorization_mapping_config,
+                metadata_path,
             )
         )
 

health-monitors/gpu-health-monitor/gpu_health_monitor/metadata/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .reader import MetadataReader
+
+__all__ = ["MetadataReader"]

health-monitors/gpu-health-monitor/gpu_health_monitor/metadata/reader.py

@@ -0,0 +1,146 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging as log
+import threading
+from typing import Optional
+
+
+class MetadataReader:
+    """Lazy-loading, thread-safe GPU metadata reader.
+
+    This class reads GPU metadata from a JSON file and provides
+    thread-safe access to GPU UUID and chassis serial information.
+    The metadata is loaded lazily on first access.
+    """
+
+    def __init__(self, metadata_path: str):
+        """Initialize the metadata reader.
+
+        Args:
+            metadata_path: Path to the GPU metadata JSON file.
+        """
+        self._path = metadata_path
+        self._metadata = None
+        self._lock = threading.RLock()
+        self._loaded = False
+
+    def _ensure_loaded(self):
+        """Load metadata on first use (lazy loading).
+
+        This method uses double-checked locking to ensure thread-safe
+        lazy initialization of the metadata.
+        """
+        if self._loaded:
+            return
+
+        with self._lock:
+            if self._loaded:
+                return
+
+            try:
+                with open(self._path, "r") as f:
+                    self._metadata = json.load(f)
+                self._loaded = True
+                gpu_count = len(self._metadata.get("gpus", []))
+                chassis = self._metadata.get("chassis_serial")
+                log.info(
+                    f"GPU metadata loaded from {self._path}: "
+                    f"{gpu_count} GPUs, chassis_serial={'present' if chassis else 'absent'}"
+                )
+            except FileNotFoundError:
+                log.warning(f"Metadata file not found: {self._path}, continuing without metadata enrichment")
+                self._metadata = {}
+                self._loaded = True
+            except Exception as e:
+                # Handles JSON decode errors, permission errors, etc.
+                log.error(f"Error loading metadata from {self._path}: {e}")
+                self._metadata = {}
+                self._loaded = True
+
+    def get_gpu_uuid(self, gpu_id: int) -> Optional[str]:
+        """Get GPU UUID by DCGM GPU ID.
+
+        Args:
+            gpu_id: The DCGM GPU ID (0, 1, 2, ...).
+
+        Returns:
+            The GPU UUID string if found, None otherwise.
+        """
+        self._ensure_loaded()
+
+        if not self._metadata:
+            return None
+
+        gpus = self._metadata.get("gpus", [])
+        for gpu in gpus:
+            if gpu.get("gpu_id") == gpu_id:
+                uuid = gpu.get("uuid")
+                if uuid:
+                    log.debug(f"Found GPU UUID for GPU {gpu_id}: {uuid}")
+                    return uuid
+                else:
+                    log.warning(f"GPU {gpu_id} found in metadata but has no UUID")
+                    return None
+
+        log.debug(f"GPU {gpu_id} not found in metadata")
+        return None
+
+    def get_pci_address(self, gpu_id: int) -> Optional[str]:
+        """Get PCI address by DCGM GPU ID.
+
+        Args:
+            gpu_id: The DCGM GPU ID (0, 1, 2, ...).
+
+        Returns:
+            The PCI address string if found, None otherwise.
+        """
+        self._ensure_loaded()
+
+        if not self._metadata:
+            return None
+
+        gpus = self._metadata.get("gpus", [])
+        for gpu in gpus:
+            if gpu.get("gpu_id") == gpu_id:
+                pci_address = gpu.get("pci_address")
+                if pci_address:
+                    log.debug(f"Found PCI address for GPU {gpu_id}: {pci_address}")
+                    return pci_address
+                else:
+                    log.warning(f"GPU {gpu_id} found in metadata but has no PCI address")
+                    return None
+
+        log.debug(f"GPU {gpu_id} not found in metadata")
+        return None
+
+    def get_chassis_serial(self) -> Optional[str]:
+        """Get chassis serial number.
+
+        Returns:
+            The chassis serial number if available, None otherwise.
+        """
+        self._ensure_loaded()
+
+        if not self._metadata:
+            return None
+
+        chassis_serial = self._metadata.get("chassis_serial")
+        if chassis_serial:
+            log.debug(f"Found chassis serial: {chassis_serial}")
+        else:
+            log.debug("No chassis serial in metadata")
+
+        return chassis_serial