fix: some stuffs

Signed-off-by: Ajay Mishra <[email protected]>

Author: XRFXLP

distros/kubernetes/nvsentinel/charts/kubernetes-object-monitor/templates/deployment.yaml

@@ -48,31 +48,31 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default ((.Values.global).image).tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
-            - "--config"
-            - "/config/config.toml"
-            - "--platform-connector-socket"
-            - "{{ .Values.platformConnector.socket }}"
-            - "--metrics-port"
-            - "{{ (.Values.global).metricsPort | default 8080 }}"
-            - "--max-concurrent-reconciles"
-            - "{{ .Values.maxConcurrentReconciles }}"
+            - "--policy-config-path=/config/config.toml"
+            - "--platform-connector-socket=unix://{{ ((.Values.global).socketPath) | default "/var/run/nvsentinel.sock" }}"
+            - "--metrics-bind-address=:{{ ((.Values.global).metricsPort) | default 2112 }}"
+            - "--health-probe-bind-address=:8081"
+            - "--max-concurrent-reconciles={{ .Values.maxConcurrentReconciles }}"
+            - "--resync-period={{ .Values.resyncPeriod }}"
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           ports:
             - name: metrics
-              containerPort: {{ (.Values.global).metricsPort | default 8080 }}
+              containerPort: {{ ((.Values.global).metricsPort) | default 2112 }}
+            - name: health
+              containerPort: 8081
           livenessProbe:
             httpGet:
               path: /healthz
-              port: metrics
+              port: health
             initialDelaySeconds: 15
             periodSeconds: 20
             timeoutSeconds: 5
             failureThreshold: 3
           readinessProbe:
             httpGet:
-              path: /healthz
-              port: metrics
+              path: /readyz
+              port: health
             initialDelaySeconds: 5
             periodSeconds: 10
             timeoutSeconds: 3
@@ -84,8 +84,8 @@ spec:
             - name: config
               mountPath: /config
               readOnly: true
-            - name: platform-connector-socket
-              mountPath: /var/run/platform-connector
+            - name: socket
+              mountPath: /var/run
       volumes:
         - name: config
           configMap:

distros/kubernetes/nvsentinel/charts/kubernetes-object-monitor/values.yaml

@@ -28,9 +28,7 @@ image:
 podAnnotations: {}
 
 maxConcurrentReconciles: 1
-
-platformConnector:
-  socket: "unix:///var/run/platform-connector/platform-connector.sock"
+resyncPeriod: 5m
 
 policies:
   - name: node-not-ready
@@ -59,8 +57,8 @@ resources:
     memory: 256Mi
 
 volumes:
-  - name: platform-connector-socket
+  - name: socket
     hostPath:
-      path: /var/run/platform-connector
+      path: /var/run/nvsentinel
       type: DirectoryOrCreate
 

distros/kubernetes/nvsentinel/values-tilt.yaml

@@ -184,6 +184,24 @@ platformConnector:
 kubernetes-object-monitor:
   logLevel: debug
 
+  policies:
+    - name: node-test-condition
+      enabled: true
+      resource:
+        group: ""
+        version: v1
+        kind: Node
+      predicate:
+        expression: |
+          resource.status.conditions.filter(c, c.type == "TestCondition" && c.status == "False").size() > 0
+      healthEvent:
+        componentClass: Node
+        isFatal: false
+        message: "Node test condition is not ready"
+        recommendedAction: CONTACT_SUPPORT
+        errorCode:
+          - NODE_TEST_CONDITION_NOT_READY
+
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:

health-monitors/kubernetes-object-monitor/Tiltfile

@@ -0,0 +1,20 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker_build(
+    "ghcr.io/nvidia/nvsentinel/kubernetes-object-monitor",
+    context="../..",
+    dockerfile="./Dockerfile"
+)
+

health-monitors/kubernetes-object-monitor/pkg/initializer/initializer.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-logr/logr"
 	pb "github.com/nvidia/nvsentinel/data-models/pkg/protos"
 	"github.com/nvidia/nvsentinel/health-monitors/kubernetes-object-monitor/pkg/annotations"
 	celenv "github.com/nvidia/nvsentinel/health-monitors/kubernetes-object-monitor/pkg/cel"
@@ -38,6 +39,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlcontroller "sigs.k8s.io/controller-runtime/pkg/controller"
+	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
 
@@ -59,6 +61,10 @@ type Components struct {
 }
 
 func InitializeAll(ctx context.Context, params Params) (*Components, error) {
+	slogHandler := slog.Default().Handler()
+	logrLogger := logr.FromSlogHandler(slogHandler)
+	ctrllog.SetLogger(logrLogger)
+
 	cfg, err := config.Load(params.PolicyConfigPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load policy config: %w", err)

tests/helpers/kube.go

@@ -1594,10 +1594,13 @@ func SetNodeConditionStatus(
 				return err
 			}
 
+			found := false
 			modified := false
 
 			for i := range node.Status.Conditions {
 				if node.Status.Conditions[i].Type == conditionType {
+					found = true
+
 					if node.Status.Conditions[i].Status != status {
 						node.Status.Conditions[i].Status = status
 						node.Status.Conditions[i].LastTransitionTime = metav1.Now()
@@ -1609,6 +1612,19 @@ func SetNodeConditionStatus(
 				}
 			}
 
+			if !found {
+				now := metav1.Now()
+				node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{
+					Type:               conditionType,
+					Status:             status,
+					LastTransitionTime: now,
+					LastHeartbeatTime:  now,
+					Reason:             "TestCondition",
+					Message:            "Set by test",
+				})
+				modified = true
+			}
+
 			if !modified {
 				return nil
 			}

tests/kubernetes_object_monitor_test.go

@@ -30,6 +30,9 @@ type k8sObjectMonitorContextKey int
 
 const (
 	k8sMonitorKeyNodeName k8sObjectMonitorContextKey = iota
+
+	annotationKey     = "nvsentinel.nvidia.com/k8s-object-monitor-policy-matches"
+	testConditionType = "TestCondition"
 )
 
 func TestKubernetesObjectMonitor(t *testing.T) {
@@ -41,12 +44,19 @@ func TestKubernetesObjectMonitor(t *testing.T) {
 		client, err := c.NewClient()
 		require.NoError(t, err)
 
-		pod, err := helpers.GetPodOnWorkerNode(ctx, t, client, helpers.NVSentinelNamespace, "kubernetes-object-monitor")
+		nodeList := &v1.NodeList{}
+		err = client.Resources().List(ctx, nodeList)
 		require.NoError(t, err)
-		require.NotNil(t, pod)
 
-		testNodeName := pod.Spec.NodeName
-		t.Logf("Using kubernetes-object-monitor pod: %s on node: %s", pod.Name, testNodeName)
+		var testNodeName string
+		for _, node := range nodeList.Items {
+			if node.Labels["type"] != "kwok" {
+				testNodeName = node.Name
+				break
+			}
+		}
+		require.NotEmpty(t, testNodeName, "no real (non-KWOK) nodes found in cluster")
+		t.Logf("Using test node: %s", testNodeName)
 
 		return context.WithValue(ctx, k8sMonitorKeyNodeName, testNodeName)
 	})
@@ -56,9 +66,9 @@ func TestKubernetesObjectMonitor(t *testing.T) {
 		require.NoError(t, err)
 
 		nodeName := ctx.Value(k8sMonitorKeyNodeName).(string)
-		t.Logf("Marking node %s as NotReady", nodeName)
+		t.Logf("Setting TestCondition to False on node %s", nodeName)
 
-		helpers.SetNodeConditionStatus(ctx, t, client, nodeName, v1.NodeReady, v1.ConditionFalse)
+		helpers.SetNodeConditionStatus(ctx, t, client, nodeName, v1.NodeConditionType(testConditionType), v1.ConditionFalse)
 
 		t.Log("Waiting for policy match annotation on node")
 		require.Eventually(t, func() bool {
@@ -68,7 +78,7 @@ func TestKubernetesObjectMonitor(t *testing.T) {
 				return false
 			}
 
-			annotation, exists := node.Annotations["nvsentinel.nvidia.com/k8s-object-monitor-policy-matches"]
+			annotation, exists := node.Annotations[annotationKey]
 			if !exists {
 				return false
 			}
@@ -85,9 +95,9 @@ func TestKubernetesObjectMonitor(t *testing.T) {
 		require.NoError(t, err)
 
 		nodeName := ctx.Value(k8sMonitorKeyNodeName).(string)
-		t.Logf("Marking node %s as Ready", nodeName)
+		t.Logf("Setting TestCondition to True on node %s", nodeName)
 
-		helpers.SetNodeConditionStatus(ctx, t, client, nodeName, v1.NodeReady, v1.ConditionTrue)
+		helpers.SetNodeConditionStatus(ctx, t, client, nodeName, v1.NodeConditionType(testConditionType), v1.ConditionTrue)
 
 		t.Log("Waiting for policy match annotation to be cleared")
 		require.Eventually(t, func() bool {
@@ -97,7 +107,7 @@ func TestKubernetesObjectMonitor(t *testing.T) {
 				return false
 			}
 
-			annotation, exists := node.Annotations["nvsentinel.nvidia.com/k8s-object-monitor-policy-matches"]
+			annotation, exists := node.Annotations[annotationKey]
 			if exists && annotation != "" {
 				t.Logf("Annotation still exists: %s", annotation)
 				return false

tilt/Tiltfile

@@ -96,6 +96,7 @@ include('../platform-connectors/Tiltfile')
 include('./simple-health-client/Tiltfile')
 include('../health-events-analyzer/Tiltfile')
 include('../health-monitors/gpu-health-monitor/Tiltfile')
+include('../health-monitors/kubernetes-object-monitor/Tiltfile')
 include('../health-monitors/syslog-health-monitor/Tiltfile')
 include('../labeler/Tiltfile')
 
@@ -160,6 +161,11 @@ k8s_resource(
     resource_deps=['platform-connectors']
 )
 
+k8s_resource(
+    'kubernetes-object-monitor',
+    resource_deps=['platform-connectors']
+)
+
 k8s_resource(
     'syslog-health-monitor-regular',
     resource_deps=['platform-connectors']