feat: make fault remediation use generic maintenance resources (#257)

Author: lalitadithya

distros/kubernetes/nvsentinel/charts/fault-remediation/templates/configmap.yaml

@@ -24,6 +24,8 @@ data:
     namespace = {{ .Values.maintenance.namespace | quote }}
     version = {{ .Values.maintenance.version | quote }}
     apiGroup = {{ .Values.maintenance.apiGroup | quote }}
+    kind = {{ .Values.maintenance.kind | quote }}
+    completeConditionType = {{ .Values.maintenance.completeConditionType | quote }}
     
     [template]
     mountPath = "/etc/config"

distros/kubernetes/nvsentinel/charts/fault-remediation/values.yaml

@@ -48,6 +48,14 @@ maintenance:
   apiGroup: "janitor.dgxc.nvidia.com"
   # API version of the maintenance CRD
   version: "v1alpha1"
+  # Kind of the maintenance CRD
+  kind: "RebootNode"
+  # completeConditionType specifies which status condition to check for completion of maintenance. 
+  # If multiple faults occur on the same node in close proximity, the condition is checked to
+  # prevent creation of duplicate maintenance requests for the same operation. 
+  # If the status is True, then it is implied that the maintenance is completed a new CR should be created
+  # If the status is False, then it is implied that the maintenance has failed a new CR can be created
+  completeConditionType: "NodeReady"
   # Kubernetes namespace where maintenance resources will be created
   namespace: "nvsentinel"
   # Names of maintenance resource types used by the janitor controller

fault-remediation/pkg/config/config.go

@@ -16,9 +16,11 @@ package config
 
 // MaintenanceResource holds configuration for the maintenance custom resource
 type MaintenanceResource struct {
-	Namespace string `toml:"namespace"`
-	Version   string `toml:"version"`
-	ApiGroup  string `toml:"apiGroup"`
+	Namespace             string `toml:"namespace"`
+	Version               string `toml:"version"`
+	ApiGroup              string `toml:"apiGroup"`
+	Kind                  string `toml:"kind"`
+	CompleteConditionType string `toml:"completeConditionType"`
 }
 
 // Template holds configuration for template files

fault-remediation/pkg/crstatus/checker.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crstatus
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/nvidia/nvsentinel/fault-remediation/pkg/config"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/restmapper"
+)
+
+type CRStatusChecker struct {
+	dynamicClient dynamic.Interface
+	restMapper    *restmapper.DeferredDiscoveryRESTMapper
+	config        *config.MaintenanceResource
+	dryRun        bool
+}
+
+func NewCRStatusChecker(
+	dynamicClient dynamic.Interface,
+	restMapper *restmapper.DeferredDiscoveryRESTMapper,
+	cfg *config.MaintenanceResource,
+	dryRun bool,
+) *CRStatusChecker {
+	return &CRStatusChecker{
+		dynamicClient: dynamicClient,
+		restMapper:    restMapper,
+		config:        cfg,
+		dryRun:        dryRun,
+	}
+}
+
+func (c *CRStatusChecker) ShouldSkipCRCreation(ctx context.Context, crName string) bool {
+	if c.dryRun {
+		slog.Info("DRY-RUN: CR doesn't exist (dry-run mode)", "crName", crName)
+		return false
+	}
+
+	gvk := schema.GroupVersionKind{
+		Group:   c.config.ApiGroup,
+		Version: c.config.Version,
+		Kind:    c.config.Kind,
+	}
+
+	mapping, err := c.restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
+	if err != nil {
+		slog.Error("Failed to get REST mapping", "gvk", gvk.String(), "error", err)
+		return false
+	}
+
+	resource, err := c.dynamicClient.Resource(mapping.Resource).Get(ctx, crName, metav1.GetOptions{})
+	if err != nil {
+		slog.Warn("Failed to get CR, allowing create", "crName", crName, "error", err)
+		return false
+	}
+
+	return c.checkCondition(resource)
+}
+
+func (c *CRStatusChecker) checkCondition(obj *unstructured.Unstructured) bool {
+	status, found, err := unstructured.NestedMap(obj.Object, "status")
+	if err != nil || !found {
+		return true
+	}
+
+	conditions, found, err := unstructured.NestedSlice(status, "conditions")
+	if err != nil || !found {
+		return true
+	}
+
+	conditionStatus := c.findConditionStatus(conditions)
+
+	return !c.isTerminal(conditionStatus)
+}
+
+func (c *CRStatusChecker) findConditionStatus(conditions []any) string {
+	for _, cond := range conditions {
+		condition, ok := cond.(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		condType, _ := condition["type"].(string)
+		if condType == c.config.CompleteConditionType {
+			condStatus, _ := condition["status"].(string)
+			return condStatus
+		}
+	}
+
+	return ""
+}
+
+func (c *CRStatusChecker) isTerminal(conditionStatus string) bool {
+	return conditionStatus == "True" || conditionStatus == "False"
+}