[Feature]: Janitor cleanup (retries, backoff, csp pkg, logging) (#247)

Signed-off-by: Mark Chmarny <[email protected]>

Author: mchmarny

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,10 +11,11 @@
 - [ ] 🔨 Build/CI
 
 ## Component(s) Affected
-- [ ] Health Monitors
 - [ ] Core Services
-- [ ] Fault Management
 - [ ] Documentation/CI
+- [ ] Fault Management
+- [ ] Health Monitors
+- [ ] Janitor
 - [ ] Other: ____________
 
 ## Testing

.github/copilot-instructions.md

@@ -9,7 +9,7 @@ NVSentinel is a GPU Node Resilience System for Kubernetes that automatically det
 ## Architecture & Technologies
 
 ### Core Technologies
-- **Language**: Go 1.25+ (primary), Python 3.12+ (monitoring tools)
+- **Language**: Go 1.25+ (primary), Python 3.10+ (monitoring tools)
 - **Container Platform**: Kubernetes 1.25+
 - **Deployment**: Helm 3.0+, Tilt (development)
 - **Storage**: MongoDB (event store with change streams)

Makefile

@@ -665,7 +665,7 @@ tilt-ci: ## Run Tilt in CI mode (no UI, waits for all resources)
 		echo "Error: tilt is not installed. Please install from https://tilt.dev/"; \
 		exit 1; \
 	fi
-	tilt ci -f tilt/Tiltfile
+	tilt ci -f tilt/Tiltfile --timeout=10m
 	@echo "Waiting for all deployments to be ready..."
 	@kubectl get deployments --all-namespaces --no-headers -o custom-columns=":metadata.namespace,:metadata.name" | while read ns name; do \
 		echo "Waiting for deployment $$name in namespace $$ns..."; \

distros/kubernetes/nvsentinel/charts/janitor/crds/janitor.dgxc.nvidia.com_gpuresets.yaml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition

distros/kubernetes/nvsentinel/charts/janitor/crds/janitor.dgxc.nvidia.com_rebootnodes.yaml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -142,6 +141,18 @@ spec:
                   - type
                   type: object
                 type: array
+              consecutiveFailures:
+                description: |-
+                  ConsecutiveFailures tracks consecutive CSP operation failures for exponential backoff
+                  Reset to 0 on successful operations
+                format: int32
+                type: integer
+              retryCount:
+                description: |-
+                  RetryCount tracks the number of reconciliation attempts for this reboot operation
+                  Used to implement maximum retry limits to prevent indefinite reconciliation
+                format: int32
+                type: integer
               startTime:
                 description: StartTime is the time when the reboot was initiated
                 format: date-time

distros/kubernetes/nvsentinel/charts/janitor/crds/janitor.dgxc.nvidia.com_terminatenodes.yaml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -142,6 +141,18 @@ spec:
                   - type
                   type: object
                 type: array
+              consecutiveFailures:
+                description: |-
+                  ConsecutiveFailures tracks consecutive CSP operation failures for exponential backoff
+                  Reset to 0 on successful operations
+                format: int32
+                type: integer
+              retryCount:
+                description: |-
+                  RetryCount tracks the number of reconciliation attempts for this terminate operation
+                  Used to implement maximum retry limits to prevent indefinite reconciliation
+                format: int32
+                type: integer
               startTime:
                 description: StartTime is the time when the termination was initiated
                 format: date-time

distros/kubernetes/nvsentinel/charts/labeler/templates/clusterrole.yaml

@@ -34,5 +34,6 @@ rules:
     verbs:
       - get
       - list
+      - watch
       - patch
       - update

janitor/Makefile

@@ -60,6 +60,8 @@ generate: ## Generate CRDs and move them to Helm chart directory
 	@# Install controller-gen if not present
 	@which controller-gen > /dev/null || (echo "Installing controller-gen..." && go install sigs.k8s.io/controller-tools/cmd/controller-gen@latest)
 	@# Generate CRDs into api/v1alpha1 directory
+	@# Note: Generated CRD YAML files do not include license headers (this is expected)
+	@# YAML files are excluded from license header checks via main Makefile: -ignore '**/*.yaml'
 	@controller-gen crd paths=./$(API_DIR) output:crd:dir=./$(API_DIR)
 	@# Move generated CRDs to Helm chart crds directory
 	@echo "Moving generated CRDs to $(CRD_OUTPUT_DIR)..."

janitor/api/v1alpha1/gpureset_types.go

@@ -25,14 +25,14 @@ import (
 type GPUSelector struct {
 	// UUIDs is a list of GPU UUIDs.
 	// +optional
-	// nolint:lll
+	//nolint:lll // kubebuilder validation pattern
 	// +kubebuilder:validation:items:Pattern="^GPU-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
 	UUIDs []string `json:"uuids,omitempty"`
 
 	// PCIBusIDs is a list of GPU PCI bus IDs.
 	// Format: "domain:bus:device.function" (e.g., "0000:01:00.0").
 	// +optional
-	// nolint:lll
+	//nolint:lll // kubebuilder validation pattern
 	// +kubebuilder:validation:items:Pattern="^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\\.[0-9a-fA-F]{1}$"
 	PCIBusIDs []string `json:"pciBusIDs,omitempty"`
 }
@@ -71,7 +71,7 @@ const (
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:scope=Cluster
 // +kubebuilder:subresource:status
-// nolint:lll
+//nolint:lll // kubebuilder printcolumn marker
 // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".spec.nodeName",description="The target node for the GPU reset"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 

janitor/api/v1alpha1/rebootnode_types.go

@@ -49,6 +49,14 @@ type RebootNodeStatus struct {
 	// CompletionTime is the time when the reboot was completed
 	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
 
+	// RetryCount tracks the number of reconciliation attempts for this reboot operation
+	// Used to implement maximum retry limits to prevent indefinite reconciliation
+	RetryCount int32 `json:"retryCount,omitempty"`
+
+	// ConsecutiveFailures tracks consecutive CSP operation failures for exponential backoff
+	// Reset to 0 on successful operations
+	ConsecutiveFailures int32 `json:"consecutiveFailures,omitempty"`
+
 	// Conditions represent the latest available observations of an object's current state.
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 }
@@ -184,6 +192,33 @@ func (r *RebootNode) SetCompletionTime() {
 	}
 }
 
+// Interface implementation for generic status update handling
+
+// GetRetryCount returns the retry count
+func (s *RebootNodeStatus) GetRetryCount() int32 {
+	return s.RetryCount
+}
+
+// GetConsecutiveFailures returns the consecutive failures count
+func (s *RebootNodeStatus) GetConsecutiveFailures() int32 {
+	return s.ConsecutiveFailures
+}
+
+// GetStartTime returns the start time
+func (s *RebootNodeStatus) GetStartTime() *metav1.Time {
+	return s.StartTime
+}
+
+// GetCompletionTime returns the completion time
+func (s *RebootNodeStatus) GetCompletionTime() *metav1.Time {
+	return s.CompletionTime
+}
+
+// GetConditions returns the conditions
+func (s *RebootNodeStatus) GetConditions() []metav1.Condition {
+	return s.Conditions
+}
+
 func init() {
 	SchemeBuilder.Register(&RebootNode{}, &RebootNodeList{})
 }