only persist key info

Author: t0mmylam

operator/api/v1alpha1/deployment_policy_types.go

@@ -239,17 +239,13 @@ type BatchProcessingState struct {
 	CurrentBatch int `json:"currentBatch,omitempty"`
 	// Number of consecutive failures
 	ConsecutiveFailures int `json:"consecutiveFailures,omitempty"`
-	// Total number of nodes processed so far
-	ProcessedNodes int `json:"processedNodes,omitempty"`
-	// Number of successful nodes in current batch
-	SuccessfulInBatch int `json:"successfulInBatch,omitempty"`
-	// Number of failed nodes in current batch
-	FailedInBatch int `json:"failedInBatch,omitempty"`
+	// Total number of nodes that have completed successfully (cumulative across all batches)
+	CompletedNodes int `json:"completedNodes,omitempty"`
+	// Total number of nodes that have failed (cumulative across all batches)
+	FailedNodes int `json:"failedNodes,omitempty"`
 	// Whether the strategy should stop processing due to failures
 	ShouldStop bool `json:"shouldStop,omitempty"`
-	// Names of nodes in the current batch (persisted across reconciles)
-	CurrentBatchNodes []string `json:"currentBatchNodes,omitempty"`
-	// Last successful batch size (for slowdown calculations)
+	// Last batch size (for slowdown calculations)
 	LastBatchSize int `json:"lastBatchSize,omitempty"`
 	// Whether the last batch failed (for slowdown logic)
 	LastBatchFailed bool `json:"lastBatchFailed,omitempty"`
@@ -271,19 +267,22 @@ func (s *DeploymentStrategy) CalculateBatchSize(totalNodes int, state *BatchProc
 
 // EvaluateBatchResult evaluates the result of a batch and updates state
 func (s *DeploymentStrategy) EvaluateBatchResult(state *BatchProcessingState, batchSize int, successCount int, failureCount int, totalNodes int) {
-	state.SuccessfulInBatch = successCount
-	state.FailedInBatch = failureCount
-	state.ProcessedNodes += batchSize
+	// Note: successCount and failureCount are deltas from the current batch
+	// CompletedNodes and FailedNodes are already updated in EvaluateCurrentBatch before this is called
 
 	// Avoid divide by zero
 	if batchSize == 0 {
 		return
 	}
 
+	// Calculate success percentage for this batch
 	successPercentage := (successCount * 100) / batchSize
+
+	// Calculate overall progress percentage
+	processedNodes := state.CompletedNodes + state.FailedNodes
 	var progressPercent int
 	if totalNodes > 0 {
-		progressPercent = (state.ProcessedNodes * 100) / totalNodes
+		progressPercent = (processedNodes * 100) / totalNodes
 	}
 
 	if successPercentage >= s.getBatchThreshold() {
@@ -346,7 +345,8 @@ func (s *DeploymentStrategy) getFailureThreshold() int {
 func (s *FixedStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessingState) int {
 	// Fixed strategy doesn't change batch size, but respects remaining nodes
 	batchSize := *s.InitialBatch
-	remaining := totalNodes - state.ProcessedNodes
+	processedNodes := state.CompletedNodes + state.FailedNodes
+	remaining := totalNodes - processedNodes
 	if batchSize > remaining {
 		batchSize = remaining
 	}
@@ -362,7 +362,8 @@ func (s *LinearStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessi
 	}
 
 	// Check if we should slow down due to last batch failure
-	progressPercent := (state.ProcessedNodes * 100) / totalNodes
+	processedNodes := state.CompletedNodes + state.FailedNodes
+	progressPercent := (processedNodes * 100) / totalNodes
 	if state.LastBatchFailed && progressPercent < *s.SafetyLimit && state.LastBatchSize > 0 {
 		// Slow down: reduce by delta from last batch size
 		batchSize = max(1, state.LastBatchSize-*s.Delta)
@@ -371,7 +372,7 @@ func (s *LinearStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessi
 		batchSize = *s.InitialBatch + (state.CurrentBatch-1)*(*s.Delta)
 	}
 
-	remaining := totalNodes - state.ProcessedNodes
+	remaining := totalNodes - processedNodes
 	if batchSize > remaining {
 		batchSize = remaining
 	}
@@ -387,7 +388,8 @@ func (s *ExponentialStrategy) CalculateBatchSize(totalNodes int, state *BatchPro
 	}
 
 	// Check if we should slow down due to last batch failure
-	progressPercent := (state.ProcessedNodes * 100) / totalNodes
+	processedNodes := state.CompletedNodes + state.FailedNodes
+	progressPercent := (processedNodes * 100) / totalNodes
 	if state.LastBatchFailed && progressPercent < *s.SafetyLimit && state.LastBatchSize > 0 && *s.GrowthFactor > 0 {
 		// Slow down: divide last batch size by growth factor
 		batchSize = max(1, state.LastBatchSize / *s.GrowthFactor)
@@ -405,7 +407,7 @@ func (s *ExponentialStrategy) CalculateBatchSize(totalNodes int, state *BatchPro
 		}
 	}
 
-	remaining := totalNodes - state.ProcessedNodes
+	remaining := totalNodes - processedNodes
 	if batchSize > remaining {
 		batchSize = remaining
 	}

operator/api/v1alpha1/zz_generated.deepcopy.go

@@ -504,37 +504,30 @@ spec:
                   description: BatchProcessingState tracks the current state of batch
                     processing for a compartment
                   properties:
+                    completedNodes:
+                      description: Total number of nodes that have completed successfully
+                        (cumulative across all batches)
+                      type: integer
                     consecutiveFailures:
                       description: Number of consecutive failures
                       type: integer
                     currentBatch:
                       description: Current batch number (starts at 1)
                       type: integer
-                    currentBatchNodes:
-                      description: Names of nodes in the current batch (persisted
-                        across reconciles)
-                      items:
-                        type: string
-                      type: array
-                    failedInBatch:
-                      description: Number of failed nodes in current batch
+                    failedNodes:
+                      description: Total number of nodes that have failed (cumulative
+                        across all batches)
                       type: integer
                     lastBatchFailed:
                       description: Whether the last batch failed (for slowdown logic)
                       type: boolean
                     lastBatchSize:
-                      description: Last successful batch size (for slowdown calculations)
-                      type: integer
-                    processedNodes:
-                      description: Total number of nodes processed so far
+                      description: Last batch size (for slowdown calculations)
                       type: integer
                     shouldStop:
                       description: Whether the strategy should stop processing due
                         to failures
                       type: boolean
-                    successfulInBatch:
-                      description: Number of successful nodes in current batch
-                      type: integer
                   type: object
                 description: CompartmentBatchStates tracks batch processing state
                   per compartment

operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml

@@ -479,9 +479,11 @@ func PersistCompartmentBatchStates(skyhook SkyhookNodes) bool {
 
 	changed := false
 	for _, compartment := range compartments {
-		// Only persist if there are nodes in the current batch
-		if len(compartment.GetBatchState().CurrentBatchNodes) > 0 {
-			skyhook.GetSkyhook().Status.CompartmentBatchStates[compartment.GetName()] = compartment.GetBatchState()
+		// Always persist batch state to maintain cumulative counters
+		batchState := compartment.GetBatchState()
+		// Only persist if there's meaningful state (batch has started or there are nodes)
+		if batchState.CurrentBatch > 0 || len(compartment.GetNodes()) > 0 {
+			skyhook.GetSkyhook().Status.CompartmentBatchStates[compartment.GetName()] = batchState
 			changed = true
 		}
 	}

operator/internal/controller/cluster_state_v2.go

@@ -100,24 +100,23 @@ func (c *Compartment) GetNodesForNextBatch() []SkyhookNode {
 		return nil
 	}
 
-	if len(c.BatchState.CurrentBatchNodes) > 0 {
-		return c.getCurrentBatchNodes()
+	// If there's a batch in progress (nodes are InProgress), don't start a new one
+	if c.getInProgressCount() > 0 {
+		return c.getInProgressNodes()
 	}
 
+	// No batch in progress, create a new one
 	return c.createNewBatch()
 }
 
-func (c *Compartment) getCurrentBatchNodes() []SkyhookNode {
-	currentBatchNodes := make([]SkyhookNode, 0)
-	for _, nodeName := range c.BatchState.CurrentBatchNodes {
-		for _, node := range c.Nodes {
-			if node.GetNode().Name == nodeName {
-				currentBatchNodes = append(currentBatchNodes, node)
-				break
-			}
+func (c *Compartment) getInProgressNodes() []SkyhookNode {
+	inProgressNodes := make([]SkyhookNode, 0)
+	for _, node := range c.Nodes {
+		if node.Status() == v1alpha1.StatusInProgress {
+			inProgressNodes = append(inProgressNodes, node)
 		}
 	}
-	return currentBatchNodes
+	return inProgressNodes
 }
 
 func (c *Compartment) createNewBatch() []SkyhookNode {
@@ -154,66 +153,54 @@ func (c *Compartment) createNewBatch() []SkyhookNode {
 		}
 	}
 
-	nodeNames := make([]string, len(selectedNodes))
-	for i, node := range selectedNodes {
-		nodeNames[i] = node.GetNode().Name
-	}
-	c.BatchState.CurrentBatchNodes = nodeNames
-
 	return selectedNodes
 }
 
 // IsBatchComplete checks if the current batch has reached terminal states
+// A batch is complete when there are no nodes in InProgress status
 func (c *Compartment) IsBatchComplete() bool {
-	if len(c.BatchState.CurrentBatchNodes) == 0 {
-		return true // No batch in progress
-	}
-
-	// Check if all batch nodes have reached terminal states
-	for _, nodeName := range c.BatchState.CurrentBatchNodes {
-		for _, node := range c.Nodes {
-			if node.GetNode().Name == nodeName {
-				if node.Status() == v1alpha1.StatusInProgress {
-					return false // Still processing
-				}
-				break
-			}
-		}
-	}
-	return true // All nodes are Complete or Erroring
+	return c.getInProgressCount() == 0
 }
 
 // EvaluateCurrentBatch evaluates the current batch result if it's complete
+// Uses delta-based tracking: compares current state to last checkpoint
 func (c *Compartment) EvaluateCurrentBatch() (bool, int, int) {
 	if !c.IsBatchComplete() {
 		return false, 0, 0 // Batch not complete yet
 	}
 
-	if len(c.BatchState.CurrentBatchNodes) == 0 {
-		return false, 0, 0 // No batch to evaluate
+	// If this is the first batch (nothing has been processed yet), skip evaluation
+	// The batch will be started in the next reconcile
+	if c.BatchState.CurrentBatch == 0 {
+		c.BatchState.CurrentBatch = 1
+		return false, 0, 0
 	}
 
-	successCount := 0
-	failureCount := 0
-
-	// Count successes and failures from the batch nodes
-	for _, nodeName := range c.BatchState.CurrentBatchNodes {
-		for _, node := range c.Nodes {
-			if node.GetNode().Name == nodeName {
-				if node.IsComplete() {
-					successCount++
-				} else if node.Status() == v1alpha1.StatusErroring {
-					failureCount++
-				}
-				break
-			}
+	// Count current state in the compartment
+	currentCompleted := 0
+	currentFailed := 0
+	for _, node := range c.Nodes {
+		if node.IsComplete() {
+			currentCompleted++
+		} else if node.Status() == v1alpha1.StatusErroring {
+			currentFailed++
 		}
 	}
 
-	// Clear the current batch since we're evaluating it
-	c.BatchState.CurrentBatchNodes = nil
+	// Calculate delta from last checkpoint
+	deltaCompleted := currentCompleted - c.BatchState.CompletedNodes
+	deltaFailed := currentFailed - c.BatchState.FailedNodes
+
+	// Only evaluate if there's actually a change (batch was processed)
+	if deltaCompleted == 0 && deltaFailed == 0 {
+		return false, 0, 0
+	}
+
+	// Update checkpoints
+	c.BatchState.CompletedNodes = currentCompleted
+	c.BatchState.FailedNodes = currentFailed
 
-	return true, successCount, failureCount
+	return true, deltaCompleted, deltaFailed
 }
 
 // EvaluateAndUpdateBatchState evaluates a completed batch and updates the persistent state
@@ -223,10 +210,8 @@ func (c *Compartment) EvaluateAndUpdateBatchState(batchSize int, successCount in
 		c.Strategy.EvaluateBatchResult(&c.BatchState, batchSize, successCount, failureCount, len(c.Nodes))
 	} else {
 		// No strategy: just update basic counters
-		c.BatchState.ProcessedNodes += batchSize
-		c.BatchState.SuccessfulInBatch = successCount
-		c.BatchState.FailedInBatch = failureCount
 		c.BatchState.CurrentBatch++
+		c.BatchState.LastBatchSize = batchSize
 	}
 }