Skip to content

Commit 2bc96a4

Browse files
t0mmylamt0mmylam
committed
feat: implement deployment strategies with compartment-based batching
1 parent 299a07d commit 2bc96a4

File tree

8 files changed

+738
-8
lines changed

8 files changed

+738
-8
lines changed

‎chart/templates/skyhook-crd.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,40 @@ spec:
498498
status:
499499
description: SkyhookStatus defines the observed state of Skyhook
500500
properties:
501+
compartmentBatchStates:
502+
additionalProperties:
503+
description: BatchProcessingState tracks the current state of batch
504+
processing for a compartment
505+
properties:
506+
consecutiveFailures:
507+
description: Number of consecutive failures
508+
type: integer
509+
currentBatch:
510+
description: Current batch number (starts at 1)
511+
type: integer
512+
currentBatchNodes:
513+
description: Names of nodes in the current batch (persisted
514+
across reconciles)
515+
items:
516+
type: string
517+
type: array
518+
failedInBatch:
519+
description: Number of failed nodes in current batch
520+
type: integer
521+
processedNodes:
522+
description: Total number of nodes processed so far
523+
type: integer
524+
shouldStop:
525+
description: Whether the strategy should stop processing due
526+
to failures
527+
type: boolean
528+
successfulInBatch:
529+
description: Number of successful nodes in current batch
530+
type: integer
531+
type: object
532+
description: CompartmentBatchStates tracks batch processing state
533+
per compartment
534+
type: object
501535
completeNodes:
502536
default: 0/0
503537
description: |-

‎operator/api/v1alpha1/deployment_policy_types.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,131 @@ func (s *DeploymentStrategy) Validate() error {
232232
return nil
233233
}
234234

235+
// BatchProcessingState tracks the current state of batch processing for a compartment
236+
type BatchProcessingState struct {
237+
// Current batch number (starts at 1)
238+
CurrentBatch int `json:"currentBatch,omitempty"`
239+
// Number of consecutive failures
240+
ConsecutiveFailures int `json:"consecutiveFailures,omitempty"`
241+
// Total number of nodes processed so far
242+
ProcessedNodes int `json:"processedNodes,omitempty"`
243+
// Number of successful nodes in current batch
244+
SuccessfulInBatch int `json:"successfulInBatch,omitempty"`
245+
// Number of failed nodes in current batch
246+
FailedInBatch int `json:"failedInBatch,omitempty"`
247+
// Whether the strategy should stop processing due to failures
248+
ShouldStop bool `json:"shouldStop,omitempty"`
249+
// Names of nodes in the current batch (persisted across reconciles)
250+
CurrentBatchNodes []string `json:"currentBatchNodes,omitempty"`
251+
}
252+
253+
// CalculateBatchSize calculates the next batch size based on the strategy
254+
func (s *DeploymentStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessingState) int {
255+
switch {
256+
case s.Fixed != nil:
257+
return s.Fixed.CalculateBatchSize(totalNodes, state)
258+
case s.Linear != nil:
259+
return s.Linear.CalculateBatchSize(totalNodes, state)
260+
case s.Exponential != nil:
261+
return s.Exponential.CalculateBatchSize(totalNodes, state)
262+
default:
263+
return 1 // fallback
264+
}
265+
}
266+
267+
// EvaluateBatchResult evaluates the result of a batch and updates state
268+
func (s *DeploymentStrategy) EvaluateBatchResult(state *BatchProcessingState, batchSize int, successCount int, failureCount int, totalNodes int) {
269+
state.SuccessfulInBatch = successCount
270+
state.FailedInBatch = failureCount
271+
state.ProcessedNodes += batchSize
272+
273+
successPercentage := (successCount * 100) / batchSize
274+
progressPercent := (state.ProcessedNodes * 100) / totalNodes
275+
276+
if successPercentage >= s.getBatchThreshold() {
277+
state.ConsecutiveFailures = 0
278+
} else {
279+
state.ConsecutiveFailures++
280+
if progressPercent < s.getSafetyLimit() && state.ConsecutiveFailures >= s.getFailureThreshold() {
281+
state.ShouldStop = true
282+
}
283+
}
284+
285+
state.CurrentBatch++
286+
}
287+
288+
// getBatchThreshold returns the batch threshold from the active strategy
289+
func (s *DeploymentStrategy) getBatchThreshold() int {
290+
switch {
291+
case s.Fixed != nil:
292+
return *s.Fixed.BatchThreshold
293+
case s.Linear != nil:
294+
return *s.Linear.BatchThreshold
295+
case s.Exponential != nil:
296+
return *s.Exponential.BatchThreshold
297+
default:
298+
return 100
299+
}
300+
}
301+
302+
// getSafetyLimit returns the safety limit from the active strategy
303+
func (s *DeploymentStrategy) getSafetyLimit() int {
304+
switch {
305+
case s.Fixed != nil:
306+
return *s.Fixed.SafetyLimit
307+
case s.Linear != nil:
308+
return *s.Linear.SafetyLimit
309+
case s.Exponential != nil:
310+
return *s.Exponential.SafetyLimit
311+
default:
312+
return 50
313+
}
314+
}
315+
316+
// getFailureThreshold returns the failure threshold from the active strategy
317+
func (s *DeploymentStrategy) getFailureThreshold() int {
318+
switch {
319+
case s.Fixed != nil:
320+
return *s.Fixed.FailureThreshold
321+
case s.Linear != nil:
322+
return *s.Linear.FailureThreshold
323+
case s.Exponential != nil:
324+
return *s.Exponential.FailureThreshold
325+
default:
326+
return 3
327+
}
328+
}
329+
330+
func (s *FixedStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessingState) int {
331+
batchSize := *s.InitialBatch
332+
remaining := totalNodes - state.ProcessedNodes
333+
if batchSize > remaining {
334+
batchSize = remaining
335+
}
336+
return max(1, batchSize)
337+
}
338+
339+
func (s *LinearStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessingState) int {
340+
batchSize := *s.InitialBatch + (state.CurrentBatch-1)*(*s.Delta)
341+
remaining := totalNodes - state.ProcessedNodes
342+
if batchSize > remaining {
343+
batchSize = remaining
344+
}
345+
return max(1, batchSize)
346+
}
347+
348+
func (s *ExponentialStrategy) CalculateBatchSize(totalNodes int, state *BatchProcessingState) int {
349+
batchSize := *s.InitialBatch
350+
for i := 1; i < state.CurrentBatch; i++ {
351+
batchSize *= *s.GrowthFactor
352+
}
353+
remaining := totalNodes - state.ProcessedNodes
354+
if batchSize > remaining {
355+
batchSize = remaining
356+
}
357+
return max(1, batchSize)
358+
}
359+
235360
// Validate validates the Compartment
236361
func (c *Compartment) Validate() error {
237362
// Validate compartment budget

‎operator/api/v1alpha1/skyhook_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,9 @@ type SkyhookStatus struct {
316316
// ConfigUpdates tracks config updates
317317
ConfigUpdates map[string][]string `json:"configUpdates,omitempty"`
318318

319+
// CompartmentBatchStates tracks batch processing state per compartment
320+
CompartmentBatchStates map[string]BatchProcessingState `json:"compartmentBatchStates,omitempty"`
321+
319322
// +kubebuilder:example=3
320323
// +kubebuilder:default=0
321324
// NodesInProgress displays the number of nodes that are currently in progress and is

‎operator/api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 27 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,40 @@ spec:
499499
status:
500500
description: SkyhookStatus defines the observed state of Skyhook
501501
properties:
502+
compartmentBatchStates:
503+
additionalProperties:
504+
description: BatchProcessingState tracks the current state of batch
505+
processing for a compartment
506+
properties:
507+
consecutiveFailures:
508+
description: Number of consecutive failures
509+
type: integer
510+
currentBatch:
511+
description: Current batch number (starts at 1)
512+
type: integer
513+
currentBatchNodes:
514+
description: Names of nodes in the current batch (persisted
515+
across reconciles)
516+
items:
517+
type: string
518+
type: array
519+
failedInBatch:
520+
description: Number of failed nodes in current batch
521+
type: integer
522+
processedNodes:
523+
description: Total number of nodes processed so far
524+
type: integer
525+
shouldStop:
526+
description: Whether the strategy should stop processing due
527+
to failures
528+
type: boolean
529+
successfulInBatch:
530+
description: Number of successful nodes in current batch
531+
type: integer
532+
type: object
533+
description: CompartmentBatchStates tracks batch processing state
534+
per compartment
535+
type: object
502536
completeNodes:
503537
default: 0/0
504538
description: |-

0 commit comments

Comments
 (0)