Add CancelHealthCheckOnNewRevision feature to avoid getting stuck on failing commits

This feature allows health checks to be cancelled when a new source revision
becomes available, preventing the controller from getting stuck waiting for
full timeout durations when fixes are already available.

Features:
- New opt-in feature flag: CancelHealthCheckOnNewRevision (default: false)
- Health checks are cancelled early when new revisions are detected
- Reduces delay from ~30s timeout to ~5s when fixes are pushed
- Preserves existing behavior when feature is disabled
- Comprehensive test coverage demonstrating both behaviors

The implementation monitors source revisions during health checks and cancels
ongoing checks when new revisions are available, allowing immediate processing
of potential fixes instead of waiting for full timeout periods.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: trashhalo

internal/controller/kustomization_controller.go

@@ -492,6 +492,7 @@ func (r *KustomizationReconciler) reconcile(
 		resourceManager,
 		patcher,
 		obj,
+		src,
 		revision,
 		originRevision,
 		isNewRevision,
@@ -936,6 +937,7 @@ func (r *KustomizationReconciler) checkHealth(ctx context.Context,
 	manager *ssa.ResourceManager,
 	patcher *patch.SerialPatcher,
 	obj *kustomizev1.Kustomization,
+	src sourcev1.Source,
 	revision string,
 	originRevision string,
 	isNewRevision bool,
@@ -982,15 +984,79 @@ func (r *KustomizationReconciler) checkHealth(ctx context.Context,
 		return fmt.Errorf("unable to update the healthy status to progressing: %w", err)
 	}
 
+	// Check if we should cancel health checks on new revisions
+	cancelOnNewRevision := false
+	if enabled, err := features.Enabled(features.CancelHealthCheckOnNewRevision); err == nil && enabled {
+		cancelOnNewRevision = true
+	}
+
+	// Create a cancellable context for health checks if the feature is enabled
+	healthCtx := ctx
+	var cancel context.CancelFunc
+	if cancelOnNewRevision {
+		healthCtx, cancel = context.WithCancel(ctx)
+		defer cancel()
+
+		// Start monitoring for new revisions
+		go func() {
+			ticker := time.NewTicker(5 * time.Second)
+			defer ticker.Stop()
+
+			for {
+				select {
+				case <-healthCtx.Done():
+					return
+				case <-ticker.C:
+					// Get the latest source artifact
+					latestSrc, err := r.getSource(ctx, obj)
+					if err == nil && latestSrc.GetArtifact() != nil {
+						if latestSrc.GetArtifact().Revision != revision {
+							ctrl.LoggerFrom(ctx).Info("New revision detected during health check, cancelling",
+								"current", revision,
+								"new", latestSrc.GetArtifact().Revision)
+							cancel()
+							return
+						}
+					}
+				}
+			}
+		}()
+	}
+
 	// Check the health with a default timeout of 30sec shorter than the reconciliation interval.
-	if err := manager.WaitForSet(toCheck, ssa.WaitOptions{
-		Interval: 5 * time.Second,
-		Timeout:  obj.GetTimeout(),
-		FailFast: r.FailFast,
-	}); err != nil {
-		conditions.MarkFalse(obj, meta.ReadyCondition, meta.HealthCheckFailedReason, "%s", err)
-		conditions.MarkFalse(obj, meta.HealthyCondition, meta.HealthCheckFailedReason, "%s", err)
-		return fmt.Errorf("health check failed after %s: %w", time.Since(checkStart).String(), err)
+	var healthErr error
+	if cancelOnNewRevision {
+		// Run health check in a goroutine and race it with revision monitoring
+		healthDone := make(chan error, 1)
+		go func() {
+			healthDone <- manager.WaitForSet(toCheck, ssa.WaitOptions{
+				Interval: 5 * time.Second,
+				Timeout:  obj.GetTimeout(),
+				FailFast: r.FailFast,
+			})
+		}()
+
+		// Wait for either health check completion or context cancellation
+		select {
+		case healthErr = <-healthDone:
+			// Health check completed normally
+		case <-healthCtx.Done():
+			// Context was cancelled due to new revision
+			healthErr = fmt.Errorf("health check cancelled due to new revision availability")
+		}
+	} else {
+		// Normal health check without cancellation
+		healthErr = manager.WaitForSet(toCheck, ssa.WaitOptions{
+			Interval: 5 * time.Second,
+			Timeout:  obj.GetTimeout(),
+			FailFast: r.FailFast,
+		})
+	}
+
+	if healthErr != nil {
+		conditions.MarkFalse(obj, meta.ReadyCondition, meta.HealthCheckFailedReason, "%s", healthErr)
+		conditions.MarkFalse(obj, meta.HealthyCondition, meta.HealthCheckFailedReason, "%s", healthErr)
+		return fmt.Errorf("health check failed after %s: %w", time.Since(checkStart).String(), healthErr)
 	}
 
 	// Emit recovery event if the previous health check failed.