refactor(operator): prometheus operand (#629)

Author: enoodle

CHANGELOG.md

@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ### Fixed
 - Fixed a bug where the scheduler would not re-try updating podgroup status after failure
 - Fixed a bug where ray workloads gang scheduling would ignore `minReplicas` if autoscaling was not set
+- KAI Config wrong statuses when prometheus operand is enabled
 
 ## [v0.9.1] - 20250-09-15
 

pkg/apis/kai/v1/config_types.go

@@ -37,6 +37,7 @@ const (
 	Available                  ConditionReason = "available"
 	Reconciled                 ConditionReason = "reconciled"
 	DependenciesFulfilled      ConditionReason = "dependencies_fulfilled"
+	DependenciesMissing        ConditionReason = "dependencies_missing"
 	PrometheusConnected        ConditionReason = "prometheus_connected"
 	PrometheusConnectionFailed ConditionReason = "prometheus_connection_failed"
 )

pkg/operator/controller/status_reconciler/status_reconciler.go

@@ -48,7 +48,7 @@ func (r *StatusReconciler) ReconcileStatus(ctx context.Context, object objectWit
 	if err := r.reconcileCondition(ctx, object, r.getAvailableCondition(ctx, object.GetGeneration())); err != nil {
 		return err
 	}
-	return r.reconcileCondition(ctx, object, r.getDependenciesFulfilledCondition(object.GetGeneration()))
+	return r.reconcileCondition(ctx, object, r.getDependenciesFulfilledCondition(ctx, object))
 }
 
 func (r *StatusReconciler) reconcileCondition(ctx context.Context, object objectWithConditions, condition metav1.Condition) error {
@@ -155,13 +155,36 @@ func (r *StatusReconciler) getAvailableCondition(ctx context.Context, gen int64)
 	}
 }
 
-func (r *StatusReconciler) getDependenciesFulfilledCondition(gen int64) metav1.Condition {
+func (r *StatusReconciler) getDependenciesFulfilledCondition(ctx context.Context, object objectWithConditions) metav1.Condition {
+	missingDependencies, err := r.deployable.HasMissingDependencies(ctx, r.Client, object.GetInternalObject())
+	if err != nil {
+		return metav1.Condition{
+			Type:               string(kaiv1.ConditionDependenciesFulfilled),
+			Status:             metav1.ConditionFalse,
+			Reason:             string(kaiv1.DependenciesMissing),
+			Message:            err.Error(),
+			ObservedGeneration: object.GetGeneration(),
+			LastTransitionTime: metav1.Now(),
+		}
+	}
+
+	if len(missingDependencies) > 0 {
+		return metav1.Condition{
+			Type:               string(kaiv1.ConditionDependenciesFulfilled),
+			Status:             metav1.ConditionFalse,
+			Reason:             string(kaiv1.DependenciesMissing),
+			Message:            missingDependencies,
+			ObservedGeneration: object.GetGeneration(),
+			LastTransitionTime: metav1.Now(),
+		}
+	}
+
 	return metav1.Condition{
 		Type:               string(kaiv1.ConditionDependenciesFulfilled),
 		Status:             metav1.ConditionTrue,
 		Reason:             string(kaiv1.DependenciesFulfilled),
 		Message:            "Dependencies are fulfilled",
-		ObservedGeneration: gen,
+		ObservedGeneration: object.GetGeneration(),
 		LastTransitionTime: metav1.Now(),
 	}
 }

pkg/operator/controller/status_reconciler/status_reconciler_test.go

@@ -188,6 +188,14 @@ func (f *fakeDeployable) IsAvailable(ctx context.Context, runtimeClient client.R
 	return f.isAvailable, nil
 }
 
+func (f *fakeDeployable) Monitor(ctx context.Context, runtimeReader client.Reader, kaiConfig *kaiv1.Config) error {
+	return nil
+}
+
+func (f *fakeDeployable) HasMissingDependencies(ctx context.Context, readerClient client.Reader, obj client.Object) (string, error) {
+	return "", nil
+}
+
 func (f *fakeDeployable) Name() string {
 	return "fakeDeployable"
 }

pkg/operator/operands/admission/admission.go

@@ -81,3 +81,7 @@ func (a *Admission) Name() string {
 func (a *Admission) Monitor(ctx context.Context, runtimeReader client.Reader, kaiConfig *kaiv1.Config) error {
 	return nil
 }
+
+func (a *Admission) HasMissingDependencies(context.Context, client.Reader, *kaiv1.Config) (string, error) {
+	return "", nil
+}

pkg/operator/operands/binder/binder.go

@@ -66,3 +66,7 @@ func (b *Binder) Name() string {
 func (b *Binder) Monitor(ctx context.Context, runtimeReader client.Reader, kaiConfig *kaiv1.Config) error {
 	return nil
 }
+
+func (b *Binder) HasMissingDependencies(context.Context, client.Reader, *kaiv1.Config) (string, error) {
+	return "", nil
+}

pkg/operator/operands/deployable/deployable.go

@@ -66,15 +66,15 @@ func (d *DeployableOperands) Deploy(
 		Controller: ptr.To(true),
 	}
 
-	if createObjectsInCluster(ctx, runtimeClient, reconcilerAsOwnerReference, objectsToCreate) != nil {
+	if err := createObjectsInCluster(ctx, runtimeClient, reconcilerAsOwnerReference, objectsToCreate); err != nil {
 		return err
 	}
 
-	if deleteObjectsInCluster(ctx, runtimeClient, objectsToDelete) != nil {
+	if err := deleteObjectsInCluster(ctx, runtimeClient, objectsToDelete); err != nil {
 		return err
 	}
 
-	if updateObjectsInCluster(ctx, runtimeClient, reconcilerAsOwnerReference, objectsToUpdate) != nil {
+	if err := updateObjectsInCluster(ctx, runtimeClient, reconcilerAsOwnerReference, objectsToUpdate); err != nil {
 		return err
 	}
 	return nil
@@ -301,3 +301,26 @@ func (d *DeployableOperands) Monitor(ctx context.Context, runtimeReader client.R
 	}
 	return nil
 }
+
+func (d *DeployableOperands) HasMissingDependencies(ctx context.Context, readerClient client.Reader, object client.Object) (string, error) {
+	var missingDependencies string
+	var err error
+
+	kaiConfig, checkForKAIConfig := object.(*kaiv1.Config)
+	if !checkForKAIConfig {
+		return "", nil
+	}
+
+	for _, operand := range d.operands {
+		// Assuming each operand has a HasDependencies method
+		missing, e := operand.HasMissingDependencies(ctx, readerClient, kaiConfig)
+		if e != nil {
+			err = errors.Join(err, e)
+		}
+		if len(missing) > 0 {
+			missingDependencies = missingDependencies + fmt.Sprintf("\n%s is missing %s", operand.Name(), missing)
+		}
+	}
+
+	return missingDependencies, err
+}