feat: add externalURL config and support (#563)

* feat: add external prometheus support

* feat: implement monitoring mechanism

* refactor: move monitoring code to operator, expand operator interface

Author: SiorMeir

deployments/kai-scheduler/crds/kai.scheduler_configs.yaml

@@ -2012,6 +2012,29 @@ spec:
                     description: Enabled defines whether a Prometheus instance should
                       be deployed
                     type: boolean
+                  externalPrometheusHealthProbe:
+                    description: ExternalPrometheusPingConfig defines the configuration
+                      for external Prometheus connectivity validation, with defaults.
+                    properties:
+                      interval:
+                        description: Interval defines the interval for external Prometheus
+                          connectivity validation (in seconds)
+                        type: integer
+                      maxRetries:
+                        description: PingsMaxRetries defines the maximum number of
+                          retries for external Prometheus connectivity validation
+                        type: integer
+                      timeout:
+                        description: PingsTimeout defines the timeout for external
+                          Prometheus connectivity validation (in seconds)
+                        type: integer
+                    type: object
+                  externalPrometheusUrl:
+                    description: |-
+                      ExternalPrometheusUrl defines the URL of an external Prometheus instance to use
+                      When set, KAI will not deploy its own Prometheus but will configure ServiceMonitors
+                      for the external instance and validate connectivity
+                    type: string
                   retentionPeriod:
                     description: RetentionPeriod defines how long to retain data (e.g.,
                       "2w", "1d", "30d")

pkg/apis/kai/v1/config_types.go

@@ -33,10 +33,12 @@ const (
 type ConditionReason string
 
 const (
-	Deployed              ConditionReason = "deployed"
-	Available             ConditionReason = "available"
-	Reconciled            ConditionReason = "reconciled"
-	DependenciesFulfilled ConditionReason = "dependencies_fulfilled"
+	Deployed                   ConditionReason = "deployed"
+	Available                  ConditionReason = "available"
+	Reconciled                 ConditionReason = "reconciled"
+	DependenciesFulfilled      ConditionReason = "dependencies_fulfilled"
+	PrometheusConnected        ConditionReason = "prometheus_connected"
+	PrometheusConnectionFailed ConditionReason = "prometheus_connection_failed"
 )
 
 // ConfigSpec defines the desired state of Config

pkg/apis/kai/v1/prometheus/prometheus.go

@@ -43,6 +43,30 @@ type Prometheus struct {
 	// ServiceMonitor defines ServiceMonitor configuration for KAI services
 	// +kubebuilder:validation:Optional
 	ServiceMonitor *ServiceMonitor `json:"serviceMonitor,omitempty"`
+
+	// ExternalPrometheusUrl defines the URL of an external Prometheus instance to use
+	// When set, KAI will not deploy its own Prometheus but will configure ServiceMonitors
+	// for the external instance and validate connectivity
+	// +kubebuilder:validation:Optional
+	ExternalPrometheusUrl *string `json:"externalPrometheusUrl,omitempty"`
+
+	// ExternalPrometheusPingConfig defines the configuration for external Prometheus connectivity validation, with defaults.
+	// +kubebuilder:validation:Optional
+	ExternalPrometheusHealthProbe *ExternalPrometheusHealthProbe `json:"externalPrometheusHealthProbe,omitempty"`
+}
+
+type ExternalPrometheusHealthProbe struct {
+	// Interval defines the interval for external Prometheus connectivity validation (in seconds)
+	// +kubebuilder:validation:Optional
+	Interval *int `json:"interval,omitempty"`
+
+	// PingsTimeout defines the timeout for external Prometheus connectivity validation (in seconds)
+	// +kubebuilder:validation:Optional
+	Timeout *int `json:"timeout,omitempty"`
+
+	// PingsMaxRetries defines the maximum number of retries for external Prometheus connectivity validation
+	// +kubebuilder:validation:Optional
+	MaxRetries *int `json:"maxRetries,omitempty"`
 }
 
 func (p *Prometheus) SetDefaultsWhereNeeded() {
@@ -53,7 +77,9 @@ func (p *Prometheus) SetDefaultsWhereNeeded() {
 	p.RetentionPeriod = common.SetDefault(p.RetentionPeriod, ptr.To("2w"))
 	p.SampleInterval = common.SetDefault(p.SampleInterval, ptr.To("1m"))
 	p.StorageClassName = common.SetDefault(p.StorageClassName, ptr.To("standard"))
-
+	p.ExternalPrometheusUrl = common.SetDefault(p.ExternalPrometheusUrl, nil)
+	p.ExternalPrometheusHealthProbe = common.SetDefault(p.ExternalPrometheusHealthProbe, &ExternalPrometheusHealthProbe{})
+	p.ExternalPrometheusHealthProbe.SetDefaultsWhereNeeded()
 	p.ServiceMonitor = common.SetDefault(p.ServiceMonitor, &ServiceMonitor{})
 	p.ServiceMonitor.SetDefaultsWhereNeeded()
 }
@@ -253,3 +279,12 @@ func (s *ServiceMonitor) SetDefaultsWhereNeeded() {
 	s.ScrapeTimeout = common.SetDefault(s.ScrapeTimeout, ptr.To("10s"))
 	s.BearerTokenFile = common.SetDefault(s.BearerTokenFile, ptr.To("/var/run/secrets/kubernetes.io/serviceaccount/token"))
 }
+
+func (p *ExternalPrometheusHealthProbe) SetDefaultsWhereNeeded() {
+	if p == nil {
+		return
+	}
+	p.Interval = common.SetDefault(p.Interval, ptr.To(30))
+	p.Timeout = common.SetDefault(p.Timeout, ptr.To(10))
+	p.MaxRetries = common.SetDefault(p.MaxRetries, ptr.To(5))
+}

pkg/apis/kai/v1/prometheus/zz_generated.deepcopy.go

@@ -123,6 +123,12 @@ func (r *ConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 	if err = r.deployable.Deploy(ctx, r.Client, kaiConfig, kaiConfig); err != nil {
 		return ctrl.Result{}, err
 	}
+
+	// Monitor all operands
+	if err = r.deployable.Monitor(ctx, r.Client, kaiConfig); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 

pkg/operator/controller/config_controller.go

@@ -73,3 +73,7 @@ func (b *Admission) IsAvailable(ctx context.Context, readerClient client.Reader)
 func (b *Admission) Name() string {
 	return "KAIAdmission"
 }
+
+func (b *Admission) Monitor(ctx context.Context, runtimeReader client.Reader, kaiConfig *kaiv1.Config) error {
+	return nil
+}

pkg/operator/operands/admission/admission.go

@@ -58,3 +58,7 @@ func (b *Binder) IsAvailable(ctx context.Context, readerClient client.Reader) (b
 func (b *Binder) Name() string {
 	return "Binder"
 }
+
+func (b *Binder) Monitor(ctx context.Context, runtimeReader client.Reader, kaiConfig *kaiv1.Config) error {
+	return nil
+}

pkg/operator/operands/binder/binder.go

@@ -292,3 +292,12 @@ func createObjectForKAIConfig(
 
 	return nil
 }
+
+func (d *DeployableOperands) Monitor(ctx context.Context, runtimeReader client.Reader, kaiConfig *kaiv1.Config) error {
+	for _, operand := range d.operands {
+		if err := operand.Monitor(ctx, runtimeReader, kaiConfig); err != nil {
+			return fmt.Errorf("failed monitoring %s: %v", operand.Name(), err)
+		}
+	}
+	return nil
+}

pkg/operator/operands/deployable/deployable.go

@@ -351,6 +351,10 @@ func (f *fakeOperand) IsAvailable(_ context.Context, _ client.Reader) (bool, err
 	return f.isAvailable, nil
 }
 
+func (f *fakeOperand) Monitor(_ context.Context, _ client.Reader, _ *kaiv1.Config) error {
+	return nil
+}
+
 func (f *fakeOperand) Name() string {
 	if f.name == "" {
 		return "fakeOperand"

pkg/operator/operands/deployable/deployable_test.go

@@ -14,5 +14,6 @@ type Operand interface {
 	DesiredState(context.Context, client.Reader, *enginev1alpha1.Config) ([]client.Object, error)
 	IsDeployed(context.Context, client.Reader) (bool, error)
 	IsAvailable(context.Context, client.Reader) (bool, error)
+	Monitor(context.Context, client.Reader, *enginev1alpha1.Config) error
 	Name() string
 }