Checks and reports S3 health

Signed-off-by: raaizik <[email protected]>

Author: raaizik

api/v1alpha1/drclusterconfig_types.go

@@ -33,7 +33,7 @@ type DRClusterConfigSpec struct {
 
 const (
 	DRClusterConfigConfigurationProcessed string = "Processed"
-	DRClusterConfigS3Reachable            string = "Reachable"
+	DRClusterConfigS3Healthy              string = "MetaDataStoresHealthy"
 )
 
 // DRClusterConfigStatus defines the observed state of DRClusterConfig

cmd/main.go

@@ -181,9 +181,10 @@ func setupReconcilersCluster(mgr ctrl.Manager, ramenConfig *ramendrv1alpha1.Rame
 	}
 
 	if err := (&controllers.DRClusterConfigReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    ctrl.Log.WithName("drcc"),
+		Client:            mgr.GetClient(),
+		Scheme:            mgr.GetScheme(),
+		Log:               ctrl.Log.WithName("drcc"),
+		ObjectStoreGetter: controllers.S3ObjectStoreGetter(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "DRClusterConfig")
 		os.Exit(1)

internal/controller/drcluster_controller.go

@@ -500,7 +500,7 @@ func validateS3Profile(ctx context.Context, apiReader client.Reader,
 	drcluster *ramen.DRCluster, listKeyPrefix string, log logr.Logger,
 ) (string, error) {
 	if drcluster.Spec.S3ProfileName != NoS3StoreAvailable {
-		if reason, err := s3ProfileValidate(ctx, apiReader, objectStoreGetter,
+		if reason, err := S3ProfileValidate(ctx, apiReader, objectStoreGetter,
 			drcluster.Spec.S3ProfileName, listKeyPrefix, log); err != nil {
 			return reason, err
 		}
@@ -509,23 +509,6 @@ func validateS3Profile(ctx context.Context, apiReader client.Reader,
 	return "", nil
 }
 
-func s3ProfileValidate(ctx context.Context, apiReader client.Reader,
-	objectStoreGetter ObjectStoreGetter, s3ProfileName, listKeyPrefix string,
-	log logr.Logger,
-) (string, error) {
-	objectStore, _, err := objectStoreGetter.ObjectStore(
-		ctx, apiReader, s3ProfileName, "drpolicy validation", log)
-	if err != nil {
-		return "s3ConnectionFailed", fmt.Errorf("%s: %w", s3ProfileName, err)
-	}
-
-	if _, err := objectStore.ListKeys(listKeyPrefix); err != nil {
-		return "s3ListFailed", fmt.Errorf("%s: %w", s3ProfileName, err)
-	}
-
-	return "", nil
-}
-
 func validateCIDRsFormat(drcluster *ramen.DRCluster, log logr.Logger) error {
 	// validate the CIDRs format
 	invalidCidrs := []string{}

internal/controller/drcluster_util.go

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: The RamenDR authors
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func S3ProfileValidate(ctx context.Context, apiReader client.Reader,
+	objectStoreGetter ObjectStoreGetter, s3ProfileName, listKeyPrefix string,
+	log logr.Logger,
+) (string, error) {
+	objectStore, _, err := objectStoreGetter.ObjectStore(
+		ctx, apiReader, s3ProfileName, "drpolicy validation", log)
+	if err != nil {
+		return "s3ConnectionFailed", fmt.Errorf("%s: %w", s3ProfileName, err)
+	}
+
+	if _, err := objectStore.ListKeys(listKeyPrefix); err != nil {
+		return "s3ListFailed", fmt.Errorf("%s: %w", s3ProfileName, err)
+	}
+
+	return "", nil
+}

internal/controller/drclusterconfig_controller.go

@@ -11,6 +11,9 @@ import (
 	"time"
 
 	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
+	v1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
 	volrep "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
 	snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
 	groupsnapv1beta1 "github.com/red-hat-storage/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
@@ -56,9 +59,10 @@ const (
 // DRClusterConfigReconciler reconciles a DRClusterConfig object
 type DRClusterConfigReconciler struct {
 	client.Client
-	Scheme      *runtime.Scheme
-	Log         logr.Logger
-	RateLimiter *workqueue.TypedRateLimiter[reconcile.Request]
+	Scheme            *runtime.Scheme
+	Log               logr.Logger
+	RateLimiter       *workqueue.TypedRateLimiter[reconcile.Request]
+	ObjectStoreGetter ObjectStoreGetter
 }
 
 //nolint:lll
@@ -148,7 +152,7 @@ func setDRClusterConfigInitialCondition(conditions *[]metav1.Condition, observed
 		Message:            message,
 	})
 	util.SetStatusConditionIfNotFound(conditions, metav1.Condition{
-		Type:               ramen.DRClusterConfigS3Reachable,
+		Type:               ramen.DRClusterConfigS3Healthy,
 		Reason:             DRClusterConfigConditionReasonInitializing,
 		ObservedGeneration: observedGeneration,
 		Status:             metav1.ConditionUnknown,
@@ -168,6 +172,18 @@ func setDRClusterConfigConfigurationProcessedCondition(conditions *[]metav1.Cond
 	})
 }
 
+func setDRClusterConfigS3ReachableCondition(conditions *[]metav1.Condition, observedGeneration int64,
+	message string, conditionStatus metav1.ConditionStatus, reason string,
+) {
+	util.SetStatusCondition(conditions, metav1.Condition{
+		Type:               ramen.DRClusterConfigS3Healthy,
+		Reason:             reason,
+		ObservedGeneration: observedGeneration,
+		Status:             conditionStatus,
+		Message:            message,
+	})
+}
+
 func (r *DRClusterConfigReconciler) GetDRClusterConfig(ctx context.Context) (*ramen.DRClusterConfig, error) {
 	drcConfigs := &ramen.DRClusterConfigList{}
 	if err := r.Client.List(ctx, drcConfigs); err != nil {
@@ -275,9 +291,66 @@ func (r *DRClusterConfigReconciler) processCreateOrUpdate(
 	setDRClusterConfigConfigurationProcessedCondition(&drCConfig.Status.Conditions, drCConfig.Generation,
 		"Configuration processed and validated", metav1.ConditionTrue, DRClusterConfigConditionConfigurationProcessed)
 
+	if err := r.validateS3Profiles(ctx, drCConfig); err != nil {
+		log.Info("Reconcile error", "error", err)
+
+		return ctrl.Result{Requeue: true}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 
+func (r *DRClusterConfigReconciler) validateS3Profiles(ctx context.Context, drCConfig *ramen.DRClusterConfig) error {
+	// Fetch the ramen config resource
+	_, ramenConfig, err := ConfigMapGet(ctx, r.Client)
+	if err != nil {
+		return fmt.Errorf("failed to get Ramen configmap: %w", err)
+	}
+
+	// Iterate all profiles listed in it and check for existing healthy ones
+	for profileIdx := range ramenConfig.S3StoreProfiles {
+		// for each profile, check that it has an actual secret attached to its secretRef ID
+		profile := ramenConfig.S3StoreProfiles[profileIdx]
+		secretRef := profile.S3SecretRef
+		secret := &v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{Name: secretRef.Name, Namespace: secretRef.Namespace},
+		}
+
+		if err := r.Client.Get(ctx, types.NamespacedName{
+			Namespace: secret.Namespace,
+			Name:      secret.Name,
+		}, secret); err != nil {
+			if !k8serrors.IsNotFound(err) {
+				setDRClusterConfigS3ReachableCondition(&drCConfig.Status.Conditions, drCConfig.Generation,
+					fmt.Sprintf("Found an unhealthy S3 profile %q for which there's no secret", profile.S3ProfileName),
+					metav1.ConditionFalse, DRClusterConfigS3Unreachable)
+
+				return fmt.Errorf("failed to get secret: %w", err)
+			}
+			// If there's no secret attached to the secretRef's namespacedname -- mark profile as unhealthy
+			setDRClusterConfigS3ReachableCondition(&drCConfig.Status.Conditions, drCConfig.Generation,
+				fmt.Sprintf("Found an unhealthy S3 profile %q for which there's no secret", profile.S3ProfileName),
+				metav1.ConditionFalse, DRClusterConfigS3Unreachable)
+
+			return fmt.Errorf("secrect not found: %w", err)
+		}
+		// Profile does have a secret. Check if it has connectivity and record in status accordingly
+		if reason, err := S3ProfileValidate(ctx, r.Client, r.ObjectStoreGetter, profile.S3ProfileName, types.NamespacedName{
+			Name: drCConfig.Name, Namespace: drCConfig.Namespace,
+		}.String(), r.Log); err != nil {
+			setDRClusterConfigS3ReachableCondition(&drCConfig.Status.Conditions, drCConfig.Generation, err.Error(),
+				metav1.ConditionFalse, reason)
+
+			return fmt.Errorf("failed to validate s3 profile: %w", err)
+		}
+	}
+	// All S3 profiles are healthy -- record to status and exit
+	setDRClusterConfigS3ReachableCondition(&drCConfig.Status.Conditions, drCConfig.Generation,
+		fmt.Sprintf("All S3 profiles are healthy"), metav1.ConditionTrue, DRClusterConfigS3Reachable)
+
+	return nil
+}
+
 // UpdateSupportedClasses updates DRClusterConfig status with a list of storage related classes that are marked for DR
 // support. The list is sorted alphabetically to avoid out of order listing and status updates due to the same
 func (r *DRClusterConfigReconciler) UpdateSupportedClasses(

internal/controller/drclusterconfig_controller_test.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+
 	volrep "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
 	snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
 	. "github.com/onsi/ginkgo/v2"
@@ -77,6 +79,7 @@ var _ = Describe("DRClusterConfigControllerTests", Ordered, func() {
 		baseVGSC, vgsc1, vgsc2 *groupsnapv1beta1.VolumeGroupSnapshotClass
 		baseNFC, nfc1, nfc2    *csiaddonsv1alpha1.NetworkFenceClass
 		classes                Classes
+		ramenConfig            *ramen.RamenConfig
 	)
 
 	BeforeAll(func() {
@@ -109,6 +112,37 @@ var _ = Describe("DRClusterConfigControllerTests", Ordered, func() {
 		k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 		Expect(err).NotTo(HaveOccurred())
 
+		By("Creating namespaces")
+
+		Expect(k8sClient.Create(context.TODO(),
+			&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ramenNamespace}})).To(Succeed())
+
+		By("Defining a ramen configuration")
+
+		ramenConfig = &ramen.RamenConfig{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "RamenConfig",
+				APIVersion: ramen.GroupVersion.String(),
+			},
+			LeaderElection: &config.LeaderElectionConfiguration{
+				LeaderElect:  new(bool),
+				ResourceName: ramencontrollers.HubLeaderElectionResourceName,
+			},
+			Metrics: ramen.ControllerMetrics{
+				BindAddress: "0", // Disable metrics
+			},
+			RamenControllerType: ramen.DRHubType,
+		}
+		ramenConfig.DrClusterOperator.DeploymentAutomationEnabled = true
+		ramenConfig.DrClusterOperator.S3SecretDistributionEnabled = true
+		configMap, err := ramencontrollers.ConfigMapNew(
+			ramenNamespace,
+			ramencontrollers.HubOperatorConfigMapName,
+			ramenConfig,
+		)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.TODO(), configMap)).To(Succeed())
+
 		By("starting the DRClusterConfig reconciler")
 
 		ramenConfig := &ramen.RamenConfig{