Merge pull request #1838 from rahulait/add-retry-for-conflicts

[improvement] : add logic to retry if update fails due to conflict

Author: tariq1890

internal/conditions/clusterpolicy.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -55,14 +56,12 @@ func (u *clusterPolicyUpdater) SetConditionsError(ctx context.Context, cr any, r
 	return u.setConditions(ctx, clusterPolicyCr, Error, reason, message)
 }
 
-func (u *clusterPolicyUpdater) setConditions(ctx context.Context, cr *nvidiav1.ClusterPolicy, statusType, reason, message string) error {
-	reqLogger := log.FromContext(ctx)
+// updateConditions updates the conditions of the ClusterPolicy CR
+func (u *clusterPolicyUpdater) updateConditions(ctx context.Context, cr *nvidiav1.ClusterPolicy, statusType, reason, message string) error {
 	// Fetch latest instance and update state to avoid version mismatch
 	instance := &nvidiav1.ClusterPolicy{}
-	err := u.client.Get(ctx, types.NamespacedName{Name: cr.Name}, instance)
-	if err != nil {
-		reqLogger.Error(err, "Failed to get ClusterPolicy instance for status update", "name", cr.Name)
-		return err
+	if err := u.client.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
+		return fmt.Errorf("failed to get ClusterPolicy instance for status update: %w", err)
 	}
 
 	switch statusType {
@@ -93,9 +92,23 @@ func (u *clusterPolicyUpdater) setConditions(ctx context.Context, cr *nvidiav1.C
 			Message: message,
 		})
 	default:
-		reqLogger.Error(nil, "Unknown status type provided", "statusType", statusType)
 		return fmt.Errorf("unknown status type provided: %s", statusType)
 	}
 
 	return u.client.Status().Update(ctx, instance)
 }
+
+// setConditions updates the conditions of the ClusterPolicy CR
+// with retry on conflict to handle version mismatches
+func (u *clusterPolicyUpdater) setConditions(ctx context.Context, cr *nvidiav1.ClusterPolicy, statusType, reason, message string) error {
+	reqLogger := log.FromContext(ctx)
+
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		return u.updateConditions(ctx, cr, statusType, reason, message)
+	})
+
+	if err != nil {
+		reqLogger.Error(err, "Failed to update ClusterPolicy status after retries", "name", cr.Name)
+	}
+	return err
+}

internal/conditions/nvidiadriver.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -61,14 +62,12 @@ func (u *nvDriverUpdater) SetConditionsError(ctx context.Context, cr any, reason
 	return u.setConditions(ctx, nvDriverCr, Error, reason, message)
 }
 
-func (u *nvDriverUpdater) setConditions(ctx context.Context, cr *nvidiav1alpha1.NVIDIADriver, statusType, reason, message string) error {
-	reqLogger := log.FromContext(ctx)
+// updateConditions updates the conditions of the NVIDIADriver CR
+func (u *nvDriverUpdater) updateConditions(ctx context.Context, cr *nvidiav1alpha1.NVIDIADriver, statusType, reason, message string) error {
 	// Fetch latest instance and update state to avoid version mismatch
 	instance := &nvidiav1alpha1.NVIDIADriver{}
-	err := u.client.Get(ctx, types.NamespacedName{Name: cr.Name}, instance)
-	if err != nil {
-		reqLogger.Error(err, "Failed to get NVIDIADriver instance for status update", "name", cr.Name)
-		return err
+	if err := u.client.Get(ctx, types.NamespacedName{Name: cr.Name}, instance); err != nil {
+		return fmt.Errorf("failed to get NVIDIADriver instance for status update: %w", err)
 	}
 
 	switch statusType {
@@ -107,9 +106,23 @@ func (u *nvDriverUpdater) setConditions(ctx context.Context, cr *nvidiav1alpha1.
 			instance.Status.State = nvidiav1alpha1.NotReady
 		}
 	default:
-		reqLogger.Error(nil, "Unknown status type provided", "statusType", statusType)
 		return fmt.Errorf("unknown status type provided: %s", statusType)
 	}
 
 	return u.client.Status().Update(ctx, instance)
 }
+
+// setConditions updates the conditions of the NVIDIADriver CR
+// with retry on conflict to handle version mismatches
+func (u *nvDriverUpdater) setConditions(ctx context.Context, cr *nvidiav1alpha1.NVIDIADriver, statusType, reason, message string) error {
+	reqLogger := log.FromContext(ctx)
+
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		return u.updateConditions(ctx, cr, statusType, reason, message)
+	})
+
+	if err != nil {
+		reqLogger.Error(err, "Failed to update NVIDIADriver status after retries", "name", cr.Name)
+	}
+	return err
+}

vendor/k8s.io/client-go/util/retry/OWNERS

@@ -990,6 +990,7 @@ k8s.io/client-go/util/flowcontrol
 k8s.io/client-go/util/homedir
 k8s.io/client-go/util/jsonpath
 k8s.io/client-go/util/keyutil
+k8s.io/client-go/util/retry
 k8s.io/client-go/util/workqueue
 # k8s.io/component-base v0.34.1
 ## explicit; go 1.24.0