Ensure we always have the *latest* ComputeDomain object when modifying

If we fail in the workqueue loop we could end up trying to modify a
stale object forever if we don't resync it.

Signed-off-by: Kevin Klues <[email protected]>

Author: klueska

cmd/compute-domain-daemon/computedomain.go

@@ -87,7 +87,14 @@ func (m *ComputeDomainManager) Start(ctx context.Context) (rerr error) {
 		}
 	}()
 
-	_, err := m.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+	err := m.informer.AddIndexers(cache.Indexers{
+		"uid": uidIndexer[*nvapi.ComputeDomain],
+	})
+	if err != nil {
+		return fmt.Errorf("error adding indexer for ComputeDomain UID: %w", err)
+	}
+
+	_, err = m.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj any) {
 			m.config.workQueue.Enqueue(obj, m.onAddOrUpdate)
 		},
@@ -133,13 +140,43 @@ func (m *ComputeDomainManager) Stop() error {
 	return nil
 }
 
+// Get gets the ComputeDomain by UID from the informer cache.
+func (m *ComputeDomainManager) Get(uid string) (*nvapi.ComputeDomain, error) {
+	objs, err := m.informer.GetIndexer().ByIndex("uid", uid)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving ComputeDomain by UID: %w", err)
+	}
+	if len(objs) == 0 {
+		return nil, nil
+	}
+	if len(objs) != 1 {
+		return nil, fmt.Errorf("multiple ComputeDomains with the same UID")
+	}
+	cd, ok := objs[0].(*nvapi.ComputeDomain)
+	if !ok {
+		return nil, fmt.Errorf("error casting to ComputeDomain")
+	}
+	return cd, nil
+}
+
 // onAddOrUpdate handles the addition or update of a ComputeDomain.
 func (m *ComputeDomainManager) onAddOrUpdate(ctx context.Context, obj any) error {
-	cd, ok := obj.(*nvapi.ComputeDomain)
+	// Cast the object to a ComputeDomain object
+	o, ok := obj.(*nvapi.ComputeDomain)
 	if !ok {
 		return fmt.Errorf("failed to cast to ComputeDomain")
 	}
 
+	// Get the latest ComputeDomain object from the informer cache since we
+	// plan to update it later and always *must* have the latest version.
+	cd, err := m.Get(string(o.GetUID()))
+	if err != nil {
+		return fmt.Errorf("error getting latest ComputeDomain: %w", err)
+	}
+	if cd == nil {
+		return nil
+	}
+
 	// Skip ComputeDomains that don't match on UUID
 	if string(cd.UID) != m.config.computeDomainUUID {
 		klog.Errorf("ComputeDomain processed with non-matching UID (%v, %v)", cd.UID, m.config.computeDomainUUID)

cmd/compute-domain-daemon/indexers.go

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func uidIndexer[T metav1.ObjectMetaAccessor](obj any) ([]string, error) {
+	d, ok := obj.(T)
+	if !ok {
+		return nil, fmt.Errorf("expected a %T but got %T", *new(T), obj)
+	}
+	return []string{string(d.GetObjectMeta().GetUID())}, nil
+}