CD plugin: always inject CD details via CDI

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

Rename 'domain' to 'domainID'

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

squash: review feedback

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

shorten comment

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

Author: jgehrcke

cmd/compute-domain-daemon/computedomain.go

@@ -186,7 +186,8 @@ func (m *ComputeDomainManager) Get(uid string) (*nvapi.ComputeDomain, error) {
 
 // onAddOrUpdate handles the addition or update of a ComputeDomain. Here, we
 // receive updates not for all CDs in the system, but only for the CD that we
-// are registered for (filtered by CD name).
+// are registered for (filtered by CD name). Note that the informer triggers
+// this callback once upon startup for all existing objects.
 func (m *ComputeDomainManager) onAddOrUpdate(ctx context.Context, obj any) error {
 	// Cast the object to a ComputeDomain object
 	o, ok := obj.(*nvapi.ComputeDomain)
@@ -212,8 +213,7 @@ func (m *ComputeDomainManager) onAddOrUpdate(ctx context.Context, obj any) error
 		return nil
 	}
 
-	// Update node info in ComputeDomain
-	// Why are we doing this (only) upon receiving another update?
+	// Update node info in ComputeDomain.
 	if err := m.UpdateComputeDomainNodeInfo(ctx, cd); err != nil {
 		return fmt.Errorf("error updating node info in ComputeDomain: %w", err)
 	}
@@ -228,7 +228,6 @@ func (m *ComputeDomainManager) UpdateComputeDomainNodeInfo(ctx context.Context,
 	var nodeInfo *nvapi.ComputeDomainNode
 
 	// Create a deep copy of the ComputeDomain to avoid modifying the original
-	// TODO: review for 10000-node-CD
 	newCD := cd.DeepCopy()
 
 	defer func() {
@@ -262,8 +261,7 @@ func (m *ComputeDomainManager) UpdateComputeDomainNodeInfo(ctx context.Context,
 			Name:     m.config.nodeName,
 			CliqueID: m.config.cliqueID,
 			Index:    nextIndex,
-			// Initialize as NotReady (will be updated by podmanager).
-			Status: nvapi.ComputeDomainStatusNotReady,
+			Status:   nvapi.ComputeDomainStatusNotReady,
 		}
 
 		klog.Infof("CD status does not contain node name '%s' yet, try to insert myself: %v", m.config.nodeName, nodeInfo)

cmd/compute-domain-kubelet-plugin/computedomain.go

@@ -60,7 +60,7 @@ type ComputeDomainManager struct {
 
 type ComputeDomainDaemonSettings struct {
 	manager         *ComputeDomainManager
-	domain          string
+	domainID        string
 	rootDir         string
 	configTmplPath  string
 	nodesConfigPath string
@@ -128,13 +128,13 @@ func (m *ComputeDomainManager) Stop() error {
 	return nil
 }
 
-func (m *ComputeDomainManager) NewSettings(domain string) *ComputeDomainDaemonSettings {
+func (m *ComputeDomainManager) NewSettings(domainID string) *ComputeDomainDaemonSettings {
 	return &ComputeDomainDaemonSettings{
 		manager:         m,
-		domain:          domain,
-		rootDir:         fmt.Sprintf("%s/%s", m.configFilesRoot, domain),
-		configTmplPath:  fmt.Sprintf("%s/%s/%s", m.configFilesRoot, domain, "config.tmpl.cfg"),
-		nodesConfigPath: fmt.Sprintf("%s/%s/%s", m.configFilesRoot, domain, "nodes_config.cfg"),
+		domainID:        domainID,
+		rootDir:         fmt.Sprintf("%s/%s", m.configFilesRoot, domainID),
+		configTmplPath:  fmt.Sprintf("%s/%s/%s", m.configFilesRoot, domainID, "config.tmpl.cfg"),
+		nodesConfigPath: fmt.Sprintf("%s/%s/%s", m.configFilesRoot, domainID, "nodes_config.cfg"),
 	}
 }
 
@@ -154,27 +154,40 @@ func (m *ComputeDomainManager) GetComputeDomainChannelContainerEdits(devRoot str
 	}
 }
 
-func (s *ComputeDomainDaemonSettings) GetDomain() string {
-	return s.domain
-}
-
-func (s *ComputeDomainDaemonSettings) GetCDIContainerEdits(ctx context.Context, devRoot string, info *nvcapDeviceInfo) (*cdiapi.ContainerEdits, error) {
-	cd, err := s.manager.GetComputeDomain(ctx, s.domain)
+// GetComputeDomainDaemonContainerEdits() returns the CDI spec edits always
+// required for launching the CD Daemon (whether or not it tries to launch an
+// IMEX daemon internally).
+func (m *ComputeDomainManager) GetComputeDomainDaemonContainerEdits(ctx context.Context, domainID string) (*cdiapi.ContainerEdits, error) {
+	cd, err := m.GetComputeDomain(ctx, domainID)
 	if err != nil {
-		return nil, fmt.Errorf("error getting compute domain: %w", err)
+		return nil, fmt.Errorf("error getting compute domain %s: %w", domainID, err)
 	}
 	if cd == nil {
-		return nil, fmt.Errorf("compute domain not found: %s", s.domain)
+		return nil, fmt.Errorf("compute domain not found: %s", domainID)
 	}
 
 	edits := &cdiapi.ContainerEdits{
 		ContainerEdits: &cdispec.ContainerEdits{
 			Env: []string{
-				fmt.Sprintf("CLIQUE_ID=%s", s.manager.cliqueID),
+				fmt.Sprintf("CLIQUE_ID=%s", m.cliqueID),
 				fmt.Sprintf("COMPUTE_DOMAIN_UUID=%s", cd.UID),
 				fmt.Sprintf("COMPUTE_DOMAIN_NAME=%s", cd.Name),
 				fmt.Sprintf("COMPUTE_DOMAIN_NAMESPACE=%s", cd.Namespace),
 			},
+		},
+	}
+	return edits, nil
+}
+
+func (s *ComputeDomainDaemonSettings) GetDomainID() string {
+	return s.domainID
+}
+
+// GetCDIContainerEditsForImex() returns the CDI spec edits only required for
+// launching the CD Daemon when it actually wraps an IMEX daemon.
+func (s *ComputeDomainDaemonSettings) GetCDIContainerEditsForImex(ctx context.Context, devRoot string, info *nvcapDeviceInfo) *cdiapi.ContainerEdits {
+	edits := &cdiapi.ContainerEdits{
+		ContainerEdits: &cdispec.ContainerEdits{
 			Mounts: []*cdispec.Mount{
 				{
 					ContainerPath: "/etc/nvidia-imex",
@@ -193,8 +206,7 @@ func (s *ComputeDomainDaemonSettings) GetCDIContainerEdits(ctx context.Context,
 			},
 		},
 	}
-
-	return edits, nil
+	return edits
 }
 
 func (s *ComputeDomainDaemonSettings) Prepare(ctx context.Context) error {

cmd/compute-domain-kubelet-plugin/device_state.go

@@ -491,6 +491,15 @@ func (s *DeviceState) applyComputeDomainDaemonConfig(ctx context.Context, config
 		ComputeDomain: config.DomainID,
 	}
 
+	// Always inject CD details into the CD daemon, in a heterogeneous CD clique
+	// ID below may be empty (while other CD details are set, and consumed by
+	// the CD daemon).
+	edits, err := s.computeDomainManager.GetComputeDomainDaemonContainerEdits(ctx, config.DomainID)
+	if err != nil {
+		return nil, fmt.Errorf("error preparing ComputeDomain daemon settings: %w", err)
+	}
+	configState.containerEdits = configState.containerEdits.Append(edits)
+
 	// Only prepare files to inject to the daemon if IMEX is supported.
 	if s.computeDomainManager.cliqueID != "" {
 		// Parse the device node info for the fabic-imex-mgmt nvcap.
@@ -508,10 +517,7 @@ func (s *DeviceState) applyComputeDomainDaemonConfig(ctx context.Context, config
 		}
 
 		// Store information about the ComputeDomain daemon in the configState.
-		edits, err := computeDomainDaemonSettings.GetCDIContainerEdits(ctx, s.cdi.devRoot, nvcapDeviceInfo)
-		if err != nil {
-			return nil, fmt.Errorf("error getting container edits for ComputeDomain daemon for requests '%v' in claim '%v': %w", requests, claim.UID, err)
-		}
+		edits := computeDomainDaemonSettings.GetCDIContainerEditsForImex(ctx, s.cdi.devRoot, nvcapDeviceInfo)
 		configState.containerEdits = configState.containerEdits.Append(edits)
 	}