Make maxNodesPerIMEXDomain configurable (default at 18)

Signed-off-by: Kevin Klues <[email protected]>

Author: klueska

cmd/compute-domain-controller/controller.go

@@ -38,6 +38,9 @@ type ManagerConfig struct {
 	// imageName is the full image name to use when rendering templates
 	imageName string
 
+	// maxNodesPerIMEXDomain is the maximum number of nodes per IMEX domain to allocate
+	maxNodesPerIMEXDomain int
+
 	// clientsets provides access to various Kubernetes API client interfaces
 	clientsets flags.ClientSets
 
@@ -67,12 +70,13 @@ func (c *Controller) Run(ctx context.Context) error {
 	workQueue := workqueue.New(workqueue.DefaultControllerRateLimiter())
 
 	managerConfig := &ManagerConfig{
-		driverName:           c.config.driverName,
-		driverNamespace:      c.config.flags.namespace,
-		additionalNamespaces: c.config.flags.additionalNamespaces.Value(),
-		imageName:            c.config.flags.imageName,
-		clientsets:           c.config.clientsets,
-		workQueue:            workQueue,
+		driverName:            c.config.driverName,
+		driverNamespace:       c.config.flags.namespace,
+		additionalNamespaces:  c.config.flags.additionalNamespaces.Value(),
+		imageName:             c.config.flags.imageName,
+		maxNodesPerIMEXDomain: c.config.flags.maxNodesPerIMEXDomain,
+		clientsets:            c.config.clientsets,
+		workQueue:             workQueue,
 	}
 
 	cdManager := NewComputeDomainManager(managerConfig)

cmd/compute-domain-controller/daemonset.go

@@ -50,6 +50,7 @@ type DaemonSetTemplateData struct {
 	ComputeDomainLabelValue   types.UID
 	ResourceClaimTemplateName string
 	ImageName                 string
+	MaxNodesPerIMEXDomain     int
 	FeatureGates              map[string]bool
 }
 
@@ -195,6 +196,7 @@ func (m *DaemonSetManager) Create(ctx context.Context, cd *nvapi.ComputeDomain)
 		ComputeDomainLabelValue:   cd.UID,
 		ResourceClaimTemplateName: rct.Name,
 		ImageName:                 m.config.imageName,
+		MaxNodesPerIMEXDomain:     m.config.maxNodesPerIMEXDomain,
 		FeatureGates:              featuregates.ToMap(),
 	}
 

cmd/compute-domain-controller/main.go

@@ -43,17 +43,19 @@ import (
 )
 
 const (
-	DriverName = "compute-domain.nvidia.com"
+	DriverName                   = "compute-domain.nvidia.com"
+	defaultMaxNodesPerIMEXDomain = 18
 )
 
 type Flags struct {
 	kubeClientConfig  flags.KubeClientConfig
 	loggingConfig     *flags.LoggingConfig
 	featureGateConfig *flags.FeatureGateConfig
 
-	podName   string
-	namespace string
-	imageName string
+	podName               string
+	namespace             string
+	imageName             string
+	maxNodesPerIMEXDomain int
 
 	httpEndpoint string
 	metricsPath  string
@@ -103,6 +105,13 @@ func newApp() *cli.App {
 			Destination: &flags.imageName,
 			EnvVars:     []string{"IMAGE_NAME"},
 		},
+		&cli.IntFlag{
+			Name:        "max-nodes-per-imex-domain",
+			Usage:       "The maximum number of possible nodes per IMEX domain",
+			Value:       defaultMaxNodesPerIMEXDomain,
+			EnvVars:     []string{"MAX_NODES_PER_IMEX_DOMAIN"},
+			Destination: &flags.maxNodesPerIMEXDomain,
+		},
 		&cli.StringFlag{
 			Category:    "HTTP server:",
 			Name:        "http-endpoint",

cmd/compute-domain-daemon/hostname.go

@@ -29,25 +29,26 @@ import (
 )
 
 const (
-	maxHostnames   = 18
 	hostsFilePath  = "/etc/hosts"
 	hostnameFormat = "compute-domain-daemon-%d"
 )
 
 // HostnameManager manages the allocation of static hostnames to IP addresses.
 type HostnameManager struct {
 	sync.Mutex
-	ipToHostname    map[string]string
-	cliqueID        string
-	nodesConfigPath string
+	ipToHostname          map[string]string
+	cliqueID              string
+	maxNodesPerIMEXDomain int
+	nodesConfigPath       string
 }
 
 // NewHostnameManager creates a new hostname manager.
-func NewHostnameManager(cliqueID string, nodesConfigPath string) *HostnameManager {
+func NewHostnameManager(cliqueID string, maxNodesPerIMEXDomain int, nodesConfigPath string) *HostnameManager {
 	return &HostnameManager{
-		ipToHostname:    make(map[string]string),
-		cliqueID:        cliqueID,
-		nodesConfigPath: nodesConfigPath,
+		ipToHostname:          make(map[string]string),
+		cliqueID:              cliqueID,
+		maxNodesPerIMEXDomain: maxNodesPerIMEXDomain,
+		nodesConfigPath:       nodesConfigPath,
 	}
 }
 
@@ -118,7 +119,7 @@ func (m *HostnameManager) allocateHostname(ip string) (string, error) {
 	}
 
 	// Find the next available hostname
-	for i := 0; i < maxHostnames; i++ {
+	for i := 0; i < m.maxNodesPerIMEXDomain; i++ {
 		hostname := fmt.Sprintf(hostnameFormat, i)
 		// Check if this hostname is already in use
 		inUse := false
@@ -135,7 +136,7 @@ func (m *HostnameManager) allocateHostname(ip string) (string, error) {
 	}
 
 	// If all hostnames are used, return an error
-	return "", fmt.Errorf("no hostnames available (max: %d)", maxHostnames)
+	return "", fmt.Errorf("no hostnames available (max: %d)", m.maxNodesPerIMEXDomain)
 }
 
 // updateHostsFile updates the /etc/hosts file with current IP to hostname mappings.
@@ -205,13 +206,13 @@ func (m *HostnameManager) WriteNodesConfig() error {
 	defer f.Close()
 
 	// Write static hostnames
-	for i := 0; i < maxHostnames; i++ {
+	for i := 0; i < m.maxNodesPerIMEXDomain; i++ {
 		hostname := fmt.Sprintf(hostnameFormat, i)
 		if _, err := fmt.Fprintf(f, "%s\n", hostname); err != nil {
 			return fmt.Errorf("failed to write to nodes config file: %w", err)
 		}
 	}
 
-	klog.Infof("Created static nodes config file with %d hostnames using format %s", maxHostnames, hostnameFormat)
+	klog.Infof("Created static nodes config file with %d hostnames using format %s", m.maxNodesPerIMEXDomain, hostnameFormat)
 	return nil
 }

cmd/compute-domain-daemon/main.go

@@ -51,6 +51,7 @@ type Flags struct {
 	computeDomainNamespace string
 	nodeName               string
 	podIP                  string
+	maxNodesPerIMEXDomain  int
 	loggingConfig          *flags.LoggingConfig
 	featureGateConfig      *flags.FeatureGateConfig
 }
@@ -124,6 +125,12 @@ func newApp() *cli.App {
 			EnvVars:     []string{"POD_IP"},
 			Destination: &flags.podIP,
 		},
+		&cli.IntFlag{
+			Name:        "max-nodes-per-imex-domain",
+			Usage:       "The maximum number of possible nodes per IMEX domain",
+			EnvVars:     []string{"MAX_NODES_PER_IMEX_DOMAIN"},
+			Destination: &flags.maxNodesPerIMEXDomain,
+		},
 	}
 	cliFlags = append(cliFlags, flags.featureGateConfig.Flags()...)
 	cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)
@@ -179,7 +186,7 @@ func run(ctx context.Context, cancel context.CancelFunc, flags *Flags) error {
 	klog.Infof("config: %v", config)
 
 	// Prepare Hostname manager
-	hostnameManager := NewHostnameManager(flags.cliqueID, nodesConfigPath)
+	hostnameManager := NewHostnameManager(flags.cliqueID, flags.maxNodesPerIMEXDomain, nodesConfigPath)
 
 	// Create static nodes config file with hostnames
 	if err := hostnameManager.WriteNodesConfig(); err != nil {

templates/compute-domain-daemon.tmpl.yaml

@@ -26,6 +26,8 @@ spec:
         image: {{ .ImageName }}
         command: ["compute-domain-daemon", "-v", "6", "run"]
         env:
+        - name: MAX_NODES_PER_IMEX_DOMAIN
+          value: "{{ .MaxNodesPerIMEXDomain }}"
         - name: NODE_NAME
           valueFrom:
             fieldRef: