api: add numNodes spec

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

Author: jgehrcke

api/nvidia.com/resource/v1beta1/computedomain.go

@@ -39,7 +39,10 @@ type ComputeDomain struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec   ComputeDomainSpec   `json:"spec,omitempty"`
+	Spec ComputeDomainSpec `json:"spec,omitempty"`
+	// Global ComputeDomain status. Updated only when `Spec.numNodes` is
+	// non-zero. Can be used to guide debugging efforts. Workload however should
+	// not rely on inspecting this field at any point during its lifecycle.
 	Status ComputeDomainStatus `json:"status,omitempty"`
 }
 
@@ -57,6 +60,17 @@ type ComputeDomainList struct {
 
 // ComputeDomainSpec provides the spec for a ComputeDomain.
 type ComputeDomainSpec struct {
+	// Intended number of IMEX daemons (i.e., individual compute nodes) in the
+	// ComputeDomain. Must be zero or greater. Recommended to be set to zero:
+	// workload must implement and consult its own source of truth for the
+	// number of workers online before trying to share GPU memory (and hence
+	// triggering IMEX interaction). When non-zero, `numNodes` is used only for
+	// setting the global ComputeDomain `Status` (indicating `Ready` when the
+	// number of ready IMEX daemons equals `numNodes`). In particular,
+	// `numNodes` does not gate the startup of IMEX daemons and their
+	// corresponding workload pods anymore (to restore this behavior, set
+	// `featureGates.IMEXDaemonsWithDNSNames=false`). This parameter is
+	// deprecated and will be removed in the next API version.
 	NumNodes int                       `json:"numNodes"`
 	Channel  *ComputeDomainChannelSpec `json:"channel"`
 }

deployments/helm/nvidia-dra-driver-gpu/crds/resource.nvidia.com_computedomains.yaml

@@ -66,6 +66,18 @@ spec:
                 - resourceClaimTemplate
                 type: object
               numNodes:
+                description: |-
+                  Intended number of IMEX daemons (i.e., individual compute nodes) in the
+                  ComputeDomain. Must be zero or greater. Recommended to be set to zero:
+                  workload must implement and consult its own source of truth for the
+                  number of workers online before trying to share GPU memory (and hence
+                  triggering IMEX interaction). When non-zero, `numNodes` is used only for
+                  setting the global ComputeDomain `Status` (indicating `Ready` when the
+                  number of ready IMEX daemons equals `numNodes`). In particular,
+                  `numNodes` does not gate the startup of IMEX daemons and their
+                  corresponding workload pods anymore (to restore this behavior, set
+                  `featureGates.IMEXDaemonsWithDNSNames=false`). This parameter is
+                  deprecated and will be removed in the next API version.
                 type: integer
             required:
             - channel
@@ -75,7 +87,10 @@ spec:
             - message: A computeDomain.spec is immutable
               rule: self == oldSelf
           status:
-            description: ComputeDomainStatus provides the status for a ComputeDomain.
+            description: |-
+              Global ComputeDomain status. Updated only when `Spec.numNodes` is
+              non-zero. Can be used to guide debugging efforts. Workload however should
+              not rely on inspecting this field at any point during its lifecycle.
             properties:
               nodes:
                 items: