Support for the DevicePluginCDIDevices feature

This patch adds support for the `DevicePluginCDIDevices` feature gate by
adding `spec.operator.useDevicePluginCDIDevicesFeature` to
`ClusterPolicy`.  When this field is set, the operator sets the
`DEVICE_LIST_STRATEGY` device plug-in environment variable to `cdi-cri`.

Signed-off-by: Jean-Francois Roy <[email protected]>

Author: jfroy

api/nvidia/v1/clusterpolicy_types.go

@@ -148,6 +148,9 @@ type OperatorSpec struct {
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="On OpenShift, enable DriverToolkit image to build and install driver modules"
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
 	UseOpenShiftDriverToolkit *bool `json:"use_ocp_driver_toolkit,omitempty"`
+
+	// UseDevicePluginCDIDevicesFeature indicates if the device plug-in should be configured to use the CDI devices feature
+	UseDevicePluginCDIDevicesFeature *bool `json:"useDevicePluginCDIDevicesFeature,omitempty"`
 }
 
 // HostPathsSpec defines various paths on the host needed by GPU Operator components
@@ -1827,6 +1830,15 @@ func ImagePullPolicy(pullPolicy string) corev1.PullPolicy {
 	return imagePullPolicy
 }
 
+// DevicePluginCDIDevicesFeatureEnabled returns true if use DevicePluginCDIDevices feature is enabled
+func (s *OperatorSpec) DevicePluginCDIDevicesFeatureEnabled() bool {
+	if s.UseDevicePluginCDIDevicesFeature == nil {
+		// default is false if not specified by user
+		return false
+	}
+	return *s.UseDevicePluginCDIDevicesFeature
+}
+
 // IsEnabled returns true if driver install is enabled(default) through gpu-operator
 func (d *DriverSpec) IsEnabled() bool {
 	if d.Enabled == nil {

api/nvidia/v1/zz_generated.deepcopy.go

@@ -1558,6 +1558,10 @@ spec:
                       image should be used on OpenShift to build and install driver
                       modules
                     type: boolean
+                  useDevicePluginCDIDevicesFeature:
+                    description: UseDevicePluginCDIDevicesFeature indicates if the device plug-in
+                      should be configured to use the CDI devices feature
+                    type: boolean
                 required:
                 - defaultRuntime
                 type: object

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -1558,6 +1558,10 @@ spec:
                       image should be used on OpenShift to build and install driver
                       modules
                     type: boolean
+                  useDevicePluginCDIDevicesFeature:
+                    description: UseDevicePluginCDIDevicesFeature indicates if the
+                      device plug-in should be configured to use the CDI devices feature
+                    type: boolean
                 required:
                 - defaultRuntime
                 type: object

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -1388,7 +1388,11 @@ func TransformDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 	// update env required for CDI support
 	if config.CDI.IsEnabled() {
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
-		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DeviceListStrategyEnvName, "envvar,cdi-annotations")
+		if config.Operator.DevicePluginCDIDevicesFeatureEnabled() {
+			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DeviceListStrategyEnvName, "cdi-cri")
+		} else {
+			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DeviceListStrategyEnvName, "envvar,cdi-annotations")
+		}
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIAnnotationPrefixEnvName, "nvidia.cdi.k8s.io/")
 		if config.Toolkit.IsEnabled() {
 			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCDIHookPathEnvName, filepath.Join(config.Toolkit.InstallDir, "toolkit/nvidia-cdi-hook"))

controllers/object_controls.go

@@ -1558,6 +1558,10 @@ spec:
                       image should be used on OpenShift to build and install driver
                       modules
                     type: boolean
+                  useDevicePluginCDIDevicesFeature:
+                    description: UseDevicePluginCDIDevicesFeature indicates if the device plug-in
+                      should be configured to use the CDI devices feature
+                    type: boolean
                 required:
                 - defaultRuntime
                 type: object

deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml

@@ -46,6 +46,9 @@ spec:
     {{- if .Values.operator.use_ocp_driver_toolkit }}
     use_ocp_driver_toolkit: {{ .Values.operator.use_ocp_driver_toolkit }}
     {{- end }}
+    {{- if .Values.operator.useDevicePluginCDIDevicesFeature }}
+    useDevicePluginCDIDevicesFeature: {{ .Values.operator.useDevicePluginCDIDevicesFeature }}
+    {{- end }}
   daemonsets:
     labels:
       {{- include "gpu-operator.operand-labels" . | nindent 6 }}

deployments/gpu-operator/templates/clusterpolicy.yaml

@@ -80,6 +80,8 @@ operator:
   # upgrade CRD on chart upgrade, requires --disable-openapi-validation flag
   # to be passed during helm upgrade.
   upgradeCRD: false
+  # use DevicePluginCDIDevices feature
+  useDevicePluginCDIDevicesFeature: false
   initContainer:
     image: cuda
     repository: nvcr.io/nvidia