Skip to content

Commit 5f5da39

Browse files
cdesiniotiscdesiniotis
committed
Use native CDI support in runtimes for workload containers when cdi.enabled=true
This commit updates the default behavior when cdi.enabled=true. We now leverage native CDI support in containerd / cri-o to inject GPU devices into workload containers. This means we no longer configure 'nvidia' as the default runtime. Our management containers will continue to use the 'nvidia' runtime to access GPUs by explicitly setting runtimeClassName=nvidia in their pod specs. Signed-off-by: Christopher Desiniotis <[email protected]>
1 parent 72c2e8a commit 5f5da39

File tree

2 files changed

+172
-16
lines changed

2 files changed

+172
-16
lines changed

‎controllers/object_controls.go‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ const (
178178
// DriverInstallDirCtrPathEnvName is the name of the envvar used by the driver-validator to represent the path
179179
// of the driver install dir mounted in the container
180180
DriverInstallDirCtrPathEnvName = "DRIVER_INSTALL_DIR_CTR_PATH"
181+
// NvidiaRuntimeSetAsDefaultEnvName is the name of the toolkit container env for configuring NVIDIA Container Runtime as the default runtime
182+
NvidiaRuntimeSetAsDefaultEnvName = "NVIDIA_RUNTIME_SET_AS_DEFAULT"
181183
)
182184

183185
// ContainerProbe defines container probe types
@@ -1223,6 +1225,24 @@ func getProxyEnv(proxyConfig *apiconfigv1.Proxy) []corev1.EnvVar {
12231225
return envVars
12241226
}
12251227

1228+
func transformToolkitForCDI(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) {
1229+
if !config.CDI.IsEnabled() {
1230+
return
1231+
}
1232+
1233+
// When CDI is enabled in GPU Operator, we leverage native CDI support in containerd / cri-o
1234+
// to inject GPUs into workloads. We do not configure 'nvidia' as the default runtime. The
1235+
// 'nvidia' runtime will be set as the runtime class for our management containers so that
1236+
// they get access to all GPUs.
1237+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
1238+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CrioConfigModeEnvName, "config")
1239+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaRuntimeSetAsDefaultEnvName, "false")
1240+
1241+
if config.CDI.IsDefault() {
1242+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCtrRuntimeModeEnvName, "cdi")
1243+
}
1244+
}
1245+
12261246
// TransformToolkit transforms Nvidia container-toolkit daemonset with required config as per ClusterPolicy
12271247
func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
12281248
// update validation container
@@ -1255,14 +1275,7 @@ func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n
12551275
}
12561276

12571277
// update env required for CDI support
1258-
if config.CDI.IsEnabled() {
1259-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
1260-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCtrRuntimeCDIPrefixesEnvName, "nvidia.cdi.k8s.io/")
1261-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CrioConfigModeEnvName, "config")
1262-
if config.CDI.IsDefault() {
1263-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCtrRuntimeModeEnvName, "cdi")
1264-
}
1265-
}
1278+
transformToolkitForCDI(obj, config, n)
12661279

12671280
// set install directory for the toolkit
12681281
if config.Toolkit.InstallDir != "" && config.Toolkit.InstallDir != DefaultToolkitInstallDir {
@@ -1386,6 +1399,20 @@ func transformForRuntime(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
13861399
return nil
13871400
}
13881401

1402+
func transformDevicePluginForCDI(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) {
1403+
if !config.CDI.IsEnabled() {
1404+
return
1405+
}
1406+
1407+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
1408+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DeviceListStrategyEnvName, "cdi-annotations,cdi-cri")
1409+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIAnnotationPrefixEnvName, "cdi.k8s.io/")
1410+
1411+
if config.Toolkit.IsEnabled() {
1412+
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCDIHookPathEnvName, filepath.Join(config.Toolkit.InstallDir, "toolkit/nvidia-cdi-hook"))
1413+
}
1414+
}
1415+
13891416
// TransformDevicePlugin transforms k8s-device-plugin daemonset with required config as per ClusterPolicy
13901417
func TransformDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
13911418
// update validation container
@@ -1441,14 +1468,7 @@ func TransformDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
14411468
applyMIGConfiguration(&(obj.Spec.Template.Spec.Containers[0]), config.MIG.Strategy)
14421469

14431470
// update env required for CDI support
1444-
if config.CDI.IsEnabled() {
1445-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIEnabledEnvName, "true")
1446-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DeviceListStrategyEnvName, "envvar,cdi-annotations")
1447-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), CDIAnnotationPrefixEnvName, "nvidia.cdi.k8s.io/")
1448-
if config.Toolkit.IsEnabled() {
1449-
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCDIHookPathEnvName, filepath.Join(config.Toolkit.InstallDir, "toolkit/nvidia-cdi-hook"))
1450-
}
1451-
}
1471+
transformDevicePluginForCDI(obj, config, n)
14521472

14531473
// update MPS volumes and set MPS_ROOT env var if a custom MPS root is configured
14541474
if config.DevicePlugin.MPS != nil && config.DevicePlugin.MPS.Root != "" &&

‎controllers/transforms_test.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1571,3 +1571,139 @@ func TestTransformNodeStatusExporter(t *testing.T) {
15711571
})
15721572
}
15731573
}
1574+
1575+
func TestTransformToolkitForCDI(t *testing.T) {
1576+
testCases := []struct {
1577+
description string
1578+
ds Daemonset
1579+
cpSpec *gpuv1.ClusterPolicySpec
1580+
expectedDs Daemonset
1581+
}{
1582+
{
1583+
description: "cdi disabled",
1584+
ds: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1585+
cpSpec: &gpuv1.ClusterPolicySpec{
1586+
CDI: gpuv1.CDIConfigSpec{
1587+
Enabled: newBoolPtr(false),
1588+
},
1589+
},
1590+
expectedDs: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1591+
},
1592+
{
1593+
description: "cdi enabled",
1594+
ds: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1595+
cpSpec: &gpuv1.ClusterPolicySpec{
1596+
CDI: gpuv1.CDIConfigSpec{
1597+
Enabled: newBoolPtr(true),
1598+
},
1599+
},
1600+
expectedDs: NewDaemonset().WithContainer(
1601+
corev1.Container{
1602+
Name: "main-ctr",
1603+
Env: []corev1.EnvVar{
1604+
{Name: CDIEnabledEnvName, Value: "true"},
1605+
{Name: CrioConfigModeEnvName, Value: "config"},
1606+
{Name: NvidiaRuntimeSetAsDefaultEnvName, Value: "false"},
1607+
},
1608+
}),
1609+
},
1610+
{
1611+
description: "cdi enabled and cdi default",
1612+
ds: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1613+
cpSpec: &gpuv1.ClusterPolicySpec{
1614+
CDI: gpuv1.CDIConfigSpec{
1615+
Enabled: newBoolPtr(true),
1616+
Default: newBoolPtr(true),
1617+
},
1618+
},
1619+
expectedDs: NewDaemonset().WithContainer(
1620+
corev1.Container{
1621+
Name: "main-ctr",
1622+
Env: []corev1.EnvVar{
1623+
{Name: CDIEnabledEnvName, Value: "true"},
1624+
{Name: CrioConfigModeEnvName, Value: "config"},
1625+
{Name: NvidiaRuntimeSetAsDefaultEnvName, Value: "false"},
1626+
{Name: NvidiaCtrRuntimeModeEnvName, Value: "cdi"},
1627+
},
1628+
}),
1629+
},
1630+
}
1631+
1632+
for _, tc := range testCases {
1633+
t.Run(tc.description, func(t *testing.T) {
1634+
transformToolkitForCDI(tc.ds.DaemonSet, tc.cpSpec, ClusterPolicyController{logger: ctrl.Log.WithName("test")})
1635+
require.EqualValues(t, tc.expectedDs, tc.ds)
1636+
})
1637+
}
1638+
}
1639+
1640+
func TestTransformDevicePluginForCDI(t *testing.T) {
1641+
testCases := []struct {
1642+
description string
1643+
ds Daemonset
1644+
cpSpec *gpuv1.ClusterPolicySpec
1645+
expectedDs Daemonset
1646+
}{
1647+
{
1648+
description: "cdi disabled",
1649+
ds: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1650+
cpSpec: &gpuv1.ClusterPolicySpec{
1651+
CDI: gpuv1.CDIConfigSpec{
1652+
Enabled: newBoolPtr(false),
1653+
},
1654+
},
1655+
expectedDs: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1656+
},
1657+
{
1658+
description: "cdi enabled, toolkit disabled",
1659+
ds: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1660+
cpSpec: &gpuv1.ClusterPolicySpec{
1661+
CDI: gpuv1.CDIConfigSpec{
1662+
Enabled: newBoolPtr(true),
1663+
},
1664+
Toolkit: gpuv1.ToolkitSpec{
1665+
Enabled: newBoolPtr(false),
1666+
},
1667+
},
1668+
expectedDs: NewDaemonset().WithContainer(
1669+
corev1.Container{
1670+
Name: "main-ctr",
1671+
Env: []corev1.EnvVar{
1672+
{Name: CDIEnabledEnvName, Value: "true"},
1673+
{Name: DeviceListStrategyEnvName, Value: "cdi-annotations,cdi-cri"},
1674+
{Name: CDIAnnotationPrefixEnvName, Value: "cdi.k8s.io/"},
1675+
},
1676+
}),
1677+
},
1678+
{
1679+
description: "cdi enabled, toolkit enabled",
1680+
ds: NewDaemonset().WithContainer(corev1.Container{Name: "main-ctr"}),
1681+
cpSpec: &gpuv1.ClusterPolicySpec{
1682+
CDI: gpuv1.CDIConfigSpec{
1683+
Enabled: newBoolPtr(true),
1684+
},
1685+
Toolkit: gpuv1.ToolkitSpec{
1686+
Enabled: newBoolPtr(true),
1687+
InstallDir: "/path/to/install",
1688+
},
1689+
},
1690+
expectedDs: NewDaemonset().WithContainer(
1691+
corev1.Container{
1692+
Name: "main-ctr",
1693+
Env: []corev1.EnvVar{
1694+
{Name: CDIEnabledEnvName, Value: "true"},
1695+
{Name: DeviceListStrategyEnvName, Value: "cdi-annotations,cdi-cri"},
1696+
{Name: CDIAnnotationPrefixEnvName, Value: "cdi.k8s.io/"},
1697+
{Name: NvidiaCDIHookPathEnvName, Value: "/path/to/install/toolkit/nvidia-cdi-hook"},
1698+
},
1699+
}),
1700+
},
1701+
}
1702+
1703+
for _, tc := range testCases {
1704+
t.Run(tc.description, func(t *testing.T) {
1705+
transformDevicePluginForCDI(tc.ds.DaemonSet, tc.cpSpec, ClusterPolicyController{logger: ctrl.Log.WithName("test")})
1706+
require.EqualValues(t, tc.expectedDs, tc.ds)
1707+
})
1708+
}
1709+
}

0 commit comments

Comments
 (0)