Skip to content

Commit a1e4b50

Browse files
elezarelezar
committed
[no-relnote] Add basic cdi-enabled tests
Signed-off-by: Evan Lezar <[email protected]>
1 parent 751604f commit a1e4b50

File tree

1 file changed

+297
-2
lines changed

1 file changed

+297
-2
lines changed

‎tools/container/nvidia-toolkit/run_test.go‎

Lines changed: 297 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ func TestApp(t *testing.T) {
9696
require.NoError(t, err)
9797

9898
artifactRoot := filepath.Join(moduleRoot, "testdata", "installer", "artifacts")
99+
hostRoot := filepath.Join(moduleRoot, "testdata", "lookup", "rootfs-1")
99100

100101
testCases := []struct {
101102
description string
@@ -163,12 +164,303 @@ swarm-resource = ""
163164
}
164165
}`,
165166
},
167+
{
168+
description: "CDI enabled enables CDI in docker",
169+
args: []string{"--cdi-enabled", "--create-device-nodes=none"},
170+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
171+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
172+
disable-require = false
173+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
174+
swarm-resource = ""
175+
176+
[nvidia-container-cli]
177+
debug = ""
178+
environment = []
179+
ldcache = ""
180+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
181+
load-kmods = true
182+
no-cgroups = false
183+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
184+
root = "/run/nvidia/driver"
185+
user = ""
186+
187+
[nvidia-container-runtime]
188+
debug = "/dev/null"
189+
log-level = "info"
190+
mode = "auto"
191+
runtimes = ["docker-runc", "runc", "crun"]
192+
193+
[nvidia-container-runtime.modes]
194+
195+
[nvidia-container-runtime.modes.cdi]
196+
annotation-prefixes = ["cdi.k8s.io/"]
197+
default-kind = "nvidia.com/gpu"
198+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
199+
200+
[nvidia-container-runtime.modes.csv]
201+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
202+
203+
[nvidia-container-runtime-hook]
204+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
205+
skip-mode-detection = true
206+
207+
[nvidia-ctk]
208+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
209+
`,
210+
expectedRuntimeConfig: `{
211+
"default-runtime": "nvidia",
212+
"features": {
213+
"cdi": true
214+
},
215+
"runtimes": {
216+
"nvidia": {
217+
"args": [],
218+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
219+
},
220+
"nvidia-cdi": {
221+
"args": [],
222+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
223+
},
224+
"nvidia-legacy": {
225+
"args": [],
226+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
227+
}
228+
}
229+
}`,
230+
},
231+
{
232+
description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in Docker",
233+
args: []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false"},
234+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
235+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
236+
disable-require = false
237+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
238+
swarm-resource = ""
239+
240+
[nvidia-container-cli]
241+
debug = ""
242+
environment = []
243+
ldcache = ""
244+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
245+
load-kmods = true
246+
no-cgroups = false
247+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
248+
root = "/run/nvidia/driver"
249+
user = ""
250+
251+
[nvidia-container-runtime]
252+
debug = "/dev/null"
253+
log-level = "info"
254+
mode = "auto"
255+
runtimes = ["docker-runc", "runc", "crun"]
256+
257+
[nvidia-container-runtime.modes]
258+
259+
[nvidia-container-runtime.modes.cdi]
260+
annotation-prefixes = ["cdi.k8s.io/"]
261+
default-kind = "nvidia.com/gpu"
262+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
263+
264+
[nvidia-container-runtime.modes.csv]
265+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
266+
267+
[nvidia-container-runtime-hook]
268+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
269+
skip-mode-detection = true
270+
271+
[nvidia-ctk]
272+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
273+
`,
274+
expectedRuntimeConfig: `{
275+
"default-runtime": "nvidia",
276+
"runtimes": {
277+
"nvidia": {
278+
"args": [],
279+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
280+
},
281+
"nvidia-cdi": {
282+
"args": [],
283+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
284+
},
285+
"nvidia-legacy": {
286+
"args": [],
287+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
288+
}
289+
}
290+
}`,
291+
},
292+
{
293+
description: "CDI enabled enables CDI in containerd",
294+
args: []string{"--cdi-enabled", "--runtime=containerd"},
295+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
296+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
297+
disable-require = false
298+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
299+
swarm-resource = ""
300+
301+
[nvidia-container-cli]
302+
debug = ""
303+
environment = []
304+
ldcache = ""
305+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
306+
load-kmods = true
307+
no-cgroups = false
308+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
309+
root = "/run/nvidia/driver"
310+
user = ""
311+
312+
[nvidia-container-runtime]
313+
debug = "/dev/null"
314+
log-level = "info"
315+
mode = "auto"
316+
runtimes = ["docker-runc", "runc", "crun"]
317+
318+
[nvidia-container-runtime.modes]
319+
320+
[nvidia-container-runtime.modes.cdi]
321+
annotation-prefixes = ["cdi.k8s.io/"]
322+
default-kind = "nvidia.com/gpu"
323+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
324+
325+
[nvidia-container-runtime.modes.csv]
326+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
327+
328+
[nvidia-container-runtime-hook]
329+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
330+
skip-mode-detection = true
331+
332+
[nvidia-ctk]
333+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
334+
`,
335+
expectedRuntimeConfig: `version = 2
336+
337+
[plugins]
338+
339+
[plugins."io.containerd.grpc.v1.cri"]
340+
enable_cdi = true
341+
342+
[plugins."io.containerd.grpc.v1.cri".containerd]
343+
default_runtime_name = "nvidia"
344+
345+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
346+
347+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
348+
privileged_without_host_devices = false
349+
runtime_engine = ""
350+
runtime_root = ""
351+
runtime_type = "io.containerd.runc.v2"
352+
353+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
354+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
355+
356+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
357+
privileged_without_host_devices = false
358+
runtime_engine = ""
359+
runtime_root = ""
360+
runtime_type = "io.containerd.runc.v2"
361+
362+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
363+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
364+
365+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
366+
privileged_without_host_devices = false
367+
runtime_engine = ""
368+
runtime_root = ""
369+
runtime_type = "io.containerd.runc.v2"
370+
371+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
372+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
373+
`,
374+
},
375+
{
376+
description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in containerd",
377+
args: []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false", "--runtime=containerd"},
378+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
379+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
380+
disable-require = false
381+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
382+
swarm-resource = ""
383+
384+
[nvidia-container-cli]
385+
debug = ""
386+
environment = []
387+
ldcache = ""
388+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
389+
load-kmods = true
390+
no-cgroups = false
391+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
392+
root = "/run/nvidia/driver"
393+
user = ""
394+
395+
[nvidia-container-runtime]
396+
debug = "/dev/null"
397+
log-level = "info"
398+
mode = "auto"
399+
runtimes = ["docker-runc", "runc", "crun"]
400+
401+
[nvidia-container-runtime.modes]
402+
403+
[nvidia-container-runtime.modes.cdi]
404+
annotation-prefixes = ["cdi.k8s.io/"]
405+
default-kind = "nvidia.com/gpu"
406+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
407+
408+
[nvidia-container-runtime.modes.csv]
409+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
410+
411+
[nvidia-container-runtime-hook]
412+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
413+
skip-mode-detection = true
414+
415+
[nvidia-ctk]
416+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
417+
`,
418+
expectedRuntimeConfig: `version = 2
419+
420+
[plugins]
421+
422+
[plugins."io.containerd.grpc.v1.cri"]
423+
424+
[plugins."io.containerd.grpc.v1.cri".containerd]
425+
default_runtime_name = "nvidia"
426+
427+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
428+
429+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
430+
privileged_without_host_devices = false
431+
runtime_engine = ""
432+
runtime_root = ""
433+
runtime_type = "io.containerd.runc.v2"
434+
435+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
436+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
437+
438+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
439+
privileged_without_host_devices = false
440+
runtime_engine = ""
441+
runtime_root = ""
442+
runtime_type = "io.containerd.runc.v2"
443+
444+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
445+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
446+
447+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
448+
privileged_without_host_devices = false
449+
runtime_engine = ""
450+
runtime_root = ""
451+
runtime_type = "io.containerd.runc.v2"
452+
453+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
454+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
455+
`,
456+
},
166457
}
167458

168459
for _, tc := range testCases {
169460
t.Run(tc.description, func(t *testing.T) {
170461
testRoot := t.TempDir()
171462

463+
cdiOutputDir := filepath.Join(testRoot, "/var/run/cdi")
172464
runtimeConfigFile := filepath.Join(testRoot, "config.file")
173465

174466
toolkitRoot := filepath.Join(testRoot, "toolkit-test")
@@ -179,10 +471,13 @@ swarm-resource = ""
179471
testArgs := []string{
180472
"nvidia-ctk-installer",
181473
"--no-daemon",
182-
"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
183-
"--source-root=" + filepath.Join(artifactRoot, "deb"),
474+
"--cdi-output-dir=" + cdiOutputDir,
184475
"--config=" + runtimeConfigFile,
476+
"--create-device-nodes=none",
477+
"--driver-root-ctr-path=" + hostRoot,
478+
"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
185479
"--restart-mode=none",
480+
"--source-root=" + filepath.Join(artifactRoot, "deb"),
186481
}
187482

188483
err := app.Run(append(testArgs, tc.args...))

0 commit comments

Comments
 (0)