Skip to content

Commit 8ad9c6a

Browse files
elezarelezar
committed
[no-relnote] Add basic cdi-enabled tests
Signed-off-by: Evan Lezar <[email protected]>
1 parent b36bd8a commit 8ad9c6a

File tree

1 file changed

+297
-2
lines changed

1 file changed

+297
-2
lines changed

‎cmd/nvidia-ctk-installer/main_test.go‎

Lines changed: 297 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ func TestApp(t *testing.T) {
9898
require.NoError(t, err)
9999

100100
artifactRoot := filepath.Join(moduleRoot, "testdata", "installer", "artifacts")
101+
hostRoot := filepath.Join(moduleRoot, "testdata", "lookup", "rootfs-1")
101102

102103
testCases := []struct {
103104
description string
@@ -165,12 +166,303 @@ swarm-resource = ""
165166
}
166167
}`,
167168
},
169+
{
170+
description: "CDI enabled enables CDI in docker",
171+
args: []string{"--cdi-enabled", "--create-device-nodes=none"},
172+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
173+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
174+
disable-require = false
175+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
176+
swarm-resource = ""
177+
178+
[nvidia-container-cli]
179+
debug = ""
180+
environment = []
181+
ldcache = ""
182+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
183+
load-kmods = true
184+
no-cgroups = false
185+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
186+
root = "/run/nvidia/driver"
187+
user = ""
188+
189+
[nvidia-container-runtime]
190+
debug = "/dev/null"
191+
log-level = "info"
192+
mode = "auto"
193+
runtimes = ["docker-runc", "runc", "crun"]
194+
195+
[nvidia-container-runtime.modes]
196+
197+
[nvidia-container-runtime.modes.cdi]
198+
annotation-prefixes = ["cdi.k8s.io/"]
199+
default-kind = "nvidia.com/gpu"
200+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
201+
202+
[nvidia-container-runtime.modes.csv]
203+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
204+
205+
[nvidia-container-runtime-hook]
206+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
207+
skip-mode-detection = true
208+
209+
[nvidia-ctk]
210+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
211+
`,
212+
expectedRuntimeConfig: `{
213+
"default-runtime": "nvidia",
214+
"features": {
215+
"cdi": true
216+
},
217+
"runtimes": {
218+
"nvidia": {
219+
"args": [],
220+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
221+
},
222+
"nvidia-cdi": {
223+
"args": [],
224+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
225+
},
226+
"nvidia-legacy": {
227+
"args": [],
228+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
229+
}
230+
}
231+
}`,
232+
},
233+
{
234+
description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in Docker",
235+
args: []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false"},
236+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
237+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
238+
disable-require = false
239+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
240+
swarm-resource = ""
241+
242+
[nvidia-container-cli]
243+
debug = ""
244+
environment = []
245+
ldcache = ""
246+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
247+
load-kmods = true
248+
no-cgroups = false
249+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
250+
root = "/run/nvidia/driver"
251+
user = ""
252+
253+
[nvidia-container-runtime]
254+
debug = "/dev/null"
255+
log-level = "info"
256+
mode = "auto"
257+
runtimes = ["docker-runc", "runc", "crun"]
258+
259+
[nvidia-container-runtime.modes]
260+
261+
[nvidia-container-runtime.modes.cdi]
262+
annotation-prefixes = ["cdi.k8s.io/"]
263+
default-kind = "nvidia.com/gpu"
264+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
265+
266+
[nvidia-container-runtime.modes.csv]
267+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
268+
269+
[nvidia-container-runtime-hook]
270+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
271+
skip-mode-detection = true
272+
273+
[nvidia-ctk]
274+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
275+
`,
276+
expectedRuntimeConfig: `{
277+
"default-runtime": "nvidia",
278+
"runtimes": {
279+
"nvidia": {
280+
"args": [],
281+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
282+
},
283+
"nvidia-cdi": {
284+
"args": [],
285+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
286+
},
287+
"nvidia-legacy": {
288+
"args": [],
289+
"path": "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
290+
}
291+
}
292+
}`,
293+
},
294+
{
295+
description: "CDI enabled enables CDI in containerd",
296+
args: []string{"--cdi-enabled", "--runtime=containerd"},
297+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
298+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
299+
disable-require = false
300+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
301+
swarm-resource = ""
302+
303+
[nvidia-container-cli]
304+
debug = ""
305+
environment = []
306+
ldcache = ""
307+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
308+
load-kmods = true
309+
no-cgroups = false
310+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
311+
root = "/run/nvidia/driver"
312+
user = ""
313+
314+
[nvidia-container-runtime]
315+
debug = "/dev/null"
316+
log-level = "info"
317+
mode = "auto"
318+
runtimes = ["docker-runc", "runc", "crun"]
319+
320+
[nvidia-container-runtime.modes]
321+
322+
[nvidia-container-runtime.modes.cdi]
323+
annotation-prefixes = ["cdi.k8s.io/"]
324+
default-kind = "nvidia.com/gpu"
325+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
326+
327+
[nvidia-container-runtime.modes.csv]
328+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
329+
330+
[nvidia-container-runtime-hook]
331+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
332+
skip-mode-detection = true
333+
334+
[nvidia-ctk]
335+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
336+
`,
337+
expectedRuntimeConfig: `version = 2
338+
339+
[plugins]
340+
341+
[plugins."io.containerd.grpc.v1.cri"]
342+
enable_cdi = true
343+
344+
[plugins."io.containerd.grpc.v1.cri".containerd]
345+
default_runtime_name = "nvidia"
346+
347+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
348+
349+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
350+
privileged_without_host_devices = false
351+
runtime_engine = ""
352+
runtime_root = ""
353+
runtime_type = "io.containerd.runc.v2"
354+
355+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
356+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
357+
358+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
359+
privileged_without_host_devices = false
360+
runtime_engine = ""
361+
runtime_root = ""
362+
runtime_type = "io.containerd.runc.v2"
363+
364+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
365+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
366+
367+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
368+
privileged_without_host_devices = false
369+
runtime_engine = ""
370+
runtime_root = ""
371+
runtime_type = "io.containerd.runc.v2"
372+
373+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
374+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
375+
`,
376+
},
377+
{
378+
description: "--enable-cdi-in-runtime=false overrides --cdi-enabled in containerd",
379+
args: []string{"--cdi-enabled", "--create-device-nodes=none", "--enable-cdi-in-runtime=false", "--runtime=containerd"},
380+
expectedToolkitConfig: `accept-nvidia-visible-devices-as-volume-mounts = false
381+
accept-nvidia-visible-devices-envvar-when-unprivileged = true
382+
disable-require = false
383+
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
384+
swarm-resource = ""
385+
386+
[nvidia-container-cli]
387+
debug = ""
388+
environment = []
389+
ldcache = ""
390+
ldconfig = "@/run/nvidia/driver/sbin/ldconfig"
391+
load-kmods = true
392+
no-cgroups = false
393+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-cli"
394+
root = "/run/nvidia/driver"
395+
user = ""
396+
397+
[nvidia-container-runtime]
398+
debug = "/dev/null"
399+
log-level = "info"
400+
mode = "auto"
401+
runtimes = ["docker-runc", "runc", "crun"]
402+
403+
[nvidia-container-runtime.modes]
404+
405+
[nvidia-container-runtime.modes.cdi]
406+
annotation-prefixes = ["cdi.k8s.io/"]
407+
default-kind = "nvidia.com/gpu"
408+
spec-dirs = ["/etc/cdi", "/var/run/cdi"]
409+
410+
[nvidia-container-runtime.modes.csv]
411+
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
412+
413+
[nvidia-container-runtime-hook]
414+
path = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime-hook"
415+
skip-mode-detection = true
416+
417+
[nvidia-ctk]
418+
path = "{{ .toolkitRoot }}/toolkit/nvidia-ctk"
419+
`,
420+
expectedRuntimeConfig: `version = 2
421+
422+
[plugins]
423+
424+
[plugins."io.containerd.grpc.v1.cri"]
425+
426+
[plugins."io.containerd.grpc.v1.cri".containerd]
427+
default_runtime_name = "nvidia"
428+
429+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
430+
431+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
432+
privileged_without_host_devices = false
433+
runtime_engine = ""
434+
runtime_root = ""
435+
runtime_type = "io.containerd.runc.v2"
436+
437+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
438+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime"
439+
440+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
441+
privileged_without_host_devices = false
442+
runtime_engine = ""
443+
runtime_root = ""
444+
runtime_type = "io.containerd.runc.v2"
445+
446+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
447+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.cdi"
448+
449+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
450+
privileged_without_host_devices = false
451+
runtime_engine = ""
452+
runtime_root = ""
453+
runtime_type = "io.containerd.runc.v2"
454+
455+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
456+
BinaryName = "{{ .toolkitRoot }}/toolkit/nvidia-container-runtime.legacy"
457+
`,
458+
},
168459
}
169460

170461
for _, tc := range testCases {
171462
t.Run(tc.description, func(t *testing.T) {
172463
testRoot := t.TempDir()
173464

465+
cdiOutputDir := filepath.Join(testRoot, "/var/run/cdi")
174466
runtimeConfigFile := filepath.Join(testRoot, "config.file")
175467

176468
toolkitRoot := filepath.Join(testRoot, "toolkit-test")
@@ -181,10 +473,13 @@ swarm-resource = ""
181473
testArgs := []string{
182474
"nvidia-ctk-installer",
183475
"--no-daemon",
184-
"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
185-
"--source-root=" + filepath.Join(artifactRoot, "deb"),
476+
"--cdi-output-dir=" + cdiOutputDir,
186477
"--config=" + runtimeConfigFile,
478+
"--create-device-nodes=none",
479+
"--driver-root-ctr-path=" + hostRoot,
480+
"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
187481
"--restart-mode=none",
482+
"--source-root=" + filepath.Join(artifactRoot, "deb"),
188483
}
189484

190485
err := app.Run(append(testArgs, tc.args...))

0 commit comments

Comments
 (0)