From 46c5d59768430acceeb663c7e8d5de68efda8355 Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Arango Gutierrez Date: Mon, 8 Sep 2025 12:09:48 +0200 Subject: [PATCH 1/2] [no-relnote] Refactor cmd/installer/container/containerd unit test Signed-off-by: Carlos Eduardo Arango Gutierrez --- .../runtime/containerd/config_test.go | 1478 +++++++++++++++++ .../runtime/containerd/config_v1_test.go | 584 ------- .../runtime/containerd/config_v2_test.go | 520 ------ .../runtime/containerd/containerd.go | 4 +- .../container/runtime/crio/config_test.go | 609 +++++++ 5 files changed, 2089 insertions(+), 1106 deletions(-) create mode 100644 cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go delete mode 100644 cmd/nvidia-ctk-installer/container/runtime/containerd/config_v1_test.go delete mode 100644 cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go create mode 100644 cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go diff --git a/cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go b/cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go new file mode 100644 index 000000000..fecb8d020 --- /dev/null +++ b/cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go @@ -0,0 +1,1478 @@ +/* +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/ + +package containerd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + cli "github.com/urfave/cli/v3" + + "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container" +) + +// TestContainerdConfigLifecycle tests the complete Setup->Cleanup lifecycle for both v1 and v2 configs. +func TestContainerdConfigLifecycle(t *testing.T) { + c := &cli.Command{ + Name: "test", + } + + testCases := []struct { + description string + containerOptions container.Options + options Options + prepareEnvironment func(*testing.T, string) error + expectedSetupError error + assertSetupPostConditions func(*testing.T, string) error + expectedCleanupError error + assertCleanupPostConditions func(*testing.T, string) error + }{ + // V1 test cases + // Note: We don't test "v1: top-level config does not exist" because new configs + // are always created as v2 configs by the containerd package + { + description: "v1: existing config without nvidia runtime", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + EnableCDI: true, + SetAsDefault: false, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial v1 config + initialConfig := `version = 1 + +[plugins] + [plugins.cri] + [plugins.cri.containerd] + default_runtime_name = "runc" + + [plugins.cri.containerd.runtimes] + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + default_runtime_name = "runc" + enable_cdi = true + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.nvidia] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + Runtime = "/usr/bin/nvidia-container-runtime" + + [plugins.cri.containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + Runtime = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins.cri.containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + Runtime = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + default_runtime_name = "runc" + enable_cdi = true + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "v1: existing config with default_runtime_name and OPTIONS inheritance", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: true, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial v1 config without a default runtime set + // This tests OPTIONS inheritance from an existing runtime + initialConfig := `version = 1 + +[plugins] + [plugins.cri] + [plugins.cri.containerd] + [plugins.cri.containerd.runtimes] + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" + Root = "/run/containerd/runc" + ShimDebug = true + SystemdCgroup = true + NoPivotRoot = false +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + default_runtime_name = "nvidia" + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.nvidia] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + NoPivotRoot = false + Root = "/run/containerd/runc" + Runtime = "/usr/bin/nvidia-container-runtime" + ShimDebug = true + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + NoPivotRoot = false + Root = "/run/containerd/runc" + Runtime = "/usr/bin/nvidia-container-runtime.cdi" + ShimDebug = true + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + NoPivotRoot = false + Root = "/run/containerd/runc" + Runtime = "/usr/bin/nvidia-container-runtime.legacy" + ShimDebug = true + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + NoPivotRoot = false + Root = "/run/containerd/runc" + Runtime = "/usr/bin/runc" + ShimDebug = true + SystemdCgroup = true +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + NoPivotRoot = false + Root = "/run/containerd/runc" + Runtime = "/usr/bin/runc" + ShimDebug = true + SystemdCgroup = true +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "v1: OPTIONS inheritance from default runtime specified by default_runtime_name", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, // Don't change the default runtime + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Create config with default_runtime_name pointing to custom runtime + // This tests that OPTIONS are inherited from the default runtime + initialConfig := `version = 1 + +[plugins] + [plugins.cri] + [plugins.cri.containerd] + default_runtime_name = "custom" + + [plugins.cri.containerd.runtimes] + [plugins.cri.containerd.runtimes.custom] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.custom.options] + Runtime = "/usr/bin/custom-runtime" + Root = "/custom/root" + ShimDebug = false + SystemdCgroup = true + NoPivotRoot = true + CustomOption = "custom-value" + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Verify that nvidia runtimes inherit OPTIONS from the default runtime (custom) + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + default_runtime_name = "custom" + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.custom] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.custom.options] + CustomOption = "custom-value" + NoPivotRoot = true + Root = "/custom/root" + Runtime = "/usr/bin/custom-runtime" + ShimDebug = false + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.nvidia] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + CustomOption = "custom-value" + NoPivotRoot = true + Root = "/custom/root" + Runtime = "/usr/bin/nvidia-container-runtime" + ShimDebug = false + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + CustomOption = "custom-value" + NoPivotRoot = true + Root = "/custom/root" + Runtime = "/usr/bin/nvidia-container-runtime.cdi" + ShimDebug = false + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + CustomOption = "custom-value" + NoPivotRoot = true + Root = "/custom/root" + Runtime = "/usr/bin/nvidia-container-runtime.legacy" + ShimDebug = false + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // After cleanup, should return to original state + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + default_runtime_name = "custom" + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.custom] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.custom.options] + CustomOption = "custom-value" + NoPivotRoot = true + Root = "/custom/root" + Runtime = "/usr/bin/custom-runtime" + ShimDebug = false + SystemdCgroup = true + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "v1: existing config with default_runtime_name set and restoration on cleanup", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: true, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial v1 config with default_runtime_name set to "runc" + initialConfig := `version = 1 + +[plugins] + [plugins.cri] + [plugins.cri.containerd] + default_runtime_name = "runc" + + [plugins.cri.containerd.runtimes] + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + default_runtime_name = "nvidia" + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.nvidia] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + Runtime = "/usr/bin/nvidia-container-runtime" + + [plugins.cri.containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + Runtime = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins.cri.containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + Runtime = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Current implementation limitation: default_runtime_name is deleted entirely + // when the default runtime is removed, rather than being restored + expected := `version = 1 + +[plugins] + + [plugins.cri] + + [plugins.cri.containerd] + + [plugins.cri.containerd.runtimes] + + [plugins.cri.containerd.runtimes.runc] + runtime_type = "io.containerd.runtime.v1.linux" + + [plugins.cri.containerd.runtimes.runc.options] + Runtime = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + // V2 test cases + { + description: "v2: top-level config does not exist", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + privileged_without_host_devices = false + runtime_engine = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi] + privileged_without_host_devices = false + runtime_engine = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy] + privileged_without_host_devices = false + runtime_engine = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + // On cleanup, config should be deleted when starting from non-existent + require.NoFileExists(t, configPath, "Config file should be deleted if it didn't exist originally") + return nil + }, + }, + { + description: "v2: existing config without nvidia runtime and CDI enabled", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + EnableCDI: true, + SetAsDefault: false, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial config + initialConfig := `version = 2 + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + enable_cdi = true + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + enable_cdi = true + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "v2: existing config with nvidia runtime using old path already present", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: true, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + initialConfig := `version = 2 + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "nvidia" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/old/path/nvidia-container-runtime" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "nvidia" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + + require.FileExists(t, configPath) + + // If file exists, verify nvidia runtimes were removed + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + contentStr := string(actual) + + require.NotContains(t, contentStr, "nvidia") + require.NotContains(t, contentStr, "nvidia-cdi") + require.NotContains(t, contentStr, "nvidia-legacy") + return nil + }, + }, + { + description: "v2: complex config with multiple plugins and settings", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + EnableCDI: true, + SetAsDefault: false, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + ContainerRuntimeModesCDIAnnotationPrefixes: []string{"cdi.k8s.io"}, + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial complex config + initialConfig := `version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.custom] + runtime_type = "io.containerd.custom.v1" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.custom.options] + TypeUrl = "custom.runtime/options" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `root = "/var/lib/containerd" +state = "/run/containerd" +version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + enable_cdi = true + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.custom] + runtime_type = "io.containerd.custom.v1" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.custom.options] + TypeUrl = "custom.runtime/options" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + container_annotations = ["cdi.k8s.io*"] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi] + container_annotations = ["cdi.k8s.io*"] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy] + container_annotations = ["cdi.k8s.io*"] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".registry] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + + require.FileExists(t, configPath) + + // If file exists, verify nvidia runtimes were removed + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + contentStr := string(actual) + + require.NotContains(t, contentStr, "nvidia") + require.NotContains(t, contentStr, "nvidia-cdi") + require.NotContains(t, contentStr, "nvidia-legacy") + return nil + }, + }, + { + description: "v2: existing config without default runtime (SetAsDefault=false)", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial config without any default runtime + initialConfig := `version = 2 + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + + // Verify that default_runtime_name is NOT present + require.NotContains(t, string(actual), "default_runtime_name") + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + + // Verify that default_runtime_name is still NOT present + require.NotContains(t, string(actual), "default_runtime_name") + require.NotContains(t, string(actual), "nvidia") + return nil + }, + }, + { + description: "v2: existing config without default runtime (SetAsDefault=true)", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: true, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial config without any default runtime + initialConfig := `version = 2 + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + # Note: No default_runtime_name specified + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "nvidia" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 2 + +[plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + + return nil + }, + }, + // V3 test cases + { + description: "v3: minimal config without nvidia runtime", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write v3 config with runc runtime + initialConfig := `version = 3 + +[plugins] + [plugins."io.containerd.cri.v1.runtime"] + [plugins."io.containerd.cri.v1.runtime".containerd] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 3 + +[plugins] + + [plugins."io.containerd.cri.v1.runtime"] + + [plugins."io.containerd.cri.v1.runtime".containerd] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Should return to original v3 config with just runc + expected := `version = 3 + +[plugins] + + [plugins."io.containerd.cri.v1.runtime"] + + [plugins."io.containerd.cri.v1.runtime".containerd] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "v3: existing config with runtime and OPTIONS inheritance", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/containerd/config.toml", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: true, + RestartMode: "none", + ExecutablePath: "not-containerd", + }, + options: Options{ + runtimeType: "io.containerd.runc.v2", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + // Write initial v3 config with a runtime that has custom OPTIONS + initialConfig := `version = 3 + +[plugins] + [plugins."io.containerd.cri.v1.runtime"] + [plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "runc" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + NoPivotRoot = false + Root = "/run/containerd/runc" +` + require.NoError(t, os.WriteFile(configPath, []byte(initialConfig), 0600)) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := `version = 3 + +[plugins] + + [plugins."io.containerd.cri.v1.runtime"] + + [plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "nvidia" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + NoPivotRoot = false + Root = "/run/containerd/runc" + SystemdCgroup = true + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-cdi] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-cdi.options] + BinaryName = "/usr/bin/nvidia-container-runtime.cdi" + NoPivotRoot = false + Root = "/run/containerd/runc" + SystemdCgroup = true + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-legacy] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-legacy.options] + BinaryName = "/usr/bin/nvidia-container-runtime.legacy" + NoPivotRoot = false + Root = "/run/containerd/runc" + SystemdCgroup = true + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + NoPivotRoot = false + Root = "/run/containerd/runc" + SystemdCgroup = true +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/containerd/config.toml") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // After cleanup, should return to original state + expected := `version = 3 + +[plugins] + + [plugins."io.containerd.cri.v1.runtime"] + + [plugins."io.containerd.cri.v1.runtime".containerd] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + NoPivotRoot = false + Root = "/run/containerd/runc" + SystemdCgroup = true +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + // Create a temporary directory for the test + testRoot := t.TempDir() + + // Update paths + tc.containerOptions.Config = strings.ReplaceAll(tc.containerOptions.Config, "{{ .testRoot }}", testRoot) + tc.containerOptions.RuntimeDir = strings.ReplaceAll(tc.containerOptions.RuntimeDir, "{{ .testRoot }}", testRoot) + + // Prepare the environment + if tc.prepareEnvironment != nil { + require.NoError(t, tc.prepareEnvironment(t, testRoot)) + } + + err := Setup(c, &tc.containerOptions, &tc.options) + require.EqualValues(t, tc.expectedSetupError, err) + + if tc.assertSetupPostConditions != nil { + require.NoError(t, tc.assertSetupPostConditions(t, testRoot)) + } + + err = Cleanup(c, &tc.containerOptions, &tc.options) + require.EqualValues(t, tc.expectedCleanupError, err) + + if tc.assertCleanupPostConditions != nil { + require.NoError(t, tc.assertCleanupPostConditions(t, testRoot)) + } + }) + } +} diff --git a/cmd/nvidia-ctk-installer/container/runtime/containerd/config_v1_test.go b/cmd/nvidia-ctk-installer/container/runtime/containerd/config_v1_test.go deleted file mode 100644 index 862d59925..000000000 --- a/cmd/nvidia-ctk-installer/container/runtime/containerd/config_v1_test.go +++ /dev/null @@ -1,584 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package containerd - -import ( - "fmt" - "testing" - - testlog "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - - "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container" - "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd" - "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml" -) - -func TestUpdateV1ConfigDefaultRuntime(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - legacyConfig bool - setAsDefault bool - runtimeName string - expectedDefaultRuntimeName interface{} - expectedDefaultRuntimeBinary interface{} - }{ - {}, - { - legacyConfig: true, - setAsDefault: false, - expectedDefaultRuntimeName: nil, - expectedDefaultRuntimeBinary: nil, - }, - { - legacyConfig: true, - setAsDefault: true, - expectedDefaultRuntimeName: nil, - expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime", - }, - { - legacyConfig: true, - setAsDefault: true, - runtimeName: "NAME", - expectedDefaultRuntimeName: nil, - expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime", - }, - { - legacyConfig: false, - setAsDefault: false, - expectedDefaultRuntimeName: nil, - expectedDefaultRuntimeBinary: nil, - }, - { - legacyConfig: false, - setAsDefault: true, - expectedDefaultRuntimeName: "nvidia", - expectedDefaultRuntimeBinary: nil, - }, - { - legacyConfig: false, - setAsDefault: true, - runtimeName: "NAME", - expectedDefaultRuntimeName: "NAME", - expectedDefaultRuntimeBinary: nil, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: tc.runtimeName, - RuntimeDir: runtimeDir, - SetAsDefault: tc.setAsDefault, - } - - v1, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.Empty), - containerd.WithRuntimeType(runtimeType), - containerd.WithUseLegacyConfig(tc.legacyConfig), - containerd.WithConfigVersion(1), - ) - require.NoError(t, err) - - err = o.UpdateConfig(v1) - require.NoError(t, err) - - cfg := v1.(*containerd.ConfigV1) - defaultRuntimeName := cfg.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}) - require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName) - - defaultRuntime := cfg.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) - if tc.expectedDefaultRuntimeBinary == nil { - require.Nil(t, defaultRuntime) - } else { - require.NotNil(t, defaultRuntime) - - expected, err := defaultRuntimeTomlConfigV1(tc.expectedDefaultRuntimeBinary.(string)) - require.NoError(t, err) - - configContents, _ := toml.Marshal(defaultRuntime.(*toml.Tree)) - expectedContents, _ := toml.Marshal(expected) - - require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, tc) - } - }) - } -} - -func TestUpdateV1Config(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - runtimeName string - expectedConfig map[string]interface{} - }{ - { - runtimeName: "nvidia", - expectedConfig: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - "Runtime": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "NAME", - expectedConfig: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "NAME": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - "Runtime": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: tc.runtimeName, - RuntimeDir: runtimeDir, - } - - v1, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.Empty), - containerd.WithRuntimeType(runtimeType), - containerd.WithConfigVersion(1), - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.UpdateConfig(v1) - require.NoError(t, err) - - expected, err := toml.TreeFromMap(tc.expectedConfig) - require.NoError(t, err) - - require.Equal(t, expected.String(), v1.String()) - }) - } -} - -func TestUpdateV1ConfigWithRuncPresent(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - runtimeName string - expectedConfig map[string]interface{} - }{ - { - runtimeName: "nvidia", - expectedConfig: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/runc-binary", - }, - }, - "nvidia": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - "Runtime": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "NAME", - expectedConfig: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/runc-binary", - }, - }, - "NAME": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - "Runtime": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - "Runtime": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: tc.runtimeName, - RuntimeDir: runtimeDir, - } - - v1, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.FromMap(runcConfigMapV1("/runc-binary"))), - containerd.WithRuntimeType(runtimeType), - containerd.WithConfigVersion(1), - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.UpdateConfig(v1) - require.NoError(t, err) - - expected, err := toml.TreeFromMap(tc.expectedConfig) - require.NoError(t, err) - - require.Equal(t, expected.String(), v1.String()) - }) - } -} - -func TestUpdateV1EnableCDI(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - enableCDI bool - expectedEnableCDIValue interface{} - }{ - {}, - { - enableCDI: false, - expectedEnableCDIValue: nil, - }, - { - enableCDI: true, - expectedEnableCDIValue: true, - }, - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("%v", tc.enableCDI), func(t *testing.T) { - o := &container.Options{ - EnableCDI: tc.enableCDI, - RuntimeName: "nvidia", - RuntimeDir: runtimeDir, - } - - cfg, err := toml.Empty.Load() - require.NoError(t, err) - - v1 := &containerd.ConfigV1{ - Logger: logger, - Tree: cfg, - RuntimeType: runtimeType, - } - - err = o.UpdateConfig(v1) - require.NoError(t, err) - - enableCDIValue := v1.GetPath([]string{"plugins", "cri", "containerd", "enable_cdi"}) - require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue) - }) - } -} - -func TestRevertV1Config(t *testing.T) { - logger, _ := testlog.NewNullLogger() - testCases := []struct { - config map[string]interface { - } - expected map[string]interface{} - }{ - {}, - { - config: map[string]interface{}{ - "version": int64(1), - }, - }, - { - config: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), - "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), - "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), - }, - }, - }, - }, - }, - }, - { - config: map[string]interface{}{ - "version": int64(1), - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"), - "nvidia-cdi": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.cdi"), - "nvidia-legacy": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime.legacy"), - }, - "default_runtime": defaultRuntimeV1("/test/runtime/dir/nvidia-container-runtime"), - "default_runtime_name": "nvidia", - }, - }, - }, - }, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: "nvidia", - } - - expected, err := toml.TreeFromMap(tc.expected) - require.NoError(t, err) - - v1, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.FromMap(tc.config)), - containerd.WithRuntimeType(runtimeType), - - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.RevertConfig(v1) - require.NoError(t, err) - - require.Equal(t, expected.String(), v1.String()) - }) - } -} - -func defaultRuntimeTomlConfigV1(binary string) (*toml.Tree, error) { - return toml.TreeFromMap(defaultRuntimeV1(binary)) -} - -func defaultRuntimeV1(binary string) map[string]interface{} { - return map[string]interface{}{ - "runtime_type": runtimeType, - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "options": map[string]interface{}{ - "BinaryName": binary, - "Runtime": binary, - }, - } -} - -func runtimeMapV1(binary string) map[string]interface{} { - return map[string]interface{}{ - "runtime_type": runtimeType, - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "options": map[string]interface{}{ - "BinaryName": binary, - "Runtime": binary, - }, - } -} - -func runcConfigMapV1(binary string) map[string]interface{} { - return map[string]interface{}{ - "plugins": map[string]interface{}{ - "cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": binary, - }, - }, - }, - }, - }, - }, - } -} diff --git a/cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go b/cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go deleted file mode 100644 index a6570e8f4..000000000 --- a/cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go +++ /dev/null @@ -1,520 +0,0 @@ -/** -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ - -package containerd - -import ( - "fmt" - "testing" - - testlog "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - - "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container" - "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine/containerd" - "github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml" -) - -const ( - runtimeType = "runtime_type" -) - -func TestUpdateV2ConfigDefaultRuntime(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - setAsDefault bool - runtimeName string - expectedDefaultRuntimeName interface{} - }{ - {}, - { - setAsDefault: false, - runtimeName: "nvidia", - expectedDefaultRuntimeName: nil, - }, - { - setAsDefault: false, - runtimeName: "NAME", - expectedDefaultRuntimeName: nil, - }, - { - setAsDefault: true, - runtimeName: "nvidia", - expectedDefaultRuntimeName: "nvidia", - }, - { - setAsDefault: true, - runtimeName: "NAME", - expectedDefaultRuntimeName: "NAME", - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: tc.runtimeName, - RuntimeDir: runtimeDir, - SetAsDefault: tc.setAsDefault, - } - - v2, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.Empty), - containerd.WithRuntimeType(runtimeType), - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.UpdateConfig(v2) - require.NoError(t, err) - - cfg := v2.(*containerd.Config) - - defaultRuntimeName := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}) - require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName) - }) - } -} - -func TestUpdateV2Config(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - runtimeName string - expectedConfig map[string]interface{} - }{ - { - runtimeName: "nvidia", - expectedConfig: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "NAME", - expectedConfig: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "NAME": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runtime_type", - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: tc.runtimeName, - RuntimeDir: runtimeDir, - } - - v2, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.Empty), - containerd.WithRuntimeType(runtimeType), - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.UpdateConfig(v2) - require.NoError(t, err) - - expected, err := toml.TreeFromMap(tc.expectedConfig) - require.NoError(t, err) - - require.Equal(t, expected.String(), v2.String()) - }) - } - -} - -func TestUpdateV2ConfigWithRuncPresent(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - runtimeName string - expectedConfig map[string]interface{} - }{ - { - runtimeName: "nvidia", - expectedConfig: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/runc-binary", - }, - }, - "nvidia": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - { - runtimeName: "NAME", - expectedConfig: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/runc-binary", - }, - }, - "NAME": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime", - }, - }, - "nvidia-cdi": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.cdi", - }, - }, - "nvidia-legacy": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "container_annotations": []string{"cdi.k8s.io/*"}, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": "/test/runtime/dir/nvidia-container-runtime.legacy", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: tc.runtimeName, - RuntimeDir: runtimeDir, - } - - v2, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.FromMap(runcConfigMapV2("/runc-binary"))), - containerd.WithRuntimeType(runtimeType), - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.UpdateConfig(v2) - require.NoError(t, err) - - expected, err := toml.TreeFromMap(tc.expectedConfig) - require.NoError(t, err) - - require.Equal(t, expected.String(), v2.String()) - }) - } -} - -func TestUpdateV2ConfigEnableCDI(t *testing.T) { - logger, _ := testlog.NewNullLogger() - const runtimeDir = "/test/runtime/dir" - - testCases := []struct { - enableCDI bool - expectedEnableCDIValue interface{} - }{ - {}, - { - enableCDI: false, - expectedEnableCDIValue: nil, - }, - { - enableCDI: true, - expectedEnableCDIValue: true, - }, - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("%v", tc.enableCDI), func(t *testing.T) { - o := &container.Options{ - EnableCDI: tc.enableCDI, - RuntimeName: "nvidia", - RuntimeDir: runtimeDir, - SetAsDefault: false, - } - - cfg, err := toml.LoadMap(map[string]interface{}{}) - require.NoError(t, err) - - v2 := &containerd.Config{ - Logger: logger, - Tree: cfg, - RuntimeType: runtimeType, - CRIRuntimePluginName: "io.containerd.grpc.v1.cri", - } - - err = o.UpdateConfig(v2) - require.NoError(t, err) - - enableCDIValue := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_cdi"}) - require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue) - }) - } -} - -func TestRevertV2Config(t *testing.T) { - logger, _ := testlog.NewNullLogger() - - testCases := []struct { - config map[string]interface { - } - expected map[string]interface{} - }{ - {}, - { - config: map[string]interface{}{ - "version": int64(2), - }, - }, - { - config: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"), - }, - }, - }, - }, - }, - }, - { - config: map[string]interface{}{ - "version": int64(2), - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "nvidia": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"), - }, - "default_runtime_name": "nvidia", - }, - }, - }, - }, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - o := &container.Options{ - RuntimeName: "nvidia", - } - - expected, err := toml.TreeFromMap(tc.expected) - require.NoError(t, err) - - v2, err := containerd.New( - containerd.WithLogger(logger), - containerd.WithConfigSource(toml.FromMap(tc.config)), - containerd.WithRuntimeType(runtimeType), - containerd.WithContainerAnnotations("cdi.k8s.io/*"), - ) - require.NoError(t, err) - - err = o.RevertConfig(v2) - require.NoError(t, err) - - require.Equal(t, expected.String(), v2.String()) - }) - } -} - -func runtimeMapV2(binary string) map[string]interface{} { - return map[string]interface{}{ - "runtime_type": runtimeType, - "runtime_root": "", - "runtime_engine": "", - "privileged_without_host_devices": false, - "options": map[string]interface{}{ - "BinaryName": binary, - }, - } -} - -func runcConfigMapV2(binary string) map[string]interface{} { - return map[string]interface{}{ - "version": 2, - "plugins": map[string]interface{}{ - "io.containerd.grpc.v1.cri": map[string]interface{}{ - "containerd": map[string]interface{}{ - "runtimes": map[string]interface{}{ - "runc": map[string]interface{}{ - "runtime_type": "runc_runtime_type", - "runtime_root": "runc_runtime_root", - "runtime_engine": "runc_runtime_engine", - "privileged_without_host_devices": true, - "options": map[string]interface{}{ - "runc-option": "value", - "BinaryName": binary, - }, - }, - }, - }, - }, - }, - } -} diff --git a/cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go b/cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go index 4e896c18b..342837f6f 100644 --- a/cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go +++ b/cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go @@ -36,7 +36,7 @@ const ( DefaultSocket = "/run/containerd/containerd.sock" DefaultRestartMode = "signal" - defaultRuntmeType = "io.containerd.runc.v2" + defaultRuntimeType = "io.containerd.runc.v2" ) // Options stores the containerd-specific options @@ -63,7 +63,7 @@ func Flags(opts *Options) []cli.Flag { &cli.StringFlag{ Name: "runtime-type", Usage: "The runtime_type to use for the configured runtime classes", - Value: defaultRuntmeType, + Value: defaultRuntimeType, Destination: &opts.runtimeType, Sources: cli.EnvVars("CONTAINERD_RUNTIME_TYPE"), }, diff --git a/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go b/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go new file mode 100644 index 000000000..385631aab --- /dev/null +++ b/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go @@ -0,0 +1,609 @@ +/** +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/ + +package crio + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + cli "github.com/urfave/cli/v3" + + "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container" + "github.com/NVIDIA/nvidia-container-toolkit/internal/config" +) + +// TestCrioConfigLifecycle tests the complete Setup->Cleanup lifecycle for both config and hook modes +func TestCrioConfigLifecycle(t *testing.T) { + c := &cli.Command{ + Name: "test", + } + + testCases := []struct { + description string + containerOptions container.Options + options Options + prepareEnvironment func(*testing.T, string) error + expectedSetupError error + assertSetupPostConditions func(*testing.T, string) error + expectedCleanupError error + assertCleanupPostConditions func(*testing.T, string) error + }{ + { + description: "config mode: top-level config does not exist", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/crio/crio.conf", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, + RestartMode: "none", + }, + options: Options{ + configMode: "config", + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.runtime] + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.nvidia] + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.NoFileExists(t, configPath) + return nil + }, + }, + { + description: "config mode: existing config without nvidia runtime", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/crio/crio.conf", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, + RestartMode: "none", + }, + options: Options{ + configMode: "config", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + configContent := `[crio] +[crio.runtime] +default_runtime = "crun" + +[crio.runtime.runtimes.crun] +runtime_path = "/usr/bin/crun" +runtime_type = "oci" +runtime_root = "/run/crun" +monitor_path = "/usr/libexec/crio/conmon" + +[crio.image] +signature_policy = "/etc/crio/policy.json" +` + err := os.WriteFile(configPath, []byte(configContent), 0600) + require.NoError(t, err) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.image] + signature_policy = "/etc/crio/policy.json" + + [crio.runtime] + default_runtime = "crun" + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.crun] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/crun" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_root = "/run/crun" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Should restore to original config + expected := ` +[crio] + + [crio.image] + signature_policy = "/etc/crio/policy.json" + + [crio.runtime] + default_runtime = "crun" + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.crun] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/crun" + runtime_root = "/run/crun" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "config mode: existing config with nvidia runtime already present", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/crio/crio.conf", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: true, + RestartMode: "none", + }, + options: Options{ + configMode: "config", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + configContent := `[crio] +[crio.runtime] +default_runtime = "nvidia" + +[crio.runtime.runtimes.crun] +runtime_path = "/usr/bin/crun" +runtime_type = "oci" + +[crio.runtime.runtimes.nvidia] +runtime_path = "/old/path/nvidia-container-runtime" +runtime_type = "oci" +` + err := os.WriteFile(configPath, []byte(configContent), 0600) + require.NoError(t, err) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.runtime] + default_runtime = "nvidia" + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.crun] + runtime_path = "/usr/bin/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia] + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Note: cleanup removes nvidia runtimes but doesn't restore original default_runtime + expected := ` +[crio] + + [crio.runtime] + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.crun] + runtime_path = "/usr/bin/crun" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + { + description: "config mode: complex config with multiple settings", + containerOptions: container.Options{ + Config: "{{ .testRoot }}/etc/crio/crio.conf", + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + SetAsDefault: false, + RestartMode: "none", + }, + options: Options{ + configMode: "config", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.NoError(t, os.MkdirAll(filepath.Dir(configPath), 0755)) + + configContent := `[crio] +[crio.runtime] +default_runtime = "crun" +conmon = "/usr/libexec/crio/conmon" +conmon_cgroup = "pod" +selinux = true + +[crio.runtime.runtimes.crun] +runtime_path = "/usr/bin/crun" +runtime_type = "oci" +runtime_root = "/run/crun" +monitor_path = "/usr/libexec/crio/conmon" + +[crio.runtime.runtimes.runc] +runtime_path = "/usr/bin/runc" +runtime_type = "oci" +runtime_root = "/run/runc" + +[crio.image] +signature_policy = "/etc/crio/policy.json" +insecure_registries = [ + "localhost:5000" +] + +[crio.network] +network_dir = "/etc/cni/net.d/" +plugin_dirs = [ + "/opt/cni/bin", + "/usr/libexec/cni" +] +` + err := os.WriteFile(configPath, []byte(configContent), 0600) + require.NoError(t, err) + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.image] + insecure_registries = ["localhost:5000"] + signature_policy = "/etc/crio/policy.json" + + [crio.network] + network_dir = "/etc/cni/net.d/" + plugin_dirs = ["/opt/cni/bin", "/usr/libexec/cni"] + + [crio.runtime] + conmon = "/usr/libexec/crio/conmon" + conmon_cgroup = "pod" + default_runtime = "crun" + selinux = true + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.crun] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/crun" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.runc] + runtime_path = "/usr/bin/runc" + runtime_root = "/run/runc" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + configPath := filepath.Join(testRoot, "etc/crio/crio.conf") + require.FileExists(t, configPath) + + actual, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Should restore to original complex config + expected := ` +[crio] + + [crio.image] + insecure_registries = ["localhost:5000"] + signature_policy = "/etc/crio/policy.json" + + [crio.network] + network_dir = "/etc/cni/net.d/" + plugin_dirs = ["/opt/cni/bin", "/usr/libexec/cni"] + + [crio.runtime] + conmon = "/usr/libexec/crio/conmon" + conmon_cgroup = "pod" + default_runtime = "crun" + selinux = true + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.crun] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/crun" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.runc] + runtime_path = "/usr/bin/runc" + runtime_root = "/run/runc" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) + return nil + }, + }, + // Hook mode test cases + { + description: "hook mode: no existing hook", + containerOptions: container.Options{ + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + RestartMode: "none", + }, + options: Options{ + configMode: "hook", + hooksDir: "{{ .testRoot }}/etc/crio/hooks.d", + hookFilename: "99-nvidia.json", + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + hookPath := filepath.Join(testRoot, "etc/crio/hooks.d/99-nvidia.json") + require.FileExists(t, hookPath) + + actual, err := os.ReadFile(hookPath) + require.NoError(t, err) + + expectedContents := filepath.Join("/usr/bin", config.NVIDIAContainerRuntimeHookExecutable) + expected := fmt.Sprintf(`{ + "version": "1.0.0", + "hook": { + "path": "%s", + "args": [ + "nvidia-container-runtime-hook", + "prestart" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ] + }, + "when": { + "always": true, + "commands": [ + ".*" + ] + }, + "stages": [ + "prestart" + ] +} +`, expectedContents) + + require.Equal(t, expected, string(actual)) + + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + hookPath := filepath.Join(testRoot, "etc/crio/hooks.d/99-nvidia.json") + require.NoFileExists(t, hookPath) + return nil + }, + }, + { + description: "hook mode: existing hook file", + containerOptions: container.Options{ + RuntimeName: "nvidia", + RuntimeDir: "/usr/bin", + RestartMode: "none", + }, + options: Options{ + configMode: "hook", + hooksDir: "{{ .testRoot }}/etc/crio/hooks.d", + hookFilename: "99-nvidia.json", + }, + prepareEnvironment: func(t *testing.T, testRoot string) error { + hooksDir := filepath.Join(testRoot, "etc/crio/hooks.d") + require.NoError(t, os.MkdirAll(hooksDir, 0755)) + + // Create existing hook with old path + existingHookJSON := `{ + "version": "1.0.0", + "hook": { + "path": "/old/path/nvidia-container-runtime-hook", + "args": [ + "nvidia-container-runtime-hook", + "prestart" + ] + }, + "when": { + "always": true + }, + "stages": [ + "prestart" + ] +}` + + hookPath := filepath.Join(hooksDir, "99-nvidia.json") + err := os.WriteFile(hookPath, []byte(existingHookJSON), 0600) + require.NoError(t, err) + + return nil + }, + assertSetupPostConditions: func(t *testing.T, testRoot string) error { + hookPath := filepath.Join(testRoot, "etc/crio/hooks.d/99-nvidia.json") + require.FileExists(t, hookPath) + + actual, err := os.ReadFile(hookPath) + require.NoError(t, err) + + expectedContents := filepath.Join("/usr/bin", config.NVIDIAContainerRuntimeHookExecutable) + expected := fmt.Sprintf(`{ + "version": "1.0.0", + "hook": { + "path": "%s", + "args": [ + "nvidia-container-runtime-hook", + "prestart" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ] + }, + "when": { + "always": true, + "commands": [ + ".*" + ] + }, + "stages": [ + "prestart" + ] +} +`, expectedContents) + + require.Equal(t, expected, string(actual)) + + return nil + }, + assertCleanupPostConditions: func(t *testing.T, testRoot string) error { + hookPath := filepath.Join(testRoot, "etc/crio/hooks.d/99-nvidia.json") + require.NoFileExists(t, hookPath) + return nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + // Update any paths as required + testRoot := t.TempDir() + tc.containerOptions.Config = strings.ReplaceAll(tc.containerOptions.Config, "{{ .testRoot }}", testRoot) + tc.options.hooksDir = strings.ReplaceAll(tc.options.hooksDir, "{{ .testRoot }}", testRoot) + + // Prepare the test environment + if tc.prepareEnvironment != nil { + require.NoError(t, tc.prepareEnvironment(t, testRoot)) + } + + err := Setup(c, &tc.containerOptions, &tc.options) + require.EqualValues(t, tc.expectedSetupError, err) + + if tc.assertSetupPostConditions != nil { + require.NoError(t, tc.assertSetupPostConditions(t, testRoot)) + } + + err = Cleanup(c, &tc.containerOptions, &tc.options) + require.EqualValues(t, tc.expectedCleanupError, err) + + if tc.assertCleanupPostConditions != nil { + require.NoError(t, tc.assertCleanupPostConditions(t, testRoot)) + } + }) + } +} From 44039746483460d0ab61abd4555a045c162defeb Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 17 Sep 2025 17:17:13 +0200 Subject: [PATCH 2/2] [no-relnote] Don't use fmt.Sprintf Signed-off-by: Evan Lezar --- .../container/runtime/crio/config_test.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go b/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go index 385631aab..3837f029a 100644 --- a/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go +++ b/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go @@ -17,7 +17,6 @@ package crio import ( - "fmt" "os" "path/filepath" "strings" @@ -27,7 +26,6 @@ import ( cli "github.com/urfave/cli/v3" "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container" - "github.com/NVIDIA/nvidia-container-toolkit/internal/config" ) // TestCrioConfigLifecycle tests the complete Setup->Cleanup lifecycle for both config and hook modes @@ -460,11 +458,10 @@ plugin_dirs = [ actual, err := os.ReadFile(hookPath) require.NoError(t, err) - expectedContents := filepath.Join("/usr/bin", config.NVIDIAContainerRuntimeHookExecutable) - expected := fmt.Sprintf(`{ + expected := `{ "version": "1.0.0", "hook": { - "path": "%s", + "path": "/usr/bin/nvidia-container-runtime-hook", "args": [ "nvidia-container-runtime-hook", "prestart" @@ -483,7 +480,7 @@ plugin_dirs = [ "prestart" ] } -`, expectedContents) +` require.Equal(t, expected, string(actual)) @@ -542,11 +539,10 @@ plugin_dirs = [ actual, err := os.ReadFile(hookPath) require.NoError(t, err) - expectedContents := filepath.Join("/usr/bin", config.NVIDIAContainerRuntimeHookExecutable) - expected := fmt.Sprintf(`{ + expected := `{ "version": "1.0.0", "hook": { - "path": "%s", + "path": "/usr/bin/nvidia-container-runtime-hook", "args": [ "nvidia-container-runtime-hook", "prestart" @@ -565,7 +561,7 @@ plugin_dirs = [ "prestart" ] } -`, expectedContents) +` require.Equal(t, expected, string(actual))