Implement Drop-In Lifecycle Support for Containerd and Crio Configuration

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>

Author: ArangoGutierrez

cmd/nvidia-ctk-installer/container/runtime/containerd/config_v2_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container"
@@ -85,8 +86,9 @@ func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
 
 			cfg := v2.(*containerd.Config)
 
-			defaultRuntimeName := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
-			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
+			// For v2 configs, default runtime is set in NVConfig (drop-in configuration)
+			defaultRuntimeName := cfg.NVConfig.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+			assert.Equal(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
 		})
 	}
 }
@@ -358,10 +360,22 @@ func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
 			err = o.UpdateConfig(v2)
 			require.NoError(t, err)
 
-			expected, err := toml.TreeFromMap(tc.expectedConfig)
-			require.NoError(t, err)
+			cfg := v2.(*containerd.Config)
 
-			require.Equal(t, expected.String(), v2.String())
+			// Verify nvidia runtimes are in NVConfig
+			for _, runtimeName := range []string{tc.runtimeName, "nvidia-cdi", "nvidia-legacy"} {
+				if tc.runtimeName == "NAME" && runtimeName == "nvidia" {
+					continue // Skip "nvidia" when runtime name is "NAME"
+				}
+				runtimePath := cfg.NVConfig.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", runtimeName, "options", "BinaryName"})
+				assert.NotNil(t, runtimePath)
+			}
+
+			// Verify main Tree remains unchanged (still contains original runc runtime)
+			originalConfig := runcConfigMapV2("/runc-binary")
+			expectedMain, err := toml.TreeFromMap(originalConfig)
+			require.NoError(t, err)
+			assert.Equal(t, expectedMain.String(), cfg.Tree.String(), "Main config should remain unchanged")
 		})
 	}
 }
@@ -397,18 +411,23 @@ func TestUpdateV2ConfigEnableCDI(t *testing.T) {
 			cfg, err := toml.LoadMap(map[string]interface{}{})
 			require.NoError(t, err)
 
+			nvConfig, err := toml.LoadMap(map[string]interface{}{})
+			require.NoError(t, err)
+
 			v2 := &containerd.Config{
 				Logger:               logger,
 				Tree:                 cfg,
+				NVConfig:             nvConfig,
 				RuntimeType:          runtimeType,
 				CRIRuntimePluginName: "io.containerd.grpc.v1.cri",
 			}
 
 			err = o.UpdateConfig(v2)
 			require.NoError(t, err)
 
-			enableCDIValue := cfg.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_cdi"})
-			require.EqualValues(t, tc.expectedEnableCDIValue, enableCDIValue)
+			// For v2 configs, enable_cdi is set in NVConfig (drop-in configuration)
+			enableCDIValue := v2.NVConfig.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_cdi"})
+			assert.Equal(t, tc.expectedEnableCDIValue, enableCDIValue)
 		})
 	}
 }

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go

@@ -47,6 +47,8 @@ type Options struct {
 	ContainerRuntimeModesCDIAnnotationPrefixes []string
 
 	runtimeConfigOverrideJSON string
+
+	dropInDir string
 }
 
 func Flags(opts *Options) []cli.Flag {
@@ -79,6 +81,12 @@ func Flags(opts *Options) []cli.Flag {
 			Value:       "{}",
 			Sources:     cli.EnvVars("RUNTIME_CONFIG_OVERRIDE", "CONTAINERD_RUNTIME_CONFIG_OVERRIDE"),
 		},
+		&cli.StringFlag{
+			Name:        "containerd-drop-in-dir",
+			Usage:       "the directory to store the drop-in configuration for the containerd runtime",
+			Value:       "/etc/containerd/conf.d",
+			Destination: &opts.dropInDir,
+		},
 	}
 
 	return flags
@@ -180,5 +188,6 @@ func getRuntimeConfig(o *container.Options, co *Options) (engine.Interface, erro
 		containerd.WithRuntimeType(co.runtimeType),
 		containerd.WithUseLegacyConfig(co.useLegacyConfig),
 		containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
+		containerd.WithDropInDir(co.dropInDir),
 	)
 }

cmd/nvidia-ctk-installer/main_test.go

@@ -437,7 +437,18 @@ swarm-resource = ""
 				"--toolkit-source-root=" + filepath.Join(artifactRoot, "deb"),
 			}
 
-			err := app.Run(context.Background(), append(testArgs, tc.args...))
+			// Add containerd drop-in directory for containerd runtime tests to avoid permission issues
+			finalArgs := make([]string, 0, len(testArgs)+len(tc.args)+1)
+			finalArgs = append(finalArgs, testArgs...)
+			finalArgs = append(finalArgs, tc.args...)
+			for _, arg := range tc.args {
+				if arg == "--runtime=containerd" {
+					finalArgs = append(finalArgs, "--containerd-drop-in-dir="+filepath.Join(testRoot, "containerd", "conf.d"))
+					break
+				}
+			}
+
+			err := app.Run(context.Background(), finalArgs)
 
 			require.NoError(t, err)
 

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -46,6 +46,10 @@ const (
 	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
 	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
 
+	defaultContainerdDropInDir = "/etc/containerd/conf.d"
+	defaultCrioDropInDir       = "/etc/crio/conf.d"
+	defaultDropInFileName      = "99.nvidia.toml"
+
 	defaultConfigSource = configSourceFile
 	configSourceCommand = "command"
 	configSourceFile    = "file"
@@ -66,13 +70,14 @@ func NewCommand(logger logger.Interface) *cli.Command {
 // config defines the options that can be set for the CLI through config files,
 // environment variables, or command line config
 type config struct {
-	dryRun         bool
-	runtime        string
-	configFilePath string
-	executablePath string
-	configSource   string
-	mode           string
-	hookFilePath   string
+	dryRun           bool
+	runtime          string
+	configFilePath   string
+	executablePath   string
+	configSource     string
+	mode             string
+	hookFilePath     string
+	dropInConfigPath string
 
 	nvidiaRuntime struct {
 		name         string
@@ -170,6 +175,11 @@ func (m command) build() *cli.Command {
 				Usage:       "Enable CDI in the configured runtime",
 				Destination: &config.cdi.enabled,
 			},
+			&cli.StringFlag{
+				Name:        "drop-in-config-path",
+				Usage:       "the path to the drop-in configuration file for the containerd runtime",
+				Destination: &config.dropInConfigPath,
+			},
 		},
 	}
 
@@ -262,24 +272,37 @@ func (m command) configureConfigFile(config *config) error {
 	}
 
 	var cfg engine.Interface
+	var outputPath string
 	switch config.runtime {
 	case "containerd":
+		if config.dropInConfigPath == "" {
+			config.dropInConfigPath = filepath.Join(defaultContainerdDropInDir, defaultDropInFileName)
+		}
 		cfg, err = containerd.New(
 			containerd.WithLogger(m.logger),
 			containerd.WithPath(config.configFilePath),
 			containerd.WithConfigSource(configSource),
+			containerd.WithDropInDir(config.dropInConfigPath),
 		)
+		outputPath = config.dropInConfigPath
 	case "crio":
+		if config.dropInConfigPath == "" {
+			// CRI-O uses .conf extension for drop-in files
+			config.dropInConfigPath = filepath.Join(defaultCrioDropInDir, "99-nvidia.conf")
+		}
 		cfg, err = crio.New(
 			crio.WithLogger(m.logger),
 			crio.WithPath(config.configFilePath),
 			crio.WithConfigSource(configSource),
+			crio.WithDropInDir(config.dropInConfigPath),
 		)
+		outputPath = config.dropInConfigPath
 	case "docker":
 		cfg, err = docker.New(
 			docker.WithLogger(m.logger),
 			docker.WithPath(config.configFilePath),
 		)
+		outputPath = config.getOutputConfigPath()
 	default:
 		err = fmt.Errorf("unrecognized runtime '%v'", config.runtime)
 	}
@@ -300,7 +323,6 @@ func (m command) configureConfigFile(config *config) error {
 		cfg.EnableCDI()
 	}
 
-	outputPath := config.getOutputConfigPath()
 	n, err := cfg.Save(outputPath)
 	if err != nil {
 		return fmt.Errorf("unable to flush config: %v", err)