Merge pull request #1251 from elezar/allow-config-source-to-be-overridden

Allow config sources to be specified for containerd and cri-o

Author: elezar

cmd/nvidia-ctk-installer/container/container.go

@@ -21,11 +21,13 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/container/operator"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 const (
@@ -53,6 +55,8 @@ type Options struct {
 	SetAsDefault  bool
 	RestartMode   string
 	HostRootMount string
+
+	ConfigSources []string
 }
 
 // Configure applies the options to the specified config
@@ -182,3 +186,38 @@ func (o Options) SystemdRestart(service string) error {
 
 	return nil
 }
+
+// GetConfigLoaders returns the loaders for the requested config sources.
+// Supported config sources can be specified as:
+//
+// * 'file[=path/to/file]': The specified file or the top-level config path is used.
+// * command: The runtime-specific function supplied as an argument is used.
+func (o Options) GetConfigLoaders(commandSourceFunc func(string, string) toml.Loader) ([]toml.Loader, error) {
+	if len(o.ConfigSources) == 0 {
+		return []toml.Loader{toml.Empty}, nil
+	}
+	var loaders []toml.Loader
+	for _, configSource := range o.ConfigSources {
+		parts := strings.SplitN(configSource, "=", 2)
+		source := parts[0]
+		switch source {
+		case "file":
+			fileSourcePath := o.TopLevelConfigPath
+			if len(parts) > 1 {
+				fileSourcePath = parts[1]
+			}
+			loaders = append(loaders, toml.FromFile(fileSourcePath))
+		case "command":
+			if commandSourceFunc == nil {
+				logrus.Warnf("Ignoring command config source")
+			}
+			if len(parts) > 1 {
+				logrus.Warnf("Ignoring additional command argument %q", parts[1])
+			}
+			loaders = append(loaders, commandSourceFunc(o.HostRootMount, o.ExecutablePath))
+		default:
+			return nil, fmt.Errorf("unsupported config source %q", configSource)
+		}
+	}
+	return loaders, nil
+}

cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go

@@ -1783,6 +1783,10 @@ version = 3
 	}
 
 	for _, tc := range testCases {
+		// Set default options that would normally be set by the CLI.
+		if tc.containerOptions.ConfigSources == nil {
+			tc.containerOptions.ConfigSources = []string{"command", "file"}
+		}
 		t.Run(tc.description, func(t *testing.T) {
 			// Create a temporary directory for the test
 			testRoot := t.TempDir()
@@ -1791,6 +1795,11 @@ version = 3
 			tc.containerOptions.TopLevelConfigPath = strings.ReplaceAll(tc.containerOptions.TopLevelConfigPath, "{{ .testRoot }}", testRoot)
 			tc.containerOptions.DropInConfig = strings.ReplaceAll(tc.containerOptions.DropInConfig, "{{ .testRoot }}", testRoot)
 			tc.containerOptions.RuntimeDir = strings.ReplaceAll(tc.containerOptions.RuntimeDir, "{{ .testRoot }}", testRoot)
+			var testConfigSources []string
+			for _, configSource := range tc.containerOptions.ConfigSources {
+				testConfigSources = append(testConfigSources, strings.ReplaceAll(configSource, "{{ .testRoot }}", testRoot))
+			}
+			tc.containerOptions.ConfigSources = testConfigSources
 
 			// Prepare the environment
 			if tc.prepareEnvironment != nil {

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go

@@ -172,12 +172,15 @@ func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error
 }
 
 func getRuntimeConfig(o *container.Options, co *Options) (engine.Interface, error) {
+	loaders, err := o.GetConfigLoaders(containerd.CommandLineSource)
+	if err != nil {
+		return nil, err
+	}
 	options := []containerd.Option{
 		containerd.WithTopLevelConfigPath(o.TopLevelConfigPath),
 		containerd.WithConfigSource(
 			toml.LoadFirst(
-				containerd.CommandLineSource(o.HostRootMount, o.ExecutablePath),
-				toml.FromFile(o.TopLevelConfigPath),
+				loaders...,
 			),
 		),
 		containerd.WithRuntimeType(co.runtimeType),

cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go

@@ -621,17 +621,24 @@ plugin_dirs = [
 		},
 	}
 
-	for i, tc := range testCases {
-		if i > 3 {
-			t.SkipNow()
+	for _, tc := range testCases {
+		// Set default options that would normally be set by the CLI.
+		if tc.containerOptions.ConfigSources == nil {
+			tc.containerOptions.ConfigSources = []string{"command", "file"}
 		}
+		tc.options.hookFilename = "99-nvidia.json"
+
 		t.Run(tc.description, func(t *testing.T) {
 			// Update any paths as required
 			testRoot := t.TempDir()
 			tc.containerOptions.TopLevelConfigPath = strings.ReplaceAll(tc.containerOptions.TopLevelConfigPath, "{{ .testRoot }}", testRoot)
 			tc.containerOptions.DropInConfig = strings.ReplaceAll(tc.containerOptions.DropInConfig, "{{ .testRoot }}", testRoot)
 			tc.options.hooksDir = strings.ReplaceAll(tc.options.hooksDir, "{{ .testRoot }}", testRoot)
-			tc.options.hookFilename = "99-nvidia.json"
+			var testConfigSources []string
+			for _, configSource := range tc.containerOptions.ConfigSources {
+				testConfigSources = append(testConfigSources, strings.ReplaceAll(configSource, "{{ .testRoot }}", testRoot))
+			}
+			tc.containerOptions.ConfigSources = testConfigSources
 
 			// Prepare the test environment
 			if tc.prepareEnvironment != nil {

cmd/nvidia-ctk-installer/container/runtime/crio/crio.go

@@ -214,12 +214,15 @@ func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) {
 }
 
 func getRuntimeConfig(o *container.Options) (engine.Interface, error) {
+	loaders, err := o.GetConfigLoaders(crio.CommandLineSource)
+	if err != nil {
+		return nil, err
+	}
 	return crio.New(
 		crio.WithTopLevelConfigPath(o.TopLevelConfigPath),
 		crio.WithConfigSource(
 			toml.LoadFirst(
-				crio.CommandLineSource(o.HostRootMount, o.ExecutablePath),
-				toml.FromFile(o.TopLevelConfigPath),
+				loaders...,
 			),
 		),
 	)

cmd/nvidia-ctk-installer/container/runtime/runtime.go

@@ -117,6 +117,17 @@ func Flags(opts *Options) []cli.Flag {
 			Sources:     cli.EnvVars("NVIDIA_RUNTIME_SET_AS_DEFAULT", "CONTAINERD_SET_AS_DEFAULT", "DOCKER_SET_AS_DEFAULT"),
 			Hidden:      true,
 		},
+		&cli.StringSliceFlag{
+			Name:    "config-source",
+			Aliases: []string{"config-sources"},
+			Usage: "Specify the config sources for the container runtime. Any combination of " +
+				"[command | file]. " +
+				"When `file` is specified, the absolute path to the file to be used as a config source can " +
+				"be specified as `file=/path/to/source/config.toml",
+			Value:       []string{"command", "file"},
+			Destination: &opts.ConfigSources,
+			Sources:     cli.EnvVars("RUNTIME_CONFIG_SOURCES", "RUNTIME_CONFIG_SOURCE"),
+		},
 	}
 
 	flags = append(flags, containerd.Flags(&opts.containerdOptions)...)
@@ -144,6 +155,7 @@ func (opts *Options) Validate(logger logger.Interface, c *cli.Command, runtime s
 			logger.Warningf("Ignoring drop-in-config=%q flag for %v", opts.DropInConfig, opts.RuntimeName)
 			opts.DropInConfig = ""
 		}
+		opts.ConfigSources = []string{"file"}
 	case containerd.Name:
 	case crio.Name:
 		if err := opts.crioOptions.Validate(logger, c); err != nil {
@@ -152,7 +164,6 @@ func (opts *Options) Validate(logger logger.Interface, c *cli.Command, runtime s
 	}
 
 	// Apply the runtime-specific config changes.
-	// TODO: Add the runtime-specific DropInConfigs here.
 	switch runtime {
 	case containerd.Name:
 		if opts.TopLevelConfigPath == runtimeSpecificDefault {