diff --git a/cmd/nvidia-ctk-installer/container/container.go b/cmd/nvidia-ctk-installer/container/container.go index 748956cce..90e2fcfe2 100644 --- a/cmd/nvidia-ctk-installer/container/container.go +++ b/cmd/nvidia-ctk-installer/container/container.go @@ -65,7 +65,7 @@ func (o Options) Configure(cfg engine.Interface) error { if err != nil { return fmt.Errorf("unable to update config: %v", err) } - return o.flush(cfg) + return o.Flush(cfg) } // Unconfigure removes the options from the specified config @@ -75,7 +75,7 @@ func (o Options) Unconfigure(cfg engine.Interface) error { return fmt.Errorf("unable to update config: %v", err) } - if err := o.flush(cfg); err != nil { + if err := o.Flush(cfg); err != nil { return err } @@ -93,8 +93,8 @@ func (o Options) Unconfigure(cfg engine.Interface) error { return nil } -// flush flushes the specified config to disk -func (o Options) flush(cfg engine.Interface) error { +// Flush flushes the specified config to disk +func (o Options) Flush(cfg engine.Interface) error { filepath := o.DropInConfig if filepath == "" { filepath = o.TopLevelConfigPath diff --git a/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go b/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go index f16e44b78..92d1e2ed0 100644 --- a/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go +++ b/cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go @@ -88,7 +88,30 @@ func TestCrioConfigLifecycle(t *testing.T) { }, assertCleanupPostConditions: func(t *testing.T, co *container.Options, _ *Options) error { require.NoFileExists(t, co.TopLevelConfigPath) - require.NoFileExists(t, co.DropInConfig) + // drop-in file not removed on cleanup + actual, err := os.ReadFile(co.DropInConfig) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.runtime] + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.nvidia] + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) return nil }, }, @@ -182,8 +205,6 @@ signature_policy = "/etc/crio/policy.json" assertCleanupPostConditions: func(t *testing.T, co *container.Options, o *Options) error { require.FileExists(t, co.TopLevelConfigPath) - require.NoFileExists(t, co.DropInConfig) - actualTopLevel, err := os.ReadFile(co.TopLevelConfigPath) require.NoError(t, err) @@ -203,6 +224,37 @@ signature_policy = "/etc/crio/policy.json" ` require.Equal(t, expectedTopLevel, string(actualTopLevel)) + // drop-in file not removed on cleanup + require.FileExists(t, co.DropInConfig) + actual, err := os.ReadFile(co.DropInConfig) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.runtime] + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.nvidia] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_root = "/run/crun" + runtime_type = "oci" +` + require.Equal(t, expected, string(actual)) return nil }, }, @@ -312,8 +364,33 @@ runtime_type = "oci" require.Equal(t, expectedTopLevel, string(actualTopLevel)) - require.NoFileExists(t, co.DropInConfig) + // drop-in file not removed on cleanup + // default_runtime setting removed from drop-in + require.FileExists(t, co.DropInConfig) + actual, err := os.ReadFile(co.DropInConfig) + require.NoError(t, err) + + expected := ` +[crio] + + [crio.runtime] + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.nvidia] + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_type = "oci" + [crio.runtime.runtimes.nvidia-cdi] + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_type = "oci" +` + + require.Equal(t, expected, string(actual)) return nil }, }, @@ -480,7 +557,37 @@ plugin_dirs = [ ` require.Equal(t, expected, string(actual)) - require.NoFileExists(t, co.DropInConfig) + // drop-in file not removed on cleanup + require.FileExists(t, co.DropInConfig) + actualDropIn, err := os.ReadFile(co.DropInConfig) + require.NoError(t, err) + + expectedDropIn := ` +[crio] + + [crio.runtime] + + [crio.runtime.runtimes] + + [crio.runtime.runtimes.nvidia] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-cdi] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.cdi" + runtime_root = "/run/crun" + runtime_type = "oci" + + [crio.runtime.runtimes.nvidia-legacy] + monitor_path = "/usr/libexec/crio/conmon" + runtime_path = "/usr/bin/nvidia-container-runtime.legacy" + runtime_root = "/run/crun" + runtime_type = "oci" +` + require.Equal(t, expectedDropIn, string(actualDropIn)) return nil }, diff --git a/cmd/nvidia-ctk-installer/container/runtime/crio/crio.go b/cmd/nvidia-ctk-installer/container/runtime/crio/crio.go index d2a32411a..1ba08ecc6 100644 --- a/cmd/nvidia-ctk-installer/container/runtime/crio/crio.go +++ b/cmd/nvidia-ctk-installer/container/runtime/crio/crio.go @@ -133,7 +133,7 @@ func setupHook(o *container.Options, co *Options) error { func setupConfig(o *container.Options) error { log.Infof("Updating config file") - cfg, err := getRuntimeConfig(o) + cfg, err := getRuntimeConfig(o, false) if err != nil { return fmt.Errorf("unable to load config: %v", err) } @@ -180,16 +180,24 @@ func cleanupHook(co *Options) error { // cleanupConfig removes the NVIDIA container runtime from the cri-o config func cleanupConfig(o *container.Options) error { + if !o.SetAsDefault { + return nil + } + log.Infof("Reverting config file modifications") - cfg, err := getRuntimeConfig(o) + cfg, err := getRuntimeConfig(o, true) if err != nil { return fmt.Errorf("unable to load config: %v", err) } - err = o.Unconfigure(cfg) + err = cfg.UpdateDefaultRuntime(o.RuntimeName, engine.UpdateActionUnset) if err != nil { - return fmt.Errorf("unable to unconfigure cri-o: %v", err) + return fmt.Errorf("failed to unset %q as the default runtime: %w", o.RuntimeName, err) + } + + if err := o.Flush(cfg); err != nil { + return err } err = RestartCrio(o) @@ -206,24 +214,35 @@ func RestartCrio(o *container.Options) error { } func GetLowlevelRuntimePaths(o *container.Options) ([]string, error) { - cfg, err := getRuntimeConfig(o) + cfg, err := getRuntimeConfig(o, false) if err != nil { return nil, fmt.Errorf("unable to load crio config: %w", err) } return engine.GetBinaryPathsForRuntimes(cfg), nil } -func getRuntimeConfig(o *container.Options) (engine.Interface, error) { +func getRuntimeConfig(o *container.Options, loadDestinationConfig bool) (engine.Interface, error) { loaders, err := o.GetConfigLoaders(crio.CommandLineSource) if err != nil { return nil, err } - return crio.New( + + options := []crio.Option{ crio.WithTopLevelConfigPath(o.TopLevelConfigPath), crio.WithConfigSource( toml.LoadFirst( loaders..., ), ), - ) + } + + if loadDestinationConfig { + destinationConfigPath := o.TopLevelConfigPath + if o.DropInConfig != "" { + destinationConfigPath = o.DropInConfig + } + options = append(options, crio.WithConfigDestination(toml.FromFile(destinationConfigPath))) + } + + return crio.New(options...) } diff --git a/pkg/config/engine/api.go b/pkg/config/engine/api.go index e3802f5a0..7518c4c9c 100644 --- a/pkg/config/engine/api.go +++ b/pkg/config/engine/api.go @@ -20,6 +20,12 @@ const ( // SaveToSTDOUT is used to write the specified config to stdout instead of // to a file on disk. SaveToSTDOUT = "" + // UpdateActionSet is used as an argument to UpdateDefaultRuntime + // when setting a runtime handler as the default in the config + UpdateActionSet = "set" + // UpdateActionUnset is used as an argument to UpdateDefaultRuntime + // when unsetting a runtime handler as the default in the config + UpdateActionUnset = "unset" ) // Interface defines the API for a runtime config updater. @@ -29,6 +35,7 @@ type Interface interface { EnableCDI() GetRuntimeConfig(string) (RuntimeConfig, error) RemoveRuntime(string) error + UpdateDefaultRuntime(string, string) error Save(string) (int64, error) String() string } diff --git a/pkg/config/engine/config.go b/pkg/config/engine/config.go index 3c286b94a..d24d210ab 100644 --- a/pkg/config/engine/config.go +++ b/pkg/config/engine/config.go @@ -43,6 +43,7 @@ type RuntimeConfigDestination interface { AddRuntimeWithOptions(string, string, bool, interface{}) error EnableCDI() RemoveRuntime(string) error + UpdateDefaultRuntime(string, string) error Save(string) (int64, error) String() string } @@ -60,6 +61,14 @@ func (c *Config) RemoveRuntime(runtime string) error { return c.Destination.RemoveRuntime(runtime) } +// UpdateDefaultRuntime updates the default runtime setting in the destination config. +// When action is 'set' the provided runtime name is set as the default. +// When action is 'unset' we make sure the provided runtime name is not +// the default. +func (c *Config) UpdateDefaultRuntime(runtime string, action string) error { + return c.Destination.UpdateDefaultRuntime(runtime, action) +} + // EnableCDI enables CDI in the destination config. func (c *Config) EnableCDI() { c.Destination.EnableCDI() diff --git a/pkg/config/engine/containerd/config.go b/pkg/config/engine/containerd/config.go index 9cd2e0c0b..86cc08ab9 100644 --- a/pkg/config/engine/containerd/config.go +++ b/pkg/config/engine/containerd/config.go @@ -154,3 +154,33 @@ func (c *Config) RemoveRuntime(name string) error { *c.Tree = config return nil } + +// UpdateDefaultRuntime updates the default runtime setting in the config. +// When action is 'set' the provided runtime name is set as the default. +// When action is 'unset' we make sure the provided runtime name is not +// the default. +func (c *Config) UpdateDefaultRuntime(name string, action string) error { + if action != engine.UpdateActionSet && action != engine.UpdateActionUnset { + return fmt.Errorf("invalid action %q, valid actions are %q and %q", action, engine.UpdateActionSet, engine.UpdateActionUnset) + } + + if c == nil || c.Tree == nil { + if action == engine.UpdateActionSet { + return fmt.Errorf("config toml is nil") + } + return nil + } + + config := *c.Tree + if action == engine.UpdateActionSet { + config.SetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}, name) + } else { + defaultRuntime, ok := config.GetPath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}).(string) + if ok && defaultRuntime == name { + config.DeletePath([]string{"plugins", c.CRIRuntimePluginName, "containerd", "default_runtime_name"}) + } + } + + *c.Tree = config + return nil +} diff --git a/pkg/config/engine/containerd/config_drop_in.go b/pkg/config/engine/containerd/config_drop_in.go index 1a387ff48..8264523e4 100644 --- a/pkg/config/engine/containerd/config_drop_in.go +++ b/pkg/config/engine/containerd/config_drop_in.go @@ -96,6 +96,14 @@ func (c *ConfigWithDropIn) RemoveRuntime(name string) error { return c.Interface.RemoveRuntime(name) } +// UpdateDefaultRuntime updates the default runtime setting in the drop-in config. +// When action is 'set' the provided runtime name is set as the default. +// When action is 'unset' we make sure the provided runtime name is not +// the default. +func (c *ConfigWithDropIn) UpdateDefaultRuntime(name string, action string) error { + return c.Interface.UpdateDefaultRuntime(name, action) +} + // flush saves the top-level config to its path. // If the config is empty, the file will be deleted. func (c *topLevelConfig) Save(dropInPath string) (int64, error) { diff --git a/pkg/config/engine/containerd/config_v1.go b/pkg/config/engine/containerd/config_v1.go index 6e87a1ef9..14b68ed92 100644 --- a/pkg/config/engine/containerd/config_v1.go +++ b/pkg/config/engine/containerd/config_v1.go @@ -120,6 +120,10 @@ func (c *ConfigV1) RemoveRuntime(name string) error { return nil } +func (c *ConfigV1) UpdateDefaultRuntime(name string, action string) error { + return fmt.Errorf("this method is not implemented") +} + // Save writes the config to a file func (c ConfigV1) Save(path string) (int64, error) { return (Config)(c).Save(path) diff --git a/pkg/config/engine/crio/crio.go b/pkg/config/engine/crio/crio.go index 9effd7246..f6819612d 100644 --- a/pkg/config/engine/crio/crio.go +++ b/pkg/config/engine/crio/crio.go @@ -67,13 +67,23 @@ func New(opts ...Option) (engine.Interface, error) { return nil, err } + var destinationConfig *toml.Tree + if b.configDestination != nil { + destinationConfig, err = b.configDestination.Load() + if err != nil { + return nil, err + } + } else { + destinationConfig = toml.NewEmpty() + } + cfg := &engine.Config{ Source: &Config{ Tree: sourceConfig, Logger: b.logger, }, Destination: &Config{ - Tree: toml.NewEmpty(), + Tree: destinationConfig, Logger: b.logger, }, } @@ -167,6 +177,38 @@ func (c *Config) RemoveRuntime(name string) error { return nil } +// UpdateDefaultRuntime updates the default runtime setting in the config. +// When action is 'set' the provided runtime name is set as the default. +// When action is 'unset' we make sure the provided runtime name is not +// the default. +func (c *Config) UpdateDefaultRuntime(name string, action string) error { + if action != engine.UpdateActionSet && action != engine.UpdateActionUnset { + return fmt.Errorf("invalid action %q, valid actions are %q and %q", action, engine.UpdateActionSet, engine.UpdateActionUnset) + } + + if c == nil || c.Tree == nil { + if action == engine.UpdateActionSet { + return fmt.Errorf("config toml is nil") + } + return nil + } + + config := *c.Tree + + if action == engine.UpdateActionSet { + config.SetPath([]string{"crio", "runtime", "default_runtime"}, name) + } else { + if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok { + if runtime == name { + config.DeletePath([]string{"crio", "runtime", "default_runtime"}) + } + } + } + + *c.Tree = config + return nil +} + func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) { if c == nil || c.Tree == nil { return nil, fmt.Errorf("config is nil") diff --git a/pkg/config/engine/crio/option.go b/pkg/config/engine/crio/option.go index 4df7528f2..d6856efc9 100644 --- a/pkg/config/engine/crio/option.go +++ b/pkg/config/engine/crio/option.go @@ -24,6 +24,7 @@ import ( type builder struct { logger logger.Interface configSource toml.Loader + configDestination toml.Loader topLevelConfigPath string } @@ -50,3 +51,10 @@ func WithConfigSource(configSource toml.Loader) Option { b.configSource = configSource } } + +// WithConfigDestination sets the TOML destination for the config. +func WithConfigDestination(configDestination toml.Loader) Option { + return func(b *builder) { + b.configDestination = configDestination + } +} diff --git a/pkg/config/engine/docker/docker.go b/pkg/config/engine/docker/docker.go index 86512df93..2cae9e71f 100644 --- a/pkg/config/engine/docker/docker.go +++ b/pkg/config/engine/docker/docker.go @@ -150,6 +150,39 @@ func (c *Config) RemoveRuntime(name string) error { return nil } +// UpdateDefaultRuntime updates the default runtime setting in the config. +// When action is 'set' the provided runtime name is set as the default. +// When action is 'unset' we make sure the provided runtime name is not +// the default. +func (c *Config) UpdateDefaultRuntime(name string, action string) error { + if action != engine.UpdateActionSet && action != engine.UpdateActionUnset { + return fmt.Errorf("invalid action %q, valid actions are %q and %q", action, engine.UpdateActionSet, engine.UpdateActionUnset) + } + + if c == nil { + if action == engine.UpdateActionSet { + return fmt.Errorf("config toml is nil") + } + return nil + } + + config := *c + + if action == engine.UpdateActionSet { + config["default-runtime"] = name + } else { + if _, exists := config["default-runtime"]; exists { + defaultRuntime := config["default-runtime"].(string) + if defaultRuntime == name { + config["default-runtime"] = defaultDockerRuntime + } + } + } + + *c = config + return nil +} + // Save writes the config to the specified path func (c Config) Save(path string) (int64, error) { output, err := json.MarshalIndent(c, "", " ")