Add CRI plugin config from source containerd config to drop-in file

This change updates the drop-in file contents that the nvidia-ctk-installer creates
for containerd. In addition to adding nvidia runtimes, the nvidia-ctk-installer
also includes all existing configuration present in the source containerd configuration
for the CRI plugin. That is, all configuration already present in the
'plugins."io.containerd.grpc.v1.cri"' section will be added to our drop-in file.

This is needed due to how containerd merges configuration from multiple files.
See containerd/containerd#5837

Signed-off-by: Christopher Desiniotis <[email protected]>

Author: cdesiniotis

cmd/nvidia-ctk-installer/container/container.go

@@ -17,6 +17,7 @@
 package container
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -69,7 +70,23 @@ func (o Options) Unconfigure(cfg engine.Interface) error {
 	if err != nil {
 		return fmt.Errorf("unable to update config: %v", err)
 	}
-	return o.flush(cfg)
+
+	if err := o.flush(cfg); err != nil {
+		return err
+	}
+
+	if o.DropInConfig == "" {
+		return nil
+	}
+	// When a drop-in config is used, we remove the drop-in file explicitly.
+	// This is require for cases where we may have to include other contents
+	// in the drop-in file and as such it may not be empty when we flush it.
+	err = os.Remove(o.DropInConfig)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove drop-in config file: %w", err)
+	}
+
+	return nil
 }
 
 // flush flushes the specified config to disk

cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go

@@ -789,6 +789,7 @@ version = 2
     enable_cdi = true
 
     [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
 
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
 
@@ -809,6 +810,12 @@ version = 2
 
           [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
             BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
 `
 				require.Equal(t, expectedDropIn, string(actualDropIn))
 				return nil
@@ -954,6 +961,12 @@ version = 2
 
           [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
             BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
 `
 				require.Equal(t, expectedDropIn, string(actualDropIn))
 				return nil
@@ -1106,9 +1119,17 @@ version = 2
     enable_cdi = true
 
     [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      snapshotter = "overlayfs"
 
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
 
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.custom]
+          runtime_type = "io.containerd.custom.v1"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.custom.options]
+            TypeUrl = "custom.runtime/options"
+
         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
           container_annotations = ["cdi.k8s.io*"]
           runtime_type = "io.containerd.runc.v2"
@@ -1132,6 +1153,20 @@ version = 2
           [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
             BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
             SystemdCgroup = true
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
+            SystemdCgroup = true
+
+    [plugins."io.containerd.grpc.v1.cri".registry]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
 `
 				require.Equal(t, expectedDropIn, string(actualDropIn))
 				return nil
@@ -1278,6 +1313,12 @@ version = 2
 
           [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
             BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
 `
 
 				require.Equal(t, expectedDropIn, string(actualDropIn))
@@ -1407,6 +1448,12 @@ version = 2
 
           [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
             BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
 `
 				require.Equal(t, expectedDropIn, string(actualDropIn))
 				return nil
@@ -1533,6 +1580,12 @@ version = 3
 
           [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-legacy.options]
             BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
+
+        [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
 `
 				require.Equal(t, expectedDropIn, string(actualDropIn))
 
@@ -1678,6 +1731,15 @@ version = 3
             NoPivotRoot = false
             Root = "/run/containerd/runc"
             SystemdCgroup = true
+
+        [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
+            NoPivotRoot = false
+            Root = "/run/containerd/runc"
+            SystemdCgroup = true
 `
 				require.Equal(t, expectedDropIn, string(actualDropIn))
 

pkg/config/engine/containerd/config_test.go

@@ -94,7 +94,15 @@ func TestAddRuntime(t *testing.T) {
 			[plugins]
 			[plugins."io.containerd.grpc.v1.cri"]
 				[plugins."io.containerd.grpc.v1.cri".containerd]
-				[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+                    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+                    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+					privileged_without_host_devices = true
+					runtime_engine = "engine"
+					runtime_root = "root"
+					runtime_type = "type"
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+						BinaryName = "/usr/bin/runc"
+						SystemdCgroup = true
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
 					privileged_without_host_devices = true
 					runtime_engine = "engine"
@@ -128,7 +136,16 @@ func TestAddRuntime(t *testing.T) {
 			[plugins]
 			[plugins."io.containerd.grpc.v1.cri"]
 				[plugins."io.containerd.grpc.v1.cri".containerd]
+				default_runtime_name = "default"
 				[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.default]
+					privileged_without_host_devices = true
+					runtime_engine = "engine"
+					runtime_root = "root"
+					runtime_type = "type"
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.default.options]
+						BinaryName = "/usr/bin/default"
+						SystemdCgroup = true
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
 					privileged_without_host_devices = true
 					runtime_engine = "engine"
@@ -170,7 +187,24 @@ func TestAddRuntime(t *testing.T) {
 			[plugins]
 			[plugins."io.containerd.grpc.v1.cri"]
 				[plugins."io.containerd.grpc.v1.cri".containerd]
+				default_runtime_name = "default"
 				[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+					privileged_without_host_devices = true
+					runtime_engine = "engine"
+					runtime_root = "root"
+					runtime_type = "type"
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+						BinaryName = "/usr/bin/runc"
+						SystemdCgroup = true
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.default]
+					privileged_without_host_devices = false
+					runtime_engine = "defaultengine"
+					runtime_root = "defaultroot"
+					runtime_type = "defaulttype"
+					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.default.options]
+						BinaryName = "/usr/bin/default"
+						SystemdCgroup = false
 					[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
 					privileged_without_host_devices = false
 					runtime_engine = "defaultengine"
@@ -225,6 +259,14 @@ func TestAddRuntime(t *testing.T) {
 			[plugins."io.containerd.cri.v1.runtime"]
 				[plugins."io.containerd.cri.v1.runtime".containerd]
 				[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
+					privileged_without_host_devices = true
+					runtime_engine = "engine"
+					runtime_root = "root"
+					runtime_type = "type"
+					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
+						BinaryName = "/usr/bin/runc"
+						SystemdCgroup = true
 					[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.test]
 					privileged_without_host_devices = true
 					runtime_engine = "engine"

pkg/config/engine/containerd/containerd.go

@@ -133,10 +133,15 @@ func New(opts ...Option) (engine.Interface, error) {
 		}
 		dropInConfig := &engine.Config{
 			Source: sourceConfig,
-			// The destinationConfig is an empty config with the same options
-			// as the source config.
+			// The destinationConfig is a minimal config with the same options
+			// as the source config. The starting content of the destinationConfig
+			// is the entire plugins."io.containerd.grpc.v1.cri" section of the source
+			// config. This is needed due to how containerd merges configuration from
+			// multiple files. In particular, plugins are overridden by key and the
+			// content is not merged. This issue is fixed starting in containerd 2.1
+			// Reference: https://github.com/containerd/containerd/issues/5837
 			Destination: &Config{
-				Tree:          toml.NewEmpty(),
+				Tree:          getBaseDropInConfigTree(sourceConfig),
 				configOptions: sourceConfigOptions,
 			},
 		}
@@ -204,3 +209,17 @@ func chrootIfRequired(hostRoot string, commandLine ...string) []string {
 
 	return append([]string{"chroot", hostRoot}, commandLine...)
 }
+
+func getBaseDropInConfigTree(sourceConfig *Config) *toml.Tree {
+// getBaseDropInConfigTree returns the base config to use for the drop-in config file.
+// For older containerd versions (e.g. 1.7) the plugins section of multiple
+// configs are not fully merged, but merged by key instead. This means that we
+// need to duplicate the entire plugin-specific config in the drop in file so as
+// to not override settings applied in the base config.
+	baseDropInConfigTree := toml.NewEmpty()
+	criPlugin := sourceConfig.GetSubtreeByPath([]string{"plugins", sourceConfig.CRIRuntimePluginName})
+	if criPlugin != nil {
+		baseDropInConfigTree.SetPath([]string{"plugins", sourceConfig.CRIRuntimePluginName}, criPlugin)
+	}
+	return baseDropInConfigTree
+}