Skip to content

Commit 014d91b

Browse files
elezarelezar
committed
USE Matchers
Signed-off-by: Evan Lezar <[email protected]>
1 parent 1d40ebe commit 014d91b

File tree

2 files changed

+111
-114
lines changed

2 files changed

+111
-114
lines changed

‎tests/e2e/nvidia-ctk_containerd_test.go‎

Lines changed: 107 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ package e2e
1919
import (
2020
"context"
2121
"fmt"
22+
"path/filepath"
2223

2324
. "github.com/onsi/ginkgo/v2"
2425
. "github.com/onsi/gomega"
26+
"github.com/onsi/gomega/types"
2527
"github.com/pelletier/go-toml"
2628
)
2729

@@ -49,41 +51,38 @@ exit 1
4951

5052
// containerdTestEnv defines the test environment for different containerd versions
5153
type containerdTestEnv struct {
52-
name string
53-
image string
54-
configVersion int
55-
pluginPath string
56-
hasDefaultImports bool
54+
name string
55+
image string
56+
configVersion int64
57+
pluginPath string
5758
}
5859

5960
// Define both containerd versions to test
6061
var containerdEnvs = []containerdTestEnv{
6162
{
62-
name: "containerd-1.7",
63-
image: "kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e",
64-
configVersion: 2,
65-
pluginPath: "io.containerd.grpc.v1.cri",
66-
hasDefaultImports: false,
63+
name: "containerd-1.7",
64+
image: "kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e",
65+
configVersion: 2,
66+
pluginPath: "io.containerd.grpc.v1.cri",
6767
},
6868
{
69-
name: "containerd-2.1",
70-
image: "docker.io/kindest/base:v20250521-31a79fd4",
71-
configVersion: 3,
72-
pluginPath: "io.containerd.cri.v1.runtime",
73-
hasDefaultImports: true,
69+
name: "containerd-2.1",
70+
image: "docker.io/kindest/base:v20250521-31a79fd4",
71+
configVersion: 3,
72+
pluginPath: "io.containerd.cri.v1.runtime",
7473
},
7574
}
7675

7776
// Integration tests for containerd drop-in config functionality
7877
var _ = Describe("containerd", Ordered, ContinueOnFailure, Label("container-runtime"), func() {
7978
// Run all tests for each containerd version
8079
for _, env := range containerdEnvs {
81-
env := env // capture loop variable
82-
83-
Context(fmt.Sprintf("with %s", env.name), Ordered, func() {
80+
Context(env.name, Ordered, func() {
8481
var (
85-
nestedContainerRunner Runner
86-
containerName = fmt.Sprintf("nvctk-e2e-containerd-%s-tests", env.name)
82+
nestedContainerRunner Runner
83+
containerName = "nvctk-e2e-containerd-tests-" + env.name
84+
originalTopLevelConfigContents string
85+
// originalTopLevelConfigToml *toml.Tree
8786
)
8887

8988
// restartContainerdAndWait restarts containerd and waits for it to be ready
@@ -104,7 +103,15 @@ var _ = Describe("containerd", Ordered, ContinueOnFailure, Label("container-runt
104103
var err error
105104

106105
// Create the nested container with the global cache mounted
107-
nestedContainerRunner, err = NewNestedContainerRunner(runner, env.image, installCTK, containerName, localCacheDir)
106+
// TODO: This runner doesn't actually NEED GPU access.
107+
nestedContainerRunner, err = NewNestedContainerRunner(runner, env.image, false, containerName, localCacheDir)
108+
Expect(err).ToNot(HaveOccurred())
109+
110+
// Store the contents of the original config.
111+
originalTopLevelConfigContents, _, err = nestedContainerRunner.Run("cat /etc/containerd/config.toml")
112+
Expect(err).ToNot(HaveOccurred())
113+
114+
_, err = toml.Load(originalTopLevelConfigContents)
108115
Expect(err).ToNot(HaveOccurred())
109116

110117
// Backup original containerd configuration
@@ -113,7 +120,7 @@ var _ = Describe("containerd", Ordered, ContinueOnFailure, Label("container-runt
113120
if [ -d /etc/containerd/conf.d ]; then
114121
cp -r /etc/containerd/conf.d /tmp/containerd-conf.d.backup
115122
fi
116-
123+
117124
# Backup the original config.toml
118125
if [ -f /etc/containerd/config.toml ]; then
119126
cp /etc/containerd/config.toml /tmp/containerd-config.toml.backup
@@ -131,29 +138,6 @@ var _ = Describe("containerd", Ordered, ContinueOnFailure, Label("container-runt
131138
`)
132139
Expect(err).ToNot(HaveOccurred(), "Failed to ensure containerd is running")
133140

134-
// Version-specific setup
135-
if !env.hasDefaultImports {
136-
// For containerd 1.7.x, ensure imports directive exists
137-
_, _, err = nestedContainerRunner.Run(`set -e
138-
# Ensure config file exists
139-
if [ ! -f /etc/containerd/config.toml ]; then
140-
mkdir -p /etc/containerd
141-
containerd config default > /etc/containerd/config.toml
142-
fi
143-
144-
# Add imports directive if not present
145-
if ! grep -q "imports" /etc/containerd/config.toml; then
146-
# Create imports line
147-
cat > /tmp/imports.line <<EOF
148-
imports = ["/etc/containerd/conf.d/*.toml"]
149-
EOF
150-
# Prepend to existing config
151-
cat /etc/containerd/config.toml >> /tmp/imports.line
152-
mv /tmp/imports.line /etc/containerd/config.toml
153-
fi`)
154-
Expect(err).ToNot(HaveOccurred(), "Failed to add imports directive for containerd 1.7")
155-
}
156-
157141
// Install the NVIDIA Container Toolkit packages
158142
_, _, err = toolkitInstaller.Install(nestedContainerRunner)
159143
Expect(err).ToNot(HaveOccurred(), "Failed to install toolkit for containerd")
@@ -183,7 +167,7 @@ fi`)
183167
rm -rf /etc/containerd/conf.d
184168
mkdir -p /etc/containerd/conf.d
185169
fi
186-
170+
187171
# Restore the original config.toml
188172
if [ -f /tmp/containerd-config.toml.backup ]; then
189173
cp /tmp/containerd-config.toml.backup /etc/containerd/config.toml
@@ -202,6 +186,17 @@ fi`)
202186
_, _, err := nestedContainerRunner.Run(`nvidia-ctk runtime configure --runtime=containerd --config=/etc/containerd/config.toml --drop-in-config=/etc/containerd/conf.d/99-nvidia.toml --set-as-default --cdi.enabled`)
203187
Expect(err).ToNot(HaveOccurred(), "Failed to configure containerd")
204188

189+
topLevelConfigContents, _, err := nestedContainerRunner.Run("cat /etc/containerd/config.toml")
190+
Expect(err).ToNot(HaveOccurred())
191+
192+
_, err = toml.Load(topLevelConfigContents)
193+
Expect(err).ToNot(HaveOccurred())
194+
195+
dropInConfigContents, _, err := nestedContainerRunner.Run("cat /etc/containerd/conf.d/99-nvidia.toml")
196+
Expect(err).ToNot(HaveOccurred())
197+
198+
Expect(dropInConfigContents).ToNot(BeEmpty())
199+
205200
// restart containerd
206201
err = restartContainerdAndWait(nestedContainerRunner)
207202
Expect(err).ToNot(HaveOccurred(), "Failed to restart containerd")
@@ -211,63 +206,64 @@ fi`)
211206
Expect(err).ToNot(HaveOccurred())
212207

213208
// Parse the TOML output
214-
config, err := parseContainerdConfig(output)
209+
config, err := toml.Load(output)
215210
Expect(err).ToNot(HaveOccurred(), "Failed to parse containerd config")
216211

217-
// Verify config version
218-
version := config.Get("version")
219-
Expect(version).To(Equal(int64(env.configVersion)))
220-
221-
// Verify imports
222-
if env.configVersion == 2 {
223-
// containerd 1.7 shows actual resolved imports
224-
err = validateImports(config, []string{"/etc/containerd/conf.d/99-nvidia.toml"}, true)
225-
Expect(err).ToNot(HaveOccurred(), "Import validation failed")
226-
} else {
227-
// containerd 2.x shows the glob pattern
228-
imports := config.Get("imports")
229-
if imports != nil {
230-
importsList := imports.([]interface{})
231-
found := false
232-
for _, imp := range importsList {
233-
if impStr, ok := imp.(string); ok && impStr == "/etc/containerd/conf.d/*.toml" {
234-
found = true
235-
break
236-
}
237-
}
238-
Expect(found).To(BeTrue(), "Expected import pattern not found")
239-
}
240-
}
241-
242-
// Get plugin configuration
243-
pluginConfig, err := getPluginConfig(config, env.configVersion)
244-
Expect(err).ToNot(HaveOccurred(), "Failed to get plugin config")
245-
246-
// Verify CDI is enabled
247-
cdiEnabled, err := getCDIEnabled(pluginConfig, env.configVersion)
248-
Expect(err).ToNot(HaveOccurred(), "Failed to get CDI config")
249-
Expect(cdiEnabled).To(BeTrue(), "CDI should be enabled")
250-
251-
// Verify default runtime
252-
defaultRuntime, err := getDefaultRuntime(pluginConfig)
253-
Expect(err).ToNot(HaveOccurred(), "Failed to get default runtime")
254-
Expect(defaultRuntime).To(Equal("nvidia"), "Default runtime should be nvidia")
255-
256-
// Get runtimes configuration
257-
runtimes, err := getRuntimesConfig(pluginConfig, env.configVersion)
258-
Expect(err).ToNot(HaveOccurred(), "Failed to get runtimes config")
259-
260-
// Verify NVIDIA runtime exists and is properly configured
261-
nvidiaRuntime, exists := runtimes["nvidia"]
262-
Expect(exists).To(BeTrue(), "nvidia runtime should exist")
263-
264-
// Validate nvidia runtime configuration
265-
expectedOptions := map[string]interface{}{
266-
"BinaryName": "/usr/bin/nvidia-container-runtime",
267-
"SystemdCgroup": true,
212+
BeAValidConfig := func(env *containerdTestEnv) types.GomegaMatcher {
213+
return And(
214+
WithTransform(
215+
func(c *toml.Tree) map[string]any {
216+
return c.ToMap()
217+
},
218+
And(
219+
HaveKeyWithValue("version", env.configVersion),
220+
HaveKeyWithValue("imports", WithTransform(func(is []any) []string {
221+
var basePaths []string
222+
for _, i := range is {
223+
basePaths = append(basePaths, filepath.Dir(i.(string)))
224+
}
225+
return basePaths
226+
},
227+
// TODO: We could do better at matching the following:
228+
// /etc/containerd/conf.d/99-nvidia.toml
229+
// /etc/containerd/conf.d/*.toml
230+
ContainElements("/etc/containerd/conf.d"),
231+
)),
232+
),
233+
),
234+
WithTransform(
235+
// Get the plugins config.
236+
func(c *toml.Tree) map[string]any {
237+
pt := c.GetPath([]string{"plugins", env.pluginPath})
238+
if pt != nil {
239+
return pt.(*toml.Tree).ToMap()
240+
}
241+
return nil
242+
},
243+
And(
244+
HaveKeyWithValue("enable_cdi", true),
245+
HaveKeyWithValue("containerd",
246+
And(
247+
// TODO: This should depend on whether we set the default.
248+
HaveKeyWithValue("default_runtime_name", "nvidia"),
249+
HaveKeyWithValue("runtimes", HaveKeyWithValue("nvidia",
250+
And(
251+
HaveKeyWithValue("runtime_type", "io.containerd.runc.v2"),
252+
HaveKeyWithValue("options",
253+
And(
254+
HaveKeyWithValue("BinaryName", "/usr/bin/nvidia-container-runtime"),
255+
HaveKeyWithValue("SystemdCgroup", true),
256+
),
257+
),
258+
),
259+
)),
260+
),
261+
),
262+
),
263+
),
264+
)
268265
}
269-
err = validateRuntimeConfig(nvidiaRuntime, "", expectedOptions)
270-
Expect(err).ToNot(HaveOccurred(), "NVIDIA runtime validation failed")
266+
Expect(config).To(BeAValidConfig(&env))
271267
})
272268
})
273269

@@ -284,14 +280,14 @@ fi`)
284280
[plugins."io.containerd.grpc.v1.cri"]
285281
[plugins."io.containerd.grpc.v1.cri".containerd]
286282
default_runtime_name = "kata"
287-
283+
288284
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
289285
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
290286
runtime_type = "io.containerd.runc.v2"
291-
287+
292288
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
293289
runtime_type = "io.containerd.kata.v2"
294-
290+
295291
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
296292
ConfigPath = "/etc/kata-containers/configuration.toml"`
297293
} else {
@@ -302,14 +298,14 @@ fi`)
302298
[plugins."io.containerd.cri.v1.runtime"]
303299
[plugins."io.containerd.cri.v1.runtime".containerd]
304300
default_runtime_name = "kata"
305-
301+
306302
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
307303
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
308304
runtime_type = "io.containerd.runc.v2"
309-
305+
310306
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata]
311307
runtime_type = "io.containerd.kata.v2"
312-
308+
313309
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options]
314310
ConfigPath = "/etc/kata-containers/configuration.toml"`
315311
}
@@ -391,11 +387,11 @@ version = 3
391387
[plugins."io.containerd.cri.v1.runtime"]
392388
[plugins."io.containerd.cri.v1.runtime".containerd]
393389
default_runtime_name = "runc"
394-
390+
395391
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
396392
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
397393
runtime_type = "io.containerd.runc.v2"
398-
394+
399395
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
400396
BinaryName = "/usr/bin/runc"
401397
SystemdCgroup = true
@@ -586,7 +582,7 @@ func tomlTreeToMap(tree *toml.Tree) map[string]interface{} {
586582
}
587583

588584
// getPluginConfig navigates to the appropriate plugin configuration based on containerd version
589-
func getPluginConfig(tree *toml.Tree, version int) (*toml.Tree, error) {
585+
func getPluginConfig(tree *toml.Tree, version int64) (*toml.Tree, error) {
590586
var pluginPath []string
591587
if version == 2 {
592588
pluginPath = []string{"plugins", "io.containerd.grpc.v1.cri"}
@@ -611,7 +607,7 @@ func getPluginConfig(tree *toml.Tree, version int) (*toml.Tree, error) {
611607
}
612608

613609
// getRuntimesConfig gets the runtimes configuration from the plugin config
614-
func getRuntimesConfig(pluginConfig *toml.Tree, version int) (map[string]interface{}, error) {
610+
func getRuntimesConfig(pluginConfig *toml.Tree, _ int64) (map[string]interface{}, error) {
615611
containerdSection := pluginConfig.Get("containerd")
616612
if containerdSection == nil {
617613
return nil, fmt.Errorf("containerd section not found")
@@ -639,7 +635,7 @@ func getRuntimesConfig(pluginConfig *toml.Tree, version int) (map[string]interfa
639635
}
640636

641637
// getCDIEnabled checks if CDI is enabled in the plugin configuration
642-
func getCDIEnabled(pluginConfig *toml.Tree, version int) (bool, error) {
638+
func getCDIEnabled(pluginConfig *toml.Tree, _ int64) (bool, error) {
643639
cdiEnabled := pluginConfig.Get("enable_cdi")
644640
if cdiEnabled == nil {
645641
return false, nil // CDI not configured, default is false

‎tests/e2e/runner.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func NewRunner(opts ...runnerOption) Runner {
100100
// NewNestedContainerRunner creates a new nested container runner.
101101
// A nested container runs a container inside another container based on a
102102
// given runner (remote or local).
103-
func NewNestedContainerRunner(runner Runner, baseImage string, installCTK bool, containerName string, cacheDir string) (Runner, error) {
103+
func NewNestedContainerRunner(runner Runner, baseImage string, mountToolkitFromHost bool, containerName string, cacheDir string) (Runner, error) {
104104
// If a container with the same name exists from a previous test run, remove it first.
105105
// Ignore errors as container might not exist
106106
_, _, err := runner.Run(fmt.Sprintf("docker rm -f %s 2>/dev/null || true", containerName))
@@ -116,7 +116,8 @@ func NewNestedContainerRunner(runner Runner, baseImage string, installCTK bool,
116116
)
117117
}
118118

119-
if !installCTK {
119+
if mountToolkitFromHost {
120+
// TODO: This is actually ONLY needed for the CLI tests.
120121
// If installCTK is false, we use the preinstalled toolkit.
121122
// This means we need to add toolkit libraries and binaries from the "host"
122123

@@ -312,7 +313,7 @@ type outerContainer struct {
312313
}
313314

314315
func (o *outerContainer) Render() (string, error) {
315-
tmpl, err := template.New("startContainer").Parse(`docker run -d --name {{.Name}} --privileged --runtime=nvidia \
316+
tmpl, err := template.New("startContainer").Parse(`docker run -d --name {{.Name}} --privileged \
316317
-e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all \
317318
-e NVIDIA_DRIVER_CAPABILITIES=all \
318319
{{ range $i, $a := .AdditionalArguments -}}

0 commit comments

Comments
 (0)