Skip to content

Commit cd56671

Browse files
ArangoGutierrezArangoGutierrez
committed
[no-relnote] Add E2E for libnvidia-container
Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>
1 parent 08b3a38 commit cd56671

File tree

4 files changed

+233
-13
lines changed

4 files changed

+233
-13
lines changed

‎tests/e2e/Makefile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,14 @@ LOG_ARTIFACTS_DIR ?= $(CURDIR)/e2e_logs
2020

2121
GINKGO_BIN := $(CURDIR)/bin/ginkgo
2222

23+
# If GINKGO_FOCUS is not set, run all tests
24+
# current available tests:
25+
# - nvidia-container-cli
26+
# - docker
27+
GINKGO_FOCUS ?=
28+
2329
test: $(GINKGO_BIN)
24-
$(GINKGO_BIN) $(GINKGO_ARGS) -v --json-report ginkgo.json ./tests/e2e/...
30+
$(GINKGO_BIN) $(GINKGO_ARGS) -v --json-report ginkgo.json --focus="$(GINKGO_FOCUS)" ./tests/e2e/...
2531

2632
# test-preinstalled runs the test cases against the version of the toolkit that
2733
# is already installed (and configured for docker) on the host.

‎tests/e2e/e2e_test.go‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,8 @@ func getTestEnv() {
6262

6363
installCTK = getEnvVarOrDefault("E2E_INSTALL_CTK", false)
6464

65-
if installCTK {
66-
imageName = getRequiredEnvvar[string]("E2E_IMAGE_NAME")
67-
68-
imageTag = getRequiredEnvvar[string]("E2E_IMAGE_TAG")
69-
70-
}
65+
imageName = getRequiredEnvvar[string]("E2E_IMAGE_NAME")
66+
imageTag = getRequiredEnvvar[string]("E2E_IMAGE_TAG")
7167

7268
sshHost = getEnvVarOrDefault("E2E_SSH_HOST", "")
7369
if sshHost != "" {

‎tests/e2e/installer.go‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,18 @@ var dockerInstallTemplate = `
2828
#! /usr/bin/env bash
2929
set -xe
3030
31-
: ${IMAGE:={{.Image}}}
32-
33-
# Create a temporary directory
34-
TEMP_DIR="/tmp/ctk_e2e.$(date +%s)_$RANDOM"
35-
mkdir -p "$TEMP_DIR"
31+
# if the TEMP_DIR is already set, use it
32+
if [ -f /tmp/ctk_e2e_temp_dir.txt ]; then
33+
TEMP_DIR=$(cat /tmp/ctk_e2e_temp_dir.txt)
34+
else
35+
TEMP_DIR="/tmp/ctk_e2e.$(date +%s)_$RANDOM"
36+
echo "$TEMP_DIR" > /tmp/ctk_e2e_temp_dir.txt
37+
fi
38+
39+
# if TEMP_DIR does not exist, create it
40+
if [ ! -d "$TEMP_DIR" ]; then
41+
mkdir -p "$TEMP_DIR"
42+
fi
3643
3744
# Given that docker has an init function that checks for the existence of the
3845
# nvidia-container-toolkit, we need to create a symlink to the nvidia-container-runtime-hook
@@ -46,7 +53,7 @@ docker run --pid=host --rm -i --privileged \
4653
-v /var/run/docker.sock:/var/run/docker.sock \
4754
-v "$TEMP_DIR:$TEMP_DIR" \
4855
-v /etc/docker:/config-root \
49-
${IMAGE} \
56+
{{.Image}} \
5057
--root "$TEMP_DIR" \
5158
--runtime=docker \
5259
--config=/config-root/daemon.json \

‎tests/e2e/nvidia-container-cli_test.go‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
* Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
"text/template"
24+
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
)
28+
29+
const (
30+
libnvidiaContainerCliTestTemplate = `#!/bin/bash
31+
set -euo pipefail
32+
export DEBIAN_FRONTEND=noninteractive
33+
34+
# Add Docker's official GPG key:
35+
apt-get update
36+
apt-get install -y ca-certificates curl apt-utils gnupg2
37+
install -m 0755 -d /etc/apt/keyrings
38+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
39+
chmod a+r /etc/apt/keyrings/docker.asc
40+
41+
# Add the repository to Apt sources:
42+
echo \
43+
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
44+
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
45+
tee /etc/apt/sources.list.d/docker.list > /dev/null
46+
apt-get update
47+
48+
apt-get install -y docker-ce docker-ce-cli containerd.io
49+
50+
# start dockerd in the background
51+
dockerd &
52+
53+
# wait for dockerd to be ready
54+
while ! docker info > /dev/null 2>&1; do
55+
echo "Waiting for dockerd to be ready..."
56+
sleep 1
57+
done
58+
59+
# Create a temporary directory and rootfs path
60+
TMPDIR="$(mktemp -d)"
61+
ROOTFS="${TMPDIR}/rootfs"
62+
mkdir -p "${ROOTFS}"
63+
64+
# Expose ROOTFS for the child namespace
65+
export ROOTFS TMPDIR
66+
67+
echo "Copying package files from ${NVIDIA_TOOLKIT_IMAGE} to ${ROOTFS}"
68+
docker run --rm \
69+
-v ${TMPDIR}:/host-tmpdir \
70+
--entrypoint="sh" \
71+
${NVIDIA_TOOLKIT_IMAGE}-packaging \
72+
-c "cp -p -R /artifacts/* /host-tmpdir/"
73+
74+
dpkg -i ${TMPDIR}/packages/ubuntu18.04/amd64/libnvidia-container1_*_amd64.deb \
75+
${TMPDIR}/packages/ubuntu18.04/amd64/nvidia-container-toolkit-base_*_amd64.deb \
76+
${TMPDIR}/packages/ubuntu18.04/amd64/libnvidia-container-tools_*_amd64.deb
77+
78+
nvidia-container-cli --version
79+
80+
curl http://cdimage.ubuntu.com/ubuntu-base/releases/22.04/release/ubuntu-base-22.04-base-amd64.tar.gz | tar -C $ROOTFS -xz
81+
82+
# Enter a new mount + PID namespace so we can pivot_root without touching the
83+
# container's original filesystem.
84+
unshare --mount --pid --fork --propagation private -- sh -eux <<'IN_NS'
85+
: "${ROOTFS:?}"
86+
87+
# 1 Bind-mount the new root and make the mount private
88+
mount --bind "$ROOTFS" "$ROOTFS"
89+
mount --make-private "$ROOTFS"
90+
cd "$ROOTFS"
91+
92+
# 2 Minimal virtual filesystems
93+
mount -t proc proc proc
94+
mount -t sysfs sys sys
95+
mount -t tmpfs tmp tmp
96+
mount -t tmpfs run run
97+
98+
nvidia-container-cli --load-kmods \
99+
configure \
100+
--no-cgroups --utility --device=0 "$(pwd)"
101+
102+
# 4 Switch root into the prepared filesystem
103+
mkdir -p mnt
104+
pivot_root . mnt
105+
umount -l /mnt
106+
107+
exec nvidia-smi -L
108+
IN_NS
109+
`
110+
111+
dockerRunCmdTemplate = `docker run --name {{.ContainerName}} --privileged --runtime=nvidia \
112+
-e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all \
113+
-e NVIDIA_DRIVER_CAPABILITIES=all \
114+
-e NVIDIA_TOOLKIT_IMAGE={{.ToolkitImage}} \
115+
-v {{.ScriptPath}}:/libnvidia-container-cli.sh \
116+
ubuntu /libnvidia-container-cli.sh`
117+
)
118+
119+
var _ = Describe("nvidia-container-cli", Ordered, ContinueOnFailure, func() {
120+
var (
121+
runner Runner
122+
containerName = "nvidia-cli-e2e"
123+
hostOutput string
124+
testScriptPath = "/tmp/libnvidia-container-cli.sh"
125+
)
126+
127+
BeforeAll(func(ctx context.Context) {
128+
runner = NewRunner(
129+
WithHost(sshHost),
130+
WithPort(sshPort),
131+
WithSshKey(sshKey),
132+
WithSshUser(sshUser),
133+
)
134+
135+
if installCTK {
136+
installer, err := NewToolkitInstaller(
137+
WithRunner(runner),
138+
WithImage(imageName+":"+imageTag),
139+
WithTemplate(dockerInstallTemplate),
140+
)
141+
Expect(err).ToNot(HaveOccurred())
142+
143+
err = installer.Install()
144+
Expect(err).ToNot(HaveOccurred())
145+
}
146+
147+
// Capture the host GPU list.
148+
var err error
149+
hostOutput, _, err = runner.Run("nvidia-smi -L")
150+
Expect(err).ToNot(HaveOccurred())
151+
152+
// Normalize the output once
153+
hostOutput = strings.TrimSpace(strings.ReplaceAll(hostOutput, "\r", ""))
154+
155+
// If a container with the same name exists from a previous test run, remove it first.
156+
_, _, err = runner.Run(fmt.Sprintf("docker rm -f %s", containerName))
157+
Expect(err).ToNot(HaveOccurred())
158+
})
159+
160+
AfterAll(func(ctx context.Context) {
161+
// Cleanup: remove the container and the temporary script on the host.
162+
runner.Run(fmt.Sprintf("docker rm -f %s", containerName))
163+
164+
// Remove the script from the remote host.
165+
runner.Run(fmt.Sprintf("rm -f %s", testScriptPath))
166+
})
167+
168+
It("should report the same GPUs inside the container as on the host", func(ctx context.Context) {
169+
// Write the script to the remote host and make it executable.
170+
createScriptCmd := fmt.Sprintf(
171+
"cat > %s <<'EOF'\n%s\nEOF\nchmod +x %s",
172+
testScriptPath, libnvidiaContainerCliTestTemplate, testScriptPath,
173+
)
174+
175+
_, _, err := runner.Run(createScriptCmd)
176+
Expect(err).ToNot(HaveOccurred())
177+
178+
// Build the docker run command (detached mode) from the template so it
179+
// stays readable while still resulting in a single-line invocation.
180+
tmpl, err := template.New("dockerRun").Parse(dockerRunCmdTemplate)
181+
Expect(err).ToNot(HaveOccurred())
182+
183+
var dockerRunCmd strings.Builder
184+
err = tmpl.Execute(&dockerRunCmd, struct {
185+
ContainerName string
186+
ToolkitImage string
187+
ScriptPath string
188+
}{
189+
ContainerName: containerName,
190+
ToolkitImage: imageName + ":" + imageTag,
191+
ScriptPath: testScriptPath,
192+
})
193+
Expect(err).ToNot(HaveOccurred())
194+
195+
// Launch the container in detached mode.
196+
_, _, err = runner.Run(dockerRunCmd.String())
197+
Expect(err).ToNot(HaveOccurred())
198+
199+
// Poll the logs of the already running container until we observe
200+
// the GPU list matching the host or until a 5-minute timeout elapses.
201+
Eventually(func() string {
202+
logs, _, err := runner.Run(fmt.Sprintf("docker logs --tail 1 %s", containerName))
203+
if err != nil {
204+
return fmt.Sprintf("error: %v", err)
205+
}
206+
207+
// Docker already returns only the last line due to --tail 1, so just normalize.
208+
return strings.TrimSpace(strings.ReplaceAll(logs, "\r", ""))
209+
}, "5m", "10s").Should(Equal(hostOutput))
210+
})
211+
})

0 commit comments

Comments
 (0)