Skip to content

Commit 55205e8

Browse files
ArangoGutierrezArangoGutierrez
committed
[no-relnote] Add E2E for libnvidia-container
Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>
1 parent 08b3a38 commit 55205e8

File tree

4 files changed

+264
-13
lines changed

4 files changed

+264
-13
lines changed

‎tests/e2e/Makefile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,14 @@ LOG_ARTIFACTS_DIR ?= $(CURDIR)/e2e_logs
2020

2121
GINKGO_BIN := $(CURDIR)/bin/ginkgo
2222

23+
# If GINKGO_FOCUS is not set, run all tests
24+
# current available tests:
25+
# - nvidia-container-cli
26+
# - docker
27+
GINKGO_FOCUS ?=
28+
2329
test: $(GINKGO_BIN)
24-
$(GINKGO_BIN) $(GINKGO_ARGS) -v --json-report ginkgo.json ./tests/e2e/...
30+
$(GINKGO_BIN) $(GINKGO_ARGS) -v --json-report ginkgo.json --focus="$(GINKGO_FOCUS)" ./tests/e2e/...
2531

2632
# test-preinstalled runs the test cases against the version of the toolkit that
2733
# is already installed (and configured for docker) on the host.

‎tests/e2e/e2e_test.go‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,8 @@ func getTestEnv() {
6262

6363
installCTK = getEnvVarOrDefault("E2E_INSTALL_CTK", false)
6464

65-
if installCTK {
66-
imageName = getRequiredEnvvar[string]("E2E_IMAGE_NAME")
67-
68-
imageTag = getRequiredEnvvar[string]("E2E_IMAGE_TAG")
69-
70-
}
65+
imageName = getRequiredEnvvar[string]("E2E_IMAGE_NAME")
66+
imageTag = getRequiredEnvvar[string]("E2E_IMAGE_TAG")
7167

7268
sshHost = getEnvVarOrDefault("E2E_SSH_HOST", "")
7369
if sshHost != "" {

‎tests/e2e/installer.go‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,18 @@ var dockerInstallTemplate = `
2828
#! /usr/bin/env bash
2929
set -xe
3030
31-
: ${IMAGE:={{.Image}}}
32-
33-
# Create a temporary directory
34-
TEMP_DIR="/tmp/ctk_e2e.$(date +%s)_$RANDOM"
35-
mkdir -p "$TEMP_DIR"
31+
# if the TEMP_DIR is already set, use it
32+
if [ -f /tmp/ctk_e2e_temp_dir.txt ]; then
33+
TEMP_DIR=$(cat /tmp/ctk_e2e_temp_dir.txt)
34+
else
35+
TEMP_DIR="/tmp/ctk_e2e.$(date +%s)_$RANDOM"
36+
echo "$TEMP_DIR" > /tmp/ctk_e2e_temp_dir.txt
37+
fi
38+
39+
# if TEMP_DIR does not exist, create it
40+
if [ ! -d "$TEMP_DIR" ]; then
41+
mkdir -p "$TEMP_DIR"
42+
fi
3643
3744
# Given that docker has an init function that checks for the existence of the
3845
# nvidia-container-toolkit, we need to create a symlink to the nvidia-container-runtime-hook
@@ -46,7 +53,7 @@ docker run --pid=host --rm -i --privileged \
4653
-v /var/run/docker.sock:/var/run/docker.sock \
4754
-v "$TEMP_DIR:$TEMP_DIR" \
4855
-v /etc/docker:/config-root \
49-
${IMAGE} \
56+
{{.Image}} \
5057
--root "$TEMP_DIR" \
5158
--runtime=docker \
5259
--config=/config-root/daemon.json \

‎tests/e2e/nvidia-container-cli_test.go‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
/*
2+
* Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
"text/template"
24+
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
)
28+
29+
const (
30+
installDockerTemplate = `docker exec -u root {{.ContainerName}} bash -c '
31+
export DEBIAN_FRONTEND=noninteractive
32+
33+
# Add Docker official GPG key:
34+
apt-get update
35+
apt-get install -y ca-certificates curl apt-utils gnupg2
36+
install -m 0755 -d /etc/apt/keyrings
37+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
38+
chmod a+r /etc/apt/keyrings/docker.asc
39+
40+
# Add the repository to Apt sources:
41+
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo \"${UBUNTU_CODENAME:-$VERSION_CODENAME}\") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
42+
apt-get update
43+
44+
apt-get install -y docker-ce docker-ce-cli containerd.io
45+
46+
# start dockerd in the background
47+
dockerd &
48+
49+
# wait for dockerd to be ready with timeout
50+
timeout=30
51+
elapsed=0
52+
while ! docker info > /dev/null 2>&1 && [ $elapsed -lt $timeout ]; do
53+
echo "Waiting for dockerd to be ready..."
54+
sleep 1
55+
elapsed=$((elapsed + 1))
56+
done
57+
if [ $elapsed -ge $timeout ]; then
58+
echo "Docker failed to start within $timeout seconds"
59+
exit 1
60+
fi
61+
'`
62+
installCTKTemplate = `docker exec -u root {{.ContainerName}} bash -c '
63+
# Create a temporary directory and rootfs path
64+
TMPDIR="$(mktemp -d)"
65+
66+
# Expose TMPDIR for the child namespace
67+
export TMPDIR
68+
69+
docker run --rm -v ${TMPDIR}:/host-tmpdir --entrypoint="sh" {{.ToolkitImage}}-packaging -c "cp -p -R /artifacts/* /host-tmpdir/"
70+
dpkg -i ${TMPDIR}/packages/ubuntu18.04/amd64/libnvidia-container1_*_amd64.deb ${TMPDIR}/packages/ubuntu18.04/amd64/nvidia-container-toolkit-base_*_amd64.deb ${TMPDIR}/packages/ubuntu18.04/amd64/libnvidia-container-tools_*_amd64.deb
71+
72+
nvidia-container-cli --version
73+
'`
74+
75+
libnvidiaContainerCliTestTemplate = `docker exec -u root {{.ContainerName}} bash -c '
76+
# Create a temporary directory and rootfs path
77+
TMPDIR="$(mktemp -d)"
78+
ROOTFS="${TMPDIR}/rootfs"
79+
mkdir -p "${ROOTFS}"
80+
81+
# Expose ROOTFS for the child namespace
82+
export ROOTFS TMPDIR
83+
84+
# Download Ubuntu base image with error handling
85+
curl -fsSL http://cdimage.ubuntu.com/ubuntu-base/releases/22.04/release/ubuntu-base-22.04-base-amd64.tar.gz | tar -C $ROOTFS -xz || {
86+
echo "Failed to download or extract Ubuntu base image"
87+
exit 1
88+
}
89+
90+
# Enter a new mount + PID namespace so we can pivot_root without touching the
91+
# container'\''s original filesystem.
92+
unshare --mount --pid --fork --propagation private -- sh -eux <<'\''IN_NS'\''
93+
: "${ROOTFS:?}"
94+
95+
# 1 Bind-mount the new root and make the mount private
96+
mount --bind "$ROOTFS" "$ROOTFS"
97+
mount --make-private "$ROOTFS"
98+
cd "$ROOTFS"
99+
100+
# 2 Minimal virtual filesystems
101+
mount -t proc proc proc
102+
mount -t sysfs sys sys
103+
mount -t tmpfs tmp tmp
104+
mount -t tmpfs run run
105+
106+
# 3 Configure NVIDIA devices
107+
nvidia-container-cli --load-kmods configure --ldconfig=@/sbin/ldconfig.real --no-cgroups --utility --device 0 $(pwd)
108+
109+
# 4 Switch root into the prepared filesystem
110+
pivot_root . mnt
111+
umount -l mnt
112+
nvidia-smi -L
113+
114+
IN_NS
115+
'`
116+
117+
dockerRunCmdTemplate = `docker run -d --name {{.ContainerName}} --privileged --runtime=nvidia \
118+
-e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all \
119+
-e NVIDIA_DRIVER_CAPABILITIES=all \
120+
ubuntu sleep infinity`
121+
)
122+
123+
var _ = Describe("nvidia-container-cli", Ordered, ContinueOnFailure, func() {
124+
var (
125+
runner Runner
126+
containerName = "nvidia-cli-e2e"
127+
hostOutput string
128+
)
129+
130+
BeforeAll(func(ctx context.Context) {
131+
runner = NewRunner(
132+
WithHost(sshHost),
133+
WithPort(sshPort),
134+
WithSshKey(sshKey),
135+
WithSshUser(sshUser),
136+
)
137+
138+
if installCTK {
139+
installer, err := NewToolkitInstaller(
140+
WithRunner(runner),
141+
WithImage(imageName+":"+imageTag),
142+
WithTemplate(dockerInstallTemplate),
143+
)
144+
Expect(err).ToNot(HaveOccurred())
145+
146+
err = installer.Install()
147+
Expect(err).ToNot(HaveOccurred())
148+
}
149+
150+
// Capture the host GPU list.
151+
var err error
152+
hostOutput, _, err = runner.Run("nvidia-smi -L")
153+
Expect(err).ToNot(HaveOccurred())
154+
155+
// Normalize the output once
156+
hostOutput = strings.TrimSpace(strings.ReplaceAll(hostOutput, "\r", ""))
157+
158+
// If a container with the same name exists from a previous test run, remove it first.
159+
// Ignore errors as container might not exist
160+
runner.Run(fmt.Sprintf("docker rm -f %s 2>/dev/null || true", containerName)) //nolint:errcheck
161+
})
162+
163+
AfterAll(func(ctx context.Context) {
164+
// Cleanup: remove the container and the temporary script on the host.
165+
// Use || true to ensure cleanup doesn't fail the test
166+
runner.Run(fmt.Sprintf("docker rm -f %s 2>/dev/null || true", containerName)) //nolint:errcheck
167+
})
168+
169+
It("should report the same GPUs inside the container as on the host", func(ctx context.Context) {
170+
// Build the docker run command (detached mode) from the template so it
171+
// stays readable while still resulting in a single-line invocation.
172+
tmpl, err := template.New("dockerRun").Parse(dockerRunCmdTemplate)
173+
Expect(err).ToNot(HaveOccurred())
174+
175+
var dockerRunCmd strings.Builder
176+
err = tmpl.Execute(&dockerRunCmd, struct {
177+
ContainerName string
178+
}{
179+
ContainerName: containerName,
180+
})
181+
Expect(err).ToNot(HaveOccurred())
182+
183+
// Launch the container in detached mode.
184+
_, _, err = runner.Run(dockerRunCmd.String())
185+
Expect(err).ToNot(HaveOccurred())
186+
187+
// check the container is running
188+
_, _, err = runner.Run("docker ps -a")
189+
Expect(err).ToNot(HaveOccurred())
190+
191+
// Install docker and nvidia-container-toolkit in the container.
192+
// Run as root and use bash for better compatibility
193+
tmpl, err = template.New("dockerInstall").Parse(installDockerTemplate)
194+
Expect(err).ToNot(HaveOccurred())
195+
196+
var dockerInstall strings.Builder
197+
err = tmpl.Execute(&dockerInstall, struct {
198+
ContainerName string
199+
}{
200+
ContainerName: containerName,
201+
})
202+
Expect(err).ToNot(HaveOccurred())
203+
204+
_, _, err = runner.Run(dockerInstall.String())
205+
Expect(err).ToNot(HaveOccurred())
206+
207+
// Build the docker run command (detached mode) from the template so it
208+
// stays readable while still resulting in a single-line invocation.
209+
tmpl, err = template.New("toolkitInstall").Parse(installCTKTemplate)
210+
Expect(err).ToNot(HaveOccurred())
211+
212+
var toolkitInstall strings.Builder
213+
err = tmpl.Execute(&toolkitInstall, struct {
214+
ContainerName string
215+
ToolkitImage string
216+
}{
217+
ContainerName: containerName,
218+
ToolkitImage: imageName + ":" + imageTag,
219+
})
220+
Expect(err).ToNot(HaveOccurred())
221+
222+
_, _, err = runner.Run(toolkitInstall.String())
223+
Expect(err).ToNot(HaveOccurred())
224+
225+
// Run the test script in the container.
226+
// Capture but don't fail on errors - we'll check the results via container logs.
227+
tmpl, err = template.New("libnvidiaContainerCliTest").Parse(libnvidiaContainerCliTestTemplate)
228+
Expect(err).ToNot(HaveOccurred())
229+
230+
var libnvidiaContainerCliTest strings.Builder
231+
err = tmpl.Execute(&libnvidiaContainerCliTest, struct {
232+
ContainerName string
233+
}{
234+
ContainerName: containerName,
235+
})
236+
Expect(err).ToNot(HaveOccurred())
237+
238+
output, _, err := runner.Run(libnvidiaContainerCliTest.String())
239+
Expect(err).ToNot(HaveOccurred())
240+
Expect(output).To(ContainSubstring(hostOutput))
241+
})
242+
})

0 commit comments

Comments
 (0)