Skip to content

Commit 73b8412

Browse files
tariq1890tariq1890
committed
integrate gpu-driver-util into the driver images
Signed-off-by: Tariq Ibrahim <[email protected]>
1 parent 0f4ecad commit 73b8412

File tree

13 files changed

+160
-32
lines changed

13 files changed

+160
-32
lines changed

‎rhel8/Dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ ENV PATH /usr/local/go/bin:$PATH
1717
WORKDIR /work
1818

1919
RUN git clone https://github.com/NVIDIA/gpu-driver-container driver && \
20-
cd driver/vgpu/src && \
21-
go build -o vgpu-util && \
22-
mv vgpu-util /work
20+
go build -C driver/vgpu/src -o vgpu-util && \
21+
mv driver/vgpu/src/vgpu-util /work && \
22+
go build -C driver/gpu-driver-util -o gpu-driver-util && \
23+
mv driver/gpu-driver-util/gpu-driver-util /work
2324

2425
FROM nvcr.io/nvidia/cuda:12.6.3-base-ubi8
2526

@@ -84,6 +85,7 @@ COPY ocp_dtk_entrypoint /usr/local/bin
8485
COPY common.sh /usr/local/bin
8586

8687
COPY --from=build /work/vgpu-util /usr/local/bin
88+
COPY --from=build /work/gpu-driver-util /usr/local/bin
8789

8890
WORKDIR /drivers
8991

‎rhel8/nvidia-driver‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@ USE_HOST_MOFED="${USE_HOST_MOFED:-false}"
1717
DNF_RELEASEVER=${DNF_RELEASEVER:-""}
1818
RHEL_VERSION=${RHEL_VERSION:-""}
1919
RHEL_MAJOR_VERSION=8
20-
21-
OPEN_KERNEL_MODULES_ENABLED=${OPEN_KERNEL_MODULES_ENABLED:-false}
22-
[[ "${OPEN_KERNEL_MODULES_ENABLED}" == "true" ]] && KERNEL_TYPE=kernel-open || KERNEL_TYPE=kernel
20+
KERNEL_MODULE_TYPE=${KERNEL_MODULE_TYPE:-auto}
2321

2422
DRIVER_ARCH=${TARGETARCH/amd64/x86_64} && DRIVER_ARCH=${DRIVER_ARCH/arm64/aarch64}
2523
echo "DRIVER_ARCH is $DRIVER_ARCH"
@@ -577,6 +575,24 @@ _start_vgpu_topology_daemon() {
577575
nvidia-topologyd
578576
}
579577

578+
_resolve_kernel_type() {
579+
if [ "${KERNEL_MODULE_TYPE}" == "proprietary" ]; then
580+
KERNEL_TYPE=kernel
581+
elif [ "${KERNEL_MODULE_TYPE}" == "open" ]; then
582+
KERNEL_TYPE=kernel-open
583+
elif [ "${KERNEL_MODULE_TYPE}" == "auto" ]; then
584+
KERNEL_TYPE=$(gpu-driver-util get-kernel-module-type -b "${DRIVER_BRANCH}")
585+
if [ $? -ne 0 ]; then
586+
echo "cannot autodetect the kernel module type, printing error logs from /var/log/gpu-driver-util.log..."
587+
tail -n 3 /var/log/gpu-driver-util.log
588+
return 1
589+
fi
590+
else
591+
echo "invalid value for the KERNEL_MODULE_TYPE variable: ${KERNEL_MODULE_TYPE}"
592+
return 1
593+
fi
594+
}
595+
580596
_prepare() {
581597
if [ "${DRIVER_TYPE}" = "vgpu" ]; then
582598
_find_vgpu_driver_version || exit 1
@@ -797,5 +813,6 @@ if [ $# -ne 0 ]; then
797813
fi
798814

799815
_resolve_rhel_version || exit 1
816+
_resolve_kernel_type || exit 1
800817

801818
$command

‎rhel9/Dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ ENV PATH /usr/local/go/bin:$PATH
1717
WORKDIR /work
1818

1919
RUN git clone https://github.com/NVIDIA/gpu-driver-container driver && \
20-
cd driver/vgpu/src && \
21-
go build -o vgpu-util && \
22-
mv vgpu-util /work
20+
go build -C driver/vgpu/src -o vgpu-util && \
21+
mv driver/vgpu/src/vgpu-util /work && \
22+
go build -C driver/gpu-driver-util -o gpu-driver-util && \
23+
mv driver/gpu-driver-util/gpu-driver-util /work
2324

2425
FROM nvcr.io/nvidia/cuda:12.6.3-base-ubi9
2526

@@ -78,6 +79,7 @@ COPY ocp_dtk_entrypoint /usr/local/bin
7879
COPY common.sh /usr/local/bin
7980

8081
COPY --from=build /work/vgpu-util /usr/local/bin
82+
COPY --from=build /work/gpu-driver-util /usr/local/bin
8183

8284
WORKDIR /drivers
8385

‎rhel9/nvidia-driver‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@ USE_HOST_MOFED="${USE_HOST_MOFED:-false}"
1717
DNF_RELEASEVER=${DNF_RELEASEVER:-""}
1818
RHEL_VERSION=${RHEL_VERSION:-""}
1919
RHEL_MAJOR_VERSION=9
20-
21-
OPEN_KERNEL_MODULES_ENABLED=${OPEN_KERNEL_MODULES_ENABLED:-false}
22-
[[ "${OPEN_KERNEL_MODULES_ENABLED}" == "true" ]] && KERNEL_TYPE=kernel-open || KERNEL_TYPE=kernel
20+
KERNEL_MODULE_TYPE=${KERNEL_MODULE_TYPE:-auto}
2321

2422
DRIVER_ARCH=${TARGETARCH/amd64/x86_64} && DRIVER_ARCH=${DRIVER_ARCH/arm64/aarch64}
2523
echo "DRIVER_ARCH is $DRIVER_ARCH"
@@ -571,6 +569,24 @@ _find_vgpu_driver_version() {
571569
return 0
572570
}
573571

572+
_resolve_kernel_type() {
573+
if [ "${KERNEL_MODULE_TYPE}" == "proprietary" ]; then
574+
KERNEL_TYPE=kernel
575+
elif [ "${KERNEL_MODULE_TYPE}" == "open" ]; then
576+
KERNEL_TYPE=kernel-open
577+
elif [ "${KERNEL_MODULE_TYPE}" == "auto" ]; then
578+
KERNEL_TYPE=$(gpu-driver-util get-kernel-module-type -b "${DRIVER_BRANCH}")
579+
if [ $? -ne 0 ]; then
580+
echo "cannot autodetect the kernel module type, printing error logs from /var/log/gpu-driver-util.log..."
581+
tail -n 3 /var/log/gpu-driver-util.log
582+
return 1
583+
fi
584+
else
585+
echo "invalid value for the KERNEL_MODULE_TYPE variable: ${KERNEL_MODULE_TYPE}"
586+
return 1
587+
fi
588+
}
589+
574590
_start_vgpu_topology_daemon() {
575591
type nvidia-topologyd > /dev/null 2>&1 || return 0
576592
echo "Starting nvidia-topologyd.."
@@ -797,5 +813,6 @@ if [ $# -ne 0 ]; then
797813
fi
798814

799815
_resolve_rhel_version || exit 1
816+
_resolve_kernel_type || exit 1
800817

801818
$command

‎ubuntu20.04/Dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ ENV PATH /usr/local/go/bin:$PATH
2828
WORKDIR /work
2929

3030
RUN git clone https://github.com/NVIDIA/gpu-driver-container driver && \
31-
cd driver/vgpu/src && \
32-
go build -o vgpu-util && \
33-
mv vgpu-util /work
31+
go build -C driver/vgpu/src -o vgpu-util && \
32+
mv driver/vgpu/src/vgpu-util /work && \
33+
go build -C driver/gpu-driver-util -o gpu-driver-util && \
34+
mv driver/gpu-driver-util/gpu-driver-util /work
3435

3536
FROM nvcr.io/nvidia/cuda:12.6.3-base-ubuntu20.04
3637

@@ -72,6 +73,7 @@ RUN /tmp/install.sh reposetup && /tmp/install.sh depinstall && \
7273
COPY nvidia-driver /usr/local/bin
7374

7475
COPY --from=build /work/vgpu-util /usr/local/bin
76+
COPY --from=build /work/gpu-driver-util /usr/local/bin
7577

7678
ADD drivers drivers/
7779

‎ubuntu20.04/nvidia-driver‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@ NVIDIA_MODESET_MODULE_PARAMS=()
1616
NVIDIA_PEERMEM_MODULE_PARAMS=()
1717
TARGETARCH=${TARGETARCH:?"Missing TARGETARCH env"}
1818

19-
OPEN_KERNEL_MODULES_ENABLED=${OPEN_KERNEL_MODULES_ENABLED:-false}
20-
[[ "${OPEN_KERNEL_MODULES_ENABLED}" == "true" ]] && KERNEL_TYPE=kernel-open || KERNEL_TYPE=kernel
19+
KERNEL_MODULE_TYPE=${KERNEL_MODULE_TYPE:-auto}
2120

2221
export DEBIAN_FRONTEND=noninteractive
2322

@@ -477,6 +476,24 @@ _shutdown() {
477476
return 1
478477
}
479478

479+
_resolve_kernel_type() {
480+
if [ "${KERNEL_MODULE_TYPE}" == "proprietary" ]; then
481+
KERNEL_TYPE=kernel
482+
elif [ "${KERNEL_MODULE_TYPE}" == "open" ]; then
483+
KERNEL_TYPE=kernel-open
484+
elif [ "${KERNEL_MODULE_TYPE}" == "auto" ]; then
485+
KERNEL_TYPE=$(gpu-driver-util get-kernel-module-type -b "${DRIVER_BRANCH}")
486+
if [ $? -ne 0 ]; then
487+
echo "cannot autodetect the kernel module type, printing error logs from /var/log/gpu-driver-util.log..."
488+
tail -n 3 /var/log/gpu-driver-util.log
489+
return 1
490+
fi
491+
else
492+
echo "invalid value for the KERNEL_MODULE_TYPE variable: ${KERNEL_MODULE_TYPE}"
493+
return 1
494+
fi
495+
}
496+
480497
_find_vgpu_driver_version() {
481498
local count=""
482499
local version=""
@@ -520,6 +537,8 @@ init() {
520537
_find_vgpu_driver_version || exit 1
521538
fi
522539

540+
_resolve_kernel_type || exit 1
541+
523542
# Install the userspace components and copy the kernel module sources.
524543
sh NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION.run -x && \
525544
cd NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION && \
@@ -592,6 +611,8 @@ update() {
592611
fi
593612
exec 3>&-
594613

614+
_resolve_kernel_type || exit 1
615+
595616
# vgpu driver version is choosen dynamically during runtime, so pre-compile modules for
596617
# only non-vgpu driver types
597618
if [ "${DRIVER_TYPE}" != "vgpu" ]; then

‎ubuntu22.04/Dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ ENV PATH /usr/local/go/bin:$PATH
2828
WORKDIR /work
2929

3030
RUN git clone https://github.com/NVIDIA/gpu-driver-container driver && \
31-
cd driver/vgpu/src && \
32-
go build -o vgpu-util && \
33-
mv vgpu-util /work
31+
go build -C driver/vgpu/src -o vgpu-util && \
32+
mv driver/vgpu/src/vgpu-util /work && \
33+
go build -C driver/gpu-driver-util -o gpu-driver-util && \
34+
mv driver/gpu-driver-util/gpu-driver-util /work
3435

3536
FROM nvcr.io/nvidia/cuda:12.6.3-base-ubuntu22.04
3637

@@ -72,6 +73,7 @@ RUN /tmp/install.sh reposetup && /tmp/install.sh depinstall && \
7273
COPY nvidia-driver /usr/local/bin
7374

7475
COPY --from=build /work/vgpu-util /usr/local/bin
76+
COPY --from=build /work/gpu-driver-util /usr/local/bin
7577

7678
ADD drivers drivers/
7779

‎ubuntu22.04/nvidia-driver‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,7 @@ NVIDIA_UVM_MODULE_PARAMS=()
1515
NVIDIA_MODESET_MODULE_PARAMS=()
1616
NVIDIA_PEERMEM_MODULE_PARAMS=()
1717
TARGETARCH=${TARGETARCH:?"Missing TARGETARCH env"}
18-
19-
OPEN_KERNEL_MODULES_ENABLED=${OPEN_KERNEL_MODULES_ENABLED:-false}
20-
[[ "${OPEN_KERNEL_MODULES_ENABLED}" == "true" ]] && KERNEL_TYPE=kernel-open || KERNEL_TYPE=kernel
18+
KERNEL_MODULE_TYPE=${KERNEL_MODULE_TYPE:-auto}
2119

2220
export DEBIAN_FRONTEND=noninteractive
2321

@@ -481,6 +479,24 @@ _shutdown() {
481479
return 1
482480
}
483481

482+
_resolve_kernel_type() {
483+
if [ "${KERNEL_MODULE_TYPE}" == "proprietary" ]; then
484+
KERNEL_TYPE=kernel
485+
elif [ "${KERNEL_MODULE_TYPE}" == "open" ]; then
486+
KERNEL_TYPE=kernel-open
487+
elif [ "${KERNEL_MODULE_TYPE}" == "auto" ]; then
488+
KERNEL_TYPE=$(gpu-driver-util get-kernel-module-type -b "${DRIVER_BRANCH}")
489+
if [ $? -ne 0 ]; then
490+
echo "cannot autodetect the kernel module type, printing error logs from /var/log/gpu-driver-util.log..."
491+
tail -n 3 /var/log/gpu-driver-util.log
492+
return 1
493+
fi
494+
else
495+
echo "invalid value for the KERNEL_MODULE_TYPE variable: ${KERNEL_MODULE_TYPE}"
496+
return 1
497+
fi
498+
}
499+
484500
_find_vgpu_driver_version() {
485501
local count=""
486502
local version=""
@@ -524,6 +540,8 @@ init() {
524540
_find_vgpu_driver_version || exit 1
525541
fi
526542

543+
_resolve_kernel_type || exit 1
544+
527545
# Install the userspace components and copy the kernel module sources.
528546
sh NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION.run -x && \
529547
cd NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION && \
@@ -596,6 +614,8 @@ update() {
596614
fi
597615
exec 3>&-
598616

617+
_resolve_kernel_type || exit 1
618+
599619
# vgpu driver version is choosen dynamically during runtime, so pre-compile modules for
600620
# only non-vgpu driver types
601621
if [ "${DRIVER_TYPE}" != "vgpu" ]; then

‎ubuntu22.04/precompiled/Dockerfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ RUN echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main universe
4141
RUN curl -fsSL -o /usr/local/bin/donkey https://github.com/3XX0/donkey/releases/download/v1.1.0/donkey && \
4242
chmod +x /usr/local/bin/donkey
4343

44+
RUN git clone https://github.com/NVIDIA/gpu-driver-container driver && \
45+
go build -C driver/gpu-driver-util -o gpu-driver-util && \
46+
mv driver/gpu-driver-util/gpu-driver-util /usr/local/bin && \
47+
rm -rf driver
48+
4449
# Install / upgrade packages here that are required to resolve CVEs
4550
ARG CVE_UPDATES
4651
RUN if [ -n "${CVE_UPDATES}" ]; then \

‎ubuntu22.04/precompiled/nvidia-driver‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
set -eu
55

66
KERNEL_VERSION=$(uname -r)
7-
OPEN_KERNEL_MODULES_ENABLED="${OPEN_KERNEL_MODULES_ENABLED:-false}"
7+
KERNEL_MODULE_TYPE=${KERNEL_MODULE_TYPE:-auto}
88
RUN_DIR=/run/nvidia
99
PID_FILE=${RUN_DIR}/${0##*/}.pid
1010
DRIVER_BRANCH=${DRIVER_BRANCH:?"Missing driver version"}
@@ -96,6 +96,24 @@ _get_module_params() {
9696
fi
9797
}
9898

99+
_resolve_kernel_type() {
100+
if [ "${KERNEL_MODULE_TYPE}" == "proprietary" ]; then
101+
KERNEL_TYPE=kernel
102+
elif [ "${KERNEL_MODULE_TYPE}" == "open" ]; then
103+
KERNEL_TYPE=kernel-open
104+
elif [ "${KERNEL_MODULE_TYPE}" == "auto" ]; then
105+
KERNEL_TYPE=$(gpu-driver-util get-kernel-module-type -b "${DRIVER_BRANCH}")
106+
if [ $? -ne 0 ]; then
107+
echo "cannot autodetect the kernel module type, printing error logs from /var/log/gpu-driver-util.log..."
108+
tail -n 3 /var/log/gpu-driver-util.log
109+
return 1
110+
fi
111+
else
112+
echo "invalid value for the KERNEL_MODULE_TYPE variable: ${KERNEL_MODULE_TYPE}"
113+
return 1
114+
fi
115+
}
116+
99117
# Load the kernel modules and start persistenced.
100118
_load_driver() {
101119
echo "Parsing kernel module parameters..."
@@ -245,7 +263,7 @@ _install_driver() {
245263
xserver-xorg-video-nvidia-${DRIVER_BRANCH}-server
246264

247265
# Now install the precompiled kernel module packages signed by Canonical
248-
if [ "$OPEN_KERNEL_MODULES_ENABLED" = true ]; then
266+
if [ "${KERNEL_TYPE}" == "kernel-open" ]; then
249267
echo "Installing Open NVIDIA driver kernel modules..."
250268
apt-get install --no-install-recommends -y \
251269
linux-signatures-nvidia-${KERNEL_VERSION} \
@@ -293,6 +311,7 @@ init() {
293311
_unload_driver || exit 1
294312
_unmount_rootfs
295313

314+
_resolve_kernel_type || exit 1
296315
_install_driver
297316
_load_driver || exit 1
298317
_mount_rootfs

0 commit comments

Comments
 (0)