feat(nvidia): add nvbandwidth tests (#638)

Author: ndbaker1

hack/update-nvidia-dependencies.sh

@@ -9,16 +9,15 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
+echo "Updating aws-ofi-nccl"
 AWS_OFI_NCCL_TAG=$(curl -s https://api.github.com/repos/aws/aws-ofi-nccl/releases/latest | jq -r .tag_name | sed 's/^v//')
-LIB_NCCL_TAG=$(curl -s https://api.github.com/repos/aws/aws-ofi-nccl/releases/latest | jq -r .body | grep -oP '\[NCCL \K(\S*)(?=\])' | head -n 1 | sed 's/^v//')
-
 find . -type f -name Dockerfile -exec sed -i "s/AWS_OFI_NCCL_VERSION=.*/AWS_OFI_NCCL_VERSION=$AWS_OFI_NCCL_TAG/g" {} +
+
+echo "Updating nccl"
+LIB_NCCL_TAG=$(curl -s https://api.github.com/repos/aws/aws-ofi-nccl/releases/latest | jq -r .body | grep -oP '\[NCCL \K(\S*)(?=\])' | head -n 1 | sed 's/^v//')
 find . -type f -name Dockerfile -exec sed -i "s/LIBNCCL_VERSION=.*/LIBNCCL_VERSION=$LIB_NCCL_TAG/g" {} +
 
-CUDA=$(curl -s https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/Packages  | grep 'Package: libnccl2' -A 1 | grep $LIB_NCCL_TAG | grep -oP 'Version: \K(\S*)(?=)' | sort -r | head -n 1 | sed "s/$LIB_NCCL_TAG+cuda//")
-CUDA_VERSION_PARTS=($(echo $CUDA | tr -s '.' ' '))
-CUDA_MAJOR_VERSION=${CUDA_VERSION_PARTS[0]}
-CUDA_MINOR_VERSION=${CUDA_VERSION_PARTS[1]}
+echo "Updating nvbandwidth"
+NVBANDWIDTH_TAG=$(curl -s https://api.github.com/repos/NVIDIA/nvbandwidth/releases/latest | jq -r .tag_name)
+find . -type f -name Dockerfile -exec sed -i "s/NVBANDWIDTH_VERSION=.*/NVBANDWIDTH_VERSION=$NVBANDWIDTH_TAG/g" {} +
 
-find . -type f -name Dockerfile -exec sed -i "s/CUDA_MINOR_VERSION=.*/CUDA_MINOR_VERSION=$CUDA_MINOR_VERSION/g" {} +
-find . -type f -name Dockerfile -exec sed -i "s/CUDA_MAJOR_VERSION=.*/CUDA_MAJOR_VERSION=$CUDA_MAJOR_VERSION/g" {} +

test/cases/nvidia/manifests/job-unit-test-single-node.yaml

@@ -28,8 +28,6 @@ spec:
         imagePullPolicy: Always
         resources:
           limits:
-            cpu: "4"
-            memory: 4Gi
             nvidia.com/gpu: {{.GpuPerNode}}
           requests:
             cpu: "1"

test/images/nvidia-inference/Dockerfile

@@ -2,7 +2,7 @@
 # Base image, arguments, and environment
 ###############################################################################
 ARG CUDA_MAJOR_VERSION=12
-ARG CUDA_MINOR_VERSION=9
+ARG CUDA_MINOR_VERSION=8
 
 FROM nvidia/cuda:$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION.0-devel-ubuntu22.04
 

test/images/nvidia-training/Dockerfile

@@ -1,5 +1,5 @@
 ARG CUDA_MAJOR_VERSION=12
-ARG CUDA_MINOR_VERSION=9
+ARG CUDA_MINOR_VERSION=8
 
 # Use the NVIDIA CUDA runtime as a parent image
 FROM nvidia/cuda:$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION.0-devel-ubuntu22.04

test/images/nvidia/Dockerfile

@@ -1,5 +1,5 @@
 ARG CUDA_MAJOR_VERSION=12
-ARG CUDA_MINOR_VERSION=9
+ARG CUDA_MINOR_VERSION=8
 
 # Start with the NVIDIA CUDA base image
 FROM nvidia/cuda:$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION.0-devel-ubuntu22.04
@@ -41,6 +41,9 @@ RUN apt update -y \
 
 RUN ldconfig
 
+ENV LD_LIBRARY_PATH /opt/amazon/openmpi/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/cuda/lib:/usr/local/lib/:/usr/lib64:/usr/lib/x86_64-linux-gnu/:/usr/lib/aarch64-linux-gnu/:$LD_LIBRARY_PATH
+ENV PATH /usr/local/cuda/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/sbin:/usr/bin:/usr/local/bin:$PATH
+
 RUN mkdir -p /var/run/sshd \
  && sed -i 's/[ #]\(.*StrictHostKeyChecking \).*/ \1no/g' /etc/ssh/ssh_config \
  && echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config \
@@ -53,19 +56,21 @@ RUN git clone https://github.com/NVIDIA/cuda-samples.git /tmp/cuda-samples \
  && cd /tmp/cuda-samples/Samples/1_Utilities/deviceQuery && cmake . && make -j$(nproc) && cp deviceQuery /usr/bin \
  && cd && rm -rf /tmp/cuda-samples
 
-# Build nvbandwidth
-# TODO: see https://github.com/NVIDIA/nvbandwidth
-
-ENV LD_LIBRARY_PATH /opt/amazon/openmpi/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/cuda/lib:/usr/local/lib/:/usr/lib64:/usr/lib/x86_64-linux-gnu/:/usr/lib/aarch64-linux-gnu/:$LD_LIBRARY_PATH
-ENV PATH /usr/local/cuda/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/sbin:/usr/bin:/usr/local/bin:$PATH
-
 # Install EFA
 ARG EFA_INSTALLER_VERSION=latest
 RUN curl -sL https://efa-installer.amazonaws.com/aws-efa-installer-$EFA_INSTALLER_VERSION.tar.gz | tar xvz -C /tmp \
  && cd /tmp/aws-efa-installer \
  && ./efa_installer.sh --yes --enable-gdr --skip-kmod --skip-limit-conf --no-verify --mpi openmpi4 \
  && cd && rm -rf /tmp/aws-efa-installer
 
+# Build nvbandwidth
+ARG NVBANDWIDTH_VERSION=v0.8
+RUN apt install -y libboost-program-options-dev
+RUN git clone https://github.com/NVIDIA/nvbandwidth.git --branch $NVBANDWIDTH_VERSION /tmp/nvbandwidth \
+ && cd /tmp/nvbandwidth \
+ && cmake -DMULTINODE=1 . && make && cp nvbandwidth /usr/bin \
+ && cd && rm -rf /tmp/cuda-samples
+
 # Install NCCL
 ARG LIBNCCL_VERSION=2.27.5-1
 RUN git clone https://github.com/NVIDIA/nccl.git --branch v$LIBNCCL_VERSION /tmp/nccl \
@@ -90,6 +95,7 @@ RUN curl -sL https://github.com/aws/aws-ofi-nccl/releases/download/v$AWS_OFI_NCC
  && cd && rm -rf /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION
 
 # Install NCCL Tests
+# TODO: automate pin with version bump
 RUN git clone https://github.com/NVIDIA/nccl-tests /tmp/nccl-tests \
  && cd /tmp/nccl-tests \
  && make \

test/images/nvidia/gpu_unit_tests/README.md

@@ -41,6 +41,7 @@ Create PR with the new `tests/test_sysinfo.sh.data/xxx`
   Usually this is side effect of system misconfiguration (driver or fabric manager is not loaded)
   - test_01_device_query
   - test_02_vector_add
+  - test_03_nvbandwidth
   - test_04_dcgm_diagnostics
 
 

test/images/nvidia/gpu_unit_tests/tests/test_basic.sh

@@ -24,6 +24,11 @@ test_02_vector_add()
     assert_status_code 0 "$DEMO_SUITE_DIR/vectorAdd"
 }
 
+test_03_nvbandwidth()
+{
+    assert_status_code 0 "$DEMO_SUITE_DIR/nvbandwidth"
+}
+
 test_04_dcgm_diagnostics()
 {
     # https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/dcgm-diagnostics.html#run-levels-and-tests