Revert some NCCL dependency updates (#710)

* Revert "chore: update nvidia test dependencies (#708)"

This reverts commit 56817ed.

* Revert "chore: update nvidia test dependencies (#697)"

This reverts commit 4f8dcc7.

Author: tzneal

test/images/nvidia-training/Dockerfile

@@ -111,15 +111,15 @@ RUN curl -sL https://efa-installer.amazonaws.com/aws-efa-installer-$EFA_INSTALLE
  && cd && rm -rf /tmp/aws-efa-installer
 
 # Install NCCL
-ARG LIBNCCL_VERSION=2.28.3-1
+ARG LIBNCCL_VERSION=2.27.7-1
 RUN git clone https://github.com/NVIDIA/nccl.git --branch v$LIBNCCL_VERSION /tmp/nccl \
  && cd /tmp/nccl \
  && make -j $(nproc) \
  && make install \
  && cd && rm -rf /tmp/nccl
 
 # Install AWS-OFI-NCCL plugin
-ARG AWS_OFI_NCCL_VERSION=1.17.1
+ARG AWS_OFI_NCCL_VERSION=1.16.3
 RUN curl -sL https://github.com/aws/aws-ofi-nccl/releases/download/v$AWS_OFI_NCCL_VERSION/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION.tar.gz | tar xvz -C /tmp \
  && cd /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION \
  && ./configure \

test/images/nvidia/Dockerfile

@@ -72,15 +72,15 @@ RUN git clone https://github.com/NVIDIA/nvbandwidth.git --branch $NVBANDWIDTH_VE
  && cd && rm -rf /tmp/cuda-samples
 
 # Install NCCL
-ARG LIBNCCL_VERSION=2.28.3-1
+ARG LIBNCCL_VERSION=2.27.7-1
 RUN git clone https://github.com/NVIDIA/nccl.git --branch v$LIBNCCL_VERSION /tmp/nccl \
  && cd /tmp/nccl \
  && make -j $(nproc) \
  && make install \
  && cd && rm -rf /tmp/nccl
 
 # Install AWS-OFI-NCCL plugin
-ARG AWS_OFI_NCCL_VERSION=1.17.1
+ARG AWS_OFI_NCCL_VERSION=1.16.3
 RUN curl -sL https://github.com/aws/aws-ofi-nccl/releases/download/v$AWS_OFI_NCCL_VERSION/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION.tar.gz | tar xvz -C /tmp \
  && cd /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION \
  && ./configure \