From 5dc9bf98c3223a5858297495863490b226d984e3 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Tue, 9 Dec 2025 14:42:21 -0800 Subject: [PATCH 1/7] Use the GitHub CLI to query sccache version (#631) --- features/src/utils/devcontainer-feature.json | 2 +- features/src/utils/opt/devcontainer/bin/sccache/install.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index a1a98e2f..3fbc857d 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "26.2.3", + "version": "26.2.4", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/opt/devcontainer/bin/sccache/install.sh b/features/src/utils/opt/devcontainer/bin/sccache/install.sh index d91627aa..4c154f96 100755 --- a/features/src/utils/opt/devcontainer/bin/sccache/install.sh +++ b/features/src/utils/opt/devcontainer/bin/sccache/install.sh @@ -40,11 +40,11 @@ _install_sccache() { fi sccache_bin_dir="$(dirname "$(which sccache 2>/dev/null || echo "/usr/bin/sccache")")"; - old_sccache_version="$(sccache --version | cut -d' ' -f2 || echo "sccache not installed")"; + old_sccache_version="$(sccache --version 2>/dev/null | cut -d' ' -f2 || echo "sccache not installed")"; attempts=0 while test "${new_sccache_version:-latest}" = latest; do - new_sccache_version="$(wget --no-hsts -q -O- "https://api.github.com/repos/${github_repo}/releases/latest" | jq -r '.tag_name | sub("^v"; "")')"; + new_sccache_version="$(gh api repos/${github_repo}/releases/latest --jq '.tag_name | sub("^v"; "")')"; if test -n "${new_sccache_version:+x}"; then break; elif test "$((attempts++))" -lt 10; then @@ -61,7 +61,7 @@ _install_sccache() { if test -n "${old_sccache_version##"${new_sccache_version}"}"; then # Install sccache to "$sccache_bin_dir" echo "Downloading sccache v${new_sccache_version}..." >&2 - while ! wget --no-hsts -q -O- "https://github.com/${github_repo}/releases/download/v${new_sccache_version}/sccache-v${new_sccache_version}-$(uname -m)-unknown-linux-musl.tar.gz" \ + while ! gh release download "v${new_sccache_version}" -R "${github_repo}" -p "sccache-v${new_sccache_version}-$(uname -m)-unknown-linux-musl.tar.gz" -O- \ | sudo tar -C "$sccache_bin_dir" -zf - --overwrite --wildcards --strip-components=1 -x '*/sccache'; do if test "$((attempts++))" -lt 10; then echo "(!) Failed to download sccache v${new_sccache_version}. Retrying (${attempts}/10)..." >&2; From 094d6a7a6616dd38b0d931dcef7451a60c794477 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Tue, 9 Dec 2025 22:09:38 -0800 Subject: [PATCH 2/7] Build `cl14.43` for stdexec (#633) `cl14.44` has an internal compiler error building stdexec --- matrix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matrix.yml b/matrix.yml index c782f300..b08c2d00 100644 --- a/matrix.yml +++ b/matrix.yml @@ -170,6 +170,8 @@ include: # CTK 12.0 doesn't know that it supports 14.40+, use 14.39: - { features: [{ <<: *cuda_prev_min, <<: *cccl_cuda_opts }, { name: "cl", version: "14.39" }] } # MSVC 2022 + # Build CUDA 12.9 + 14.43 for stdexec: + - { features: [{ <<: *cuda_prev_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.43" }] } # MSVC 2022 # Still, we build a 12.0 + 14.4X image for python testing infra: - { features: [{ <<: *cuda_prev_min, <<: *cccl_cuda_opts }, { name: "cl", version: "14.44" }] } # MSVC 2022 - { features: [{ <<: *cuda_prev_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.44" }] } # MSVC 2022 From 2e4bcafa3c3b394c3148e71d86dc1417c23a8bcb Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 10 Dec 2025 13:18:13 -0600 Subject: [PATCH 3/7] Add test commands to rapids-build-utils (#630) ## Summary With help from `claude`, I am implementing my favorite feature from the old `rapids-compose`: `test-*` commands. - Add `test--cpp` commands that run C++ test scripts (e.g., `test-cudf-cpp` runs `ci/run_cudf_ctests.sh`) - Add `test--python` commands that run Python test scripts (e.g., `test-rmm-python` runs `ci/run_pytests.sh`) - Add `test-` commands that run both C++ and Python tests for a repo - Add `test-all`, `test-all-cpp`, and `test-all-python` commands Test script paths are specified in `manifest.yaml` since they vary across repos. All arguments are forwarded to the underlying scripts. ## Test plan - [x] Rebuild devcontainer and verify `test-*` commands are generated - [x] Run `test-rmm-cpp -h` to verify help text - [x] Run `test-cudf-cpp` (via script content inspection) to verify C++ tests invoke correct CI scripts - [x] Run `test-rmm-python` to verify Python tests invoke correct CI scripts - [x] Run `test-all -h` to verify aggregated command works ## Testing notes Validated using `rapidsai/devcontainers:26.02-cpp-rapids-build-utils-ubuntu24.04` with modified `rapids-build-utils` mounted: ``` # Commands generated correctly $ ls /usr/bin/test-rmm* /usr/bin/test-rmm /usr/bin/test-rmm-cpp /usr/bin/test-rmm-python # Help text works $ test-rmm-cpp -h Usage: test-rmm-cpp [OPTION]... Run rmm C++ tests. Boolean options: -h,--help Print this text. # test-rmm-cpp correctly invokes ci/run_ctests.sh $ test-rmm-cpp ./ci/run_ctests.sh: line 8: cd: /usr/bin/gtests/librmm/: No such file or directory # test-rmm-python correctly invokes ci/run_pytests.sh $ test-rmm-python ./ci/run_pytests.sh: line 10: pytest: command not found # test-all commands work $ test-all-cpp -h Usage: test-all-cpp [OPTION]... Runs test--cpp for each repo in 'rmm' 'ucxx' ... ``` --- .../devcontainer-feature.json | 2 +- .../bin/generate-scripts.sh | 52 +++++++++++++++++-- .../bin/tmpl/all.cpp.test.tmpl.sh | 32 ++++++++++++ .../bin/tmpl/all.python.test.tmpl.sh | 32 ++++++++++++ .../bin/tmpl/all.test.tmpl.sh | 32 ++++++++++++ .../bin/tmpl/cpp.test.tmpl.sh | 34 ++++++++++++ .../bin/tmpl/python.test.tmpl.sh | 34 ++++++++++++ .../bin/tmpl/repo.test.tmpl.sh | 38 ++++++++++++++ .../opt/rapids-build-utils/manifest.yaml | 28 ++++++++++ 9 files changed, 280 insertions(+), 4 deletions(-) create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.test.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.test.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.test.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.test.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.test.tmpl.sh create mode 100755 features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.test.tmpl.sh diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 1a43ccbb..3ae91b3e 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "26.2.2", + "version": "26.2.3", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 41d200e7..5c39117f 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -49,7 +49,7 @@ generate_script() { if test -n "${bin:+x}"; then ( cat - \ - | envsubst '$HOME $NAME $SRC_PATH $PY_ENV $PY_SRC $PY_LIB $BIN_DIR $CPP_ENV $CPP_LIB $CPP_SRC $CPP_CMAKE_ARGS $CPP_CPACK_ARGS $CPP_DEPS $CPP_MAX_TOTAL_SYSTEM_MEMORY $CPP_MAX_DEVICE_OBJ_MEMORY_USAGE $CPP_MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL $GIT_TAG $GIT_SSH_URL $GIT_HTTPS_URL $GIT_REPO $GIT_HOST $GIT_UPSTREAM $PIP_WHEEL_ARGS $PIP_INSTALL_ARGS' \ + | envsubst '$HOME $NAME $SRC_PATH $PY_ENV $PY_SRC $PY_LIB $PY_TEST $BIN_DIR $CPP_ENV $CPP_LIB $CPP_SRC $CPP_TEST $CPP_CMAKE_ARGS $CPP_CPACK_ARGS $CPP_DEPS $CPP_MAX_TOTAL_SYSTEM_MEMORY $CPP_MAX_DEVICE_OBJ_MEMORY_USAGE $CPP_MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL $GIT_TAG $GIT_SSH_URL $GIT_HTTPS_URL $GIT_REPO $GIT_HOST $GIT_UPSTREAM $PIP_WHEEL_ARGS $PIP_INSTALL_ARGS' \ | tee "${TMP_SCRIPT_DIR}/${bin}" >/dev/null; chmod +x "${TMP_SCRIPT_DIR}/${bin}"; @@ -150,6 +150,33 @@ generate_python_scripts() { done } +generate_cpp_test_script() { + if test -n "${CPP_TEST:-}" && test -f "${TEMPLATES}/cpp.test.tmpl.sh"; then ( + # shellcheck disable=SC2002 + cat "${TEMPLATES}/cpp.test.tmpl.sh" \ + | generate_script "test-${CPP_LIB}-cpp"; + ) || true; + fi +} + +generate_python_test_script() { + if test -n "${PY_TEST:-}" && test -f "${TEMPLATES}/python.test.tmpl.sh"; then ( + # shellcheck disable=SC2002 + cat "${TEMPLATES}/python.test.tmpl.sh" \ + | generate_script "test-${PY_LIB}-python"; + ) || true; + fi +} + +generate_repo_test_script() { + if test -f "${TEMPLATES}/repo.test.tmpl.sh"; then ( + # shellcheck disable=SC2002 + cat "${TEMPLATES}/repo.test.tmpl.sh" \ + | generate_script "test-${NAME}"; + ) || true; + fi +} + generate_scripts() { local -; set -euo pipefail; @@ -185,6 +212,7 @@ generate_scripts() { local cpp_name; local cpp_path; local cpp_sub_dir; + local cpp_test; local cpp_cmake_args; local cpp_cpack_args; local cpp_depends_length; @@ -195,6 +223,7 @@ generate_scripts() { local py_env; local py_path; local py_name; + local py_test; local py_cmake_args; local pip_wheel_args; local pip_install_args; @@ -254,6 +283,7 @@ generate_scripts() { cpp_env="${repo}_cpp_${j}_env"; cpp_name="${repo}_cpp_${j}_name"; cpp_sub_dir="${repo}_cpp_${j}_sub_dir"; + cpp_test="${repo}_cpp_${j}_test"; cpp_cmake_args="${repo}_cpp_${j}_args_cmake"; cpp_cpack_args="${repo}_cpp_${j}_args_cpack"; cpp_depends_length="${repo}_cpp_${j}_depends_length"; @@ -295,6 +325,7 @@ generate_scripts() { CPP_ENV="${!cpp_env:-}" \ CPP_LIB="${cpp_name:-}" \ CPP_SRC="${!cpp_sub_dir:-}" \ + CPP_TEST="${!cpp_test:-}" \ CPP_DEPS="${cpp_deps[*]}" \ CPP_CMAKE_ARGS="${!cpp_cmake_args:-}" \ CPP_CPACK_ARGS="${!cpp_cpack_args:-}" \ @@ -302,6 +333,11 @@ generate_scripts() { CPP_MAX_DEVICE_OBJ_MEMORY_USAGE="${!cpp_max_device_obj_memory_usage:-}" \ CPP_MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL="${!cpp_max_device_obj_to_compile_in_parallel:-}" \ generate_cpp_scripts ; + NAME="${repo_name:-}" \ + SRC_PATH=~/"${!repo_path:-}" \ + CPP_LIB="${cpp_name:-}" \ + CPP_TEST="${!cpp_test:-}" \ + generate_cpp_test_script ; fi done @@ -310,6 +346,7 @@ generate_scripts() { for ((j=0; j < ${!py_length:-0}; j+=1)); do py_env="${repo}_python_${j}_env"; py_name="${repo}_python_${j}_name"; + py_test="${repo}_python_${j}_test"; py_cmake_args="${repo}_python_${j}_args_cmake"; pip_wheel_args="${repo}_python_${j}_args_wheel"; pip_install_args="${repo}_python_${j}_args_install"; @@ -352,6 +389,11 @@ generate_scripts() { PIP_WHEEL_ARGS="${!pip_wheel_args:-}" \ PIP_INSTALL_ARGS="${!pip_install_args:-}" \ generate_python_scripts ; + NAME="${repo_name:-}" \ + SRC_PATH=~/"${!repo_path:-}" \ + PY_LIB="${py_name}" \ + PY_TEST="${!py_test:-}" \ + generate_python_test_script ; fi done @@ -362,6 +404,10 @@ generate_scripts() { PY_LIB="${py_libs[*]@Q}" \ CPP_LIB="${cpp_libs[*]@Q}" \ generate_repo_scripts ; + NAME="${repo_name:-}" \ + PY_LIB="${py_libs[*]@Q}" \ + CPP_LIB="${cpp_libs[*]@Q}" \ + generate_repo_test_script ; fi # Generate a clone script for each repo @@ -383,7 +429,7 @@ generate_scripts() { unset cpp_name_to_path; if ((${#repo_names[@]} > 0)); then - for script in "clone" "clean" "configure" "build" "cpack" "install" "uninstall"; do + for script in "clone" "clean" "configure" "build" "cpack" "install" "uninstall" "test"; do # Generate a script to run a script for all repos NAME="${cloned_repos[0]:-${repo_names[0]:-}}" \ NAMES="${repo_names[*]@Q}" \ @@ -392,7 +438,7 @@ generate_scripts() { generate_all_script ; done - for script in "clean" "build"; do + for script in "clean" "build" "test"; do # Generate a script to run C++ builds for all repos NAME="${cpp_names[0]:-${repo_names[0]:-}}" \ NAMES="${cpp_names[*]@Q}" \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.test.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.test.tmpl.sh new file mode 100755 index 00000000..cf24ae3e --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.cpp.test.tmpl.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Usage: +# test-all-cpp [OPTION]... +# +# Runs test--cpp for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# Boolean options: +# -h,--help Print this text. + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +_test_all_cpp() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'test-all test-all-cpp'; + + for name in ${NAMES}; do + if command -V test-${name}-cpp >/dev/null 2>&1; then + test-${name}-cpp "${OPTS[@]}"; + fi + done +} + +_test_all_cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.test.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.test.tmpl.sh new file mode 100755 index 00000000..c2b66a41 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.python.test.tmpl.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Usage: +# test-all-python [OPTION]... +# +# Runs test--python for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# Boolean options: +# -h,--help Print this text. + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +_test_all_python() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'test-all test-all-python'; + + for name in ${NAMES}; do + if command -V test-${name}-python >/dev/null 2>&1; then + test-${name}-python "${OPTS[@]}"; + fi + done +} + +_test_all_python "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.test.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.test.tmpl.sh new file mode 100755 index 00000000..0ee21917 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/all.test.tmpl.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Usage: +# test-all [OPTION]... +# +# Runs test- for each repo in ${NAMES}. +# +# Forwards relevant arguments to each underlying script. +# +# Boolean options: +# -h,--help Print this text. + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +test_all() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'test-all'; + + for name in ${NAMES}; do + if command -V test-${name} >/dev/null 2>&1; then + test-${name} "${OPTS[@]}"; + fi + done +} + +test_all "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.test.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.test.tmpl.sh new file mode 100755 index 00000000..84e98345 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.test.tmpl.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Usage: +# test-${CPP_LIB}-cpp [OPTION]... +# +# Run ${CPP_LIB} C++ tests. +# +# Boolean options: +# -h,--help Print this text. + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +test_${CPP_LIB}_cpp() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'test-all test-${NAME} test-${CPP_LIB}-cpp'; + + if [[ ! -d "${SRC_PATH}" ]]; then + echo "test-${CPP_LIB}-cpp: cannot access '${SRC_PATH}': No such directory" >&2; + return 1; + fi + + time ( + cd "${SRC_PATH}"; + ./${CPP_TEST} "${OPTS[@]}"; + ) +} + +test_${CPP_LIB}_cpp "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.test.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.test.tmpl.sh new file mode 100755 index 00000000..c2d180d0 --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.test.tmpl.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Usage: +# test-${PY_LIB}-python [OPTION]... +# +# Run ${PY_LIB} Python tests. +# +# Boolean options: +# -h,--help Print this text. + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +test_${PY_LIB}_python() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'test-all test-${NAME} test-${PY_LIB}-python'; + + if [[ ! -d "${SRC_PATH}" ]]; then + echo "test-${PY_LIB}-python: cannot access '${SRC_PATH}': No such directory" >&2; + return 1; + fi + + time ( + cd "${SRC_PATH}"; + ./${PY_TEST} "${OPTS[@]}"; + ) +} + +test_${PY_LIB}_python "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.test.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.test.tmpl.sh new file mode 100755 index 00000000..bf4f625f --- /dev/null +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.test.tmpl.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Usage: +# test-${NAME} [OPTION]... +# +# Run ${NAME} tests. +# +# Forwards relevant arguments to each underlying script. +# +# Boolean options: +# -h,--help Print this text. + +# shellcheck disable=SC1091 +. rapids-generate-docstring; + +test_${NAME}() { + local -; + set -euo pipefail; + + eval "$(_parse_args --take '-h,--help' "$@" <&0)"; + + # shellcheck disable=SC1091 + . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'test-all test-${NAME}'; + + for lib in ${CPP_LIB}; do + if command -V test-${lib}-cpp >/dev/null 2>&1; then + test-${lib}-cpp "${OPTS[@]}"; + fi + done + + for lib in ${PY_LIB}; do + if command -V test-${lib}-python >/dev/null 2>&1; then + test-${lib}-python "${OPTS[@]}"; + fi + done +} + +test_${NAME} "$@" <&0; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index d244fb5c..a70fcdc3 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -16,6 +16,7 @@ repos: cpp: - name: rmm sub_dir: cpp + test: ci/run_ctests.sh python: - name: librmm sub_dir: python/librmm @@ -25,6 +26,7 @@ repos: sub_dir: python/rmm depends: [rmm] args: {install: *rapids_build_backend_args} + test: ci/run_pytests.sh - name: ucxx path: ucxx @@ -57,6 +59,7 @@ repos: git: {<<: *git_defaults, repo: dask-cuda} python: - name: dask-cuda + test: ci/run_pytest.sh - name: kvikio path: kvikio @@ -64,6 +67,7 @@ repos: cpp: - name: kvikio sub_dir: cpp + test: ci/run_ctests.sh python: - name: libkvikio sub_dir: python/libkvikio @@ -73,6 +77,7 @@ repos: sub_dir: python/kvikio depends: [kvikio] args: {install: *rapids_build_backend_args} + test: ci/run_pytests.sh - name: cudf path: cudf @@ -81,9 +86,11 @@ repos: - name: cudf sub_dir: cpp depends: [rmm, kvikio] + test: ci/run_cudf_ctests.sh - name: cudf_kafka sub_dir: cpp/libcudf_kafka depends: [cudf] + test: ci/run_cudf_kafka_ctests.sh python: - name: libcudf sub_dir: python/libcudf @@ -93,23 +100,29 @@ repos: sub_dir: python/pylibcudf depends: [cudf] args: {install: *rapids_build_backend_args} + test: ci/run_pylibcudf_pytests.sh - name: cudf sub_dir: python/cudf depends: [cudf] args: {install: *rapids_build_backend_args} + test: ci/run_cudf_pytests.sh - name: dask-cudf sub_dir: python/dask_cudf args: {install: *rapids_build_backend_args} + test: ci/run_dask_cudf_pytests.sh - name: cudf-polars sub_dir: python/cudf_polars args: {install: *rapids_build_backend_args} + test: ci/run_cudf_polars_pytests.sh - name: cudf_kafka sub_dir: python/cudf_kafka depends: [cudf_kafka] args: {install: *rapids_build_backend_args} + test: ci/run_cudf_kafka_pytests.sh - name: custreamz sub_dir: python/custreamz args: {install: *rapids_build_backend_args} + test: ci/run_custreamz_pytests.sh - name: rapidsmpf path: rapidsmpf @@ -118,6 +131,7 @@ repos: - name: rapidsmpf sub_dir: cpp depends: [rmm, ucxx, cudf] + test: ci/run_ctests.sh python: - name: librapidsmpf sub_dir: python/librapidsmpf @@ -127,6 +141,7 @@ repos: sub_dir: python/rapidsmpf depends: [rapidsmpf, ucxx-python] args: {install: *rapids_build_backend_args} + test: ci/run_pytests.sh - name: raft path: raft @@ -138,6 +153,7 @@ repos: parallelism: max_device_obj_memory_usage: 3Gi args: {cmake: -DRAFT_COMPILE_LIBRARY=ON} + test: ci/run_ctests.sh python: - name: libraft sub_dir: python/libraft @@ -147,10 +163,12 @@ repos: sub_dir: python/pylibraft depends: [raft] args: {install: *rapids_build_backend_args} + test: ci/run_pylibraft_pytests.sh - name: raft-dask sub_dir: python/raft-dask depends: [ucxx, raft] args: {install: *rapids_build_backend_args} + test: ci/run_raft_dask_pytests.sh - name: cuvs path: cuvs @@ -162,6 +180,7 @@ repos: parallelism: max_device_obj_memory_usage: 3Gi args: {cmake: -DBUILD_C_LIBRARY=ON} + test: ci/run_ctests.sh python: - name: libcuvs sub_dir: python/libcuvs @@ -171,6 +190,7 @@ repos: sub_dir: python/cuvs depends: [cuvs] args: {install: *rapids_build_backend_args} + test: ci/run_cuvs_pytests.sh - name: cumlprims_mg path: cumlprims_mg @@ -191,6 +211,7 @@ repos: depends: [rmm, raft, cumlprims_mg, cuvs] parallelism: max_device_obj_memory_usage: 3Gi + test: ci/run_ctests.sh python: - name: libcuml sub_dir: python/libcuml @@ -200,6 +221,7 @@ repos: sub_dir: python/cuml depends: [cuml] args: {install: *rapids_build_backend_args} + test: ci/run_cuml_singlegpu_pytests.sh - name: cugraph path: cugraph @@ -210,6 +232,7 @@ repos: depends: [rmm, ucxx, raft] parallelism: max_device_obj_memory_usage: 6Gi + test: ci/run_ctests.sh - name: cugraph_etl sub_dir: cpp/libcugraph_etl depends: [cudf, cugraph] @@ -223,10 +246,12 @@ repos: sub_dir: python/pylibcugraph depends: [cugraph] args: {install: *rapids_build_backend_args} + test: ci/run_pylibcugraph_pytests.sh - name: cugraph sub_dir: python/cugraph depends: [cugraph] args: {install: *rapids_build_backend_args} + test: ci/run_cugraph_pytests.sh - name: cugraph-gnn path: cugraph-gnn @@ -237,6 +262,7 @@ repos: depends: [raft] args: cmake: -DCMAKE_CUDA_ARCHITECTURES="${CUDAARCHS}" + test: ci/run_ctests.sh python: - name: libwholegraph sub_dir: python/libwholegraph @@ -246,9 +272,11 @@ repos: sub_dir: python/pylibwholegraph depends: [wholegraph] args: {install: *rapids_build_backend_args} + test: ci/run_pylibwholegraph_pytests.sh - name: cugraph_pyg sub_dir: python/cugraph-pyg args: {install: *rapids_build_backend_args} + test: ci/run_cugraph_pyg_pytests.sh - name: nx-cugraph path: nx-cugraph From 8f740c4c2107132396839efd57cf0ab117443ff1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 11 Dec 2025 08:46:51 -0800 Subject: [PATCH 4/7] Make feature downloads more robust for CI environments with transparent proxies Added retry logic and better error handling to wget commands in devcontainer features to handle intermittent network issues when building through proxies: Ninja feature (features/src/ninja/install.sh): - Added --tries=3 --timeout=30 to wget for ninja binary download - Made bash-completion download non-fatal (continues on failure) - Added echo statements to show download progress Utils feature (features/src/utils/install.sh): - Added --tries=3 --timeout=30 to wget for gh-nv-gha-aws download - Added explicit error handling with exit 1 on failure - Added echo statement to show download progress These changes ensure builds can complete successfully even with transient network issues in CI environments using mitmproxy for traffic capture. --- features/src/ninja/install.sh | 18 ++++++++++++------ features/src/utils/install.sh | 8 ++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/features/src/ninja/install.sh b/features/src/ninja/install.sh index 04f23c26..1859d225 100644 --- a/features/src/ninja/install.sh +++ b/features/src/ninja/install.sh @@ -23,15 +23,21 @@ if test "$(uname -p)" = "aarch64"; then _name+="-aarch64"; fi -# Install Ninja -wget --no-hsts -q -O /tmp/ninja-linux.zip \ - "https://github.com/ninja-build/ninja/releases/download/v${NINJA_VERSION}/${_name}.zip"; +# Install Ninja with retries for network reliability +echo "Downloading ninja v${NINJA_VERSION}..."; +wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/ninja-linux.zip \ + "https://github.com/ninja-build/ninja/releases/download/v${NINJA_VERSION}/${_name}.zip" || { + echo "ERROR: Failed to download ninja after 3 attempts"; + exit 1; +}; unzip -d /usr/bin /tmp/ninja-linux.zip; chmod +x /usr/bin/ninja; -# Install Ninja bash completions -wget --no-hsts -q -O /usr/share/bash-completion/completions/ninja \ - "https://github.com/ninja-build/ninja/raw/v${NINJA_VERSION}/misc/bash-completion"; +# Install Ninja bash completions (non-fatal if it fails) +echo "Downloading ninja bash-completion..."; +wget --no-hsts -q --tries=3 --timeout=30 -O /usr/share/bash-completion/completions/ninja \ + "https://github.com/ninja-build/ninja/raw/v${NINJA_VERSION}/misc/bash-completion" || \ + echo "Warning: Failed to download ninja bash-completion (non-fatal)"; # Clean up rm -rf /var/tmp/*; diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index fb814dd1..901093d7 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -206,8 +206,12 @@ if test -n "${USERNAME:+x}"; then ; NV_GHA_AWS_VERSION=latest find_version_from_git_tags NV_GHA_AWS_VERSION https://github.com/nv-gha-runners/gh-nv-gha-aws; - wget --no-hsts -q -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \ - "https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')"; + echo "Downloading gh-nv-gha-aws v${NV_GHA_AWS_VERSION}..."; + wget --no-hsts -q --tries=3 --timeout=30 -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \ + "https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')" || { + echo "ERROR: Failed to download gh-nv-gha-aws after 3 attempts"; + exit 1; + }; chmod 0755 "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws"; cat <"$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/manifest.yml" owner: nv-gha-runners From b7e2eb4db1f69ad94297bc502df8e091ca05e1c7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 11 Dec 2025 10:01:05 -0800 Subject: [PATCH 5/7] Add timeouts to wget downloads in cmake and rapids-build-utils features cmake feature (features/src/cmake/install.sh): - Added --tries=3 --timeout=30 to both wget commands - Prevents indefinite hangs when downloading CMake installers from GitHub rapids-build-utils feature (features/src/rapids-build-utils/install.sh): - Added --tries=3 --timeout=30 to wget command for yq download - Prevents indefinite hangs when downloading yq from GitHub These timeouts prevent the 5-minute SSL connection timeouts observed with git-lfs and ensure builds fail fast rather than hanging. --- features/src/cmake/install.sh | 4 ++-- features/src/rapids-build-utils/install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/features/src/cmake/install.sh b/features/src/cmake/install.sh index b9a603e9..0583b945 100644 --- a/features/src/cmake/install.sh +++ b/features/src/cmake/install.sh @@ -15,12 +15,12 @@ echo "Downloading CMake..."; if [[ "${CMAKE_VERSION}" == "latest" ]]; then find_version_from_git_tags CMAKE_VERSION https://github.com/Kitware/CMake; - while ! wget --no-hsts -q -O /tmp/cmake_${CMAKE_VERSION}.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -p).sh; do + while ! wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/cmake_${CMAKE_VERSION}.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -p).sh; do echo "(!) cmake version ${CMAKE_VERSION} failed to download. Attempting to fall back one version to retry..."; find_prev_version_from_git_tags CMAKE_VERSION https://github.com/Kitware/CMake; done else - wget --no-hsts -q -O /tmp/cmake_${CMAKE_VERSION}.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -p).sh; + wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/cmake_${CMAKE_VERSION}.sh https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -p).sh; fi echo "Installing CMake..."; diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index 78f34900..fcc684ce 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -25,7 +25,7 @@ if ! command -V yq >/dev/null 2>&1; then YQ_VERSION=4.46.1; find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; - while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do + while ! wget --no-hsts -q --tries=3 --timeout=30 -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry..."; find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; done From 1a1bb19ab3d2916fc02da4df898f0e45a3a1c24a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 11 Dec 2025 10:46:02 -0800 Subject: [PATCH 6/7] Make gh-nv-gha-aws wget more robust and verbose for mitmproxy debugging Changes to the gh-nv-gha-aws download: - Replace -q with --verbose for detailed connection debugging - Add --ca-certificate=/etc/ssl/certs/ca-certificates.crt to explicitly use CA bundle - Add --waitretry=5 to wait 5 seconds between retry attempts (reduces hammering) - Add --dns-timeout=10 for faster DNS failure detection - Add --connect-timeout=30 separate from read timeout - Improve error message to show the actual URL being attempted This helps diagnose SSL/connection issues when traffic goes through mitmproxy's transparent proxy, and makes retries more robust with delays between attempts. --- features/src/utils/install.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 901093d7..c7074f47 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -207,9 +207,14 @@ if test -n "${USERNAME:+x}"; then NV_GHA_AWS_VERSION=latest find_version_from_git_tags NV_GHA_AWS_VERSION https://github.com/nv-gha-runners/gh-nv-gha-aws; echo "Downloading gh-nv-gha-aws v${NV_GHA_AWS_VERSION}..."; - wget --no-hsts -q --tries=3 --timeout=30 -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \ + # Use verbose output for debugging, explicit CA cert for mitmproxy, and retry delays + wget --no-hsts --verbose --tries=3 --timeout=30 --waitretry=5 \ + --ca-certificate=/etc/ssl/certs/ca-certificates.crt \ + --dns-timeout=10 --connect-timeout=30 \ + -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \ "https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')" || { - echo "ERROR: Failed to download gh-nv-gha-aws after 3 attempts"; + echo "ERROR: Failed to download gh-nv-gha-aws after 3 attempts with 5s delays"; + echo "URL attempted: https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')"; exit 1; }; chmod 0755 "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws"; From 67ef3110b00ef5958c8388c5933fe961bd4d2b06 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 11 Dec 2025 10:49:40 -0800 Subject: [PATCH 7/7] add retries to all wget calls, update feature versions --- features/src/cccl-dev/devcontainer-feature.json | 2 +- features/src/cccl-dev/install.sh | 2 +- features/src/cmake/devcontainer-feature.json | 2 +- features/src/cuda/devcontainer-feature.json | 2 +- features/src/cuda/install.sh | 4 ++-- features/src/gcc/devcontainer-feature.json | 2 +- features/src/gitlab-cli/devcontainer-feature.json | 2 +- features/src/gitlab-cli/install.sh | 2 +- features/src/llvm/devcontainer-feature.json | 2 +- features/src/llvm/llvm.sh | 6 +++--- features/src/mambaforge/devcontainer-feature.json | 2 +- features/src/mambaforge/install.sh | 2 +- features/src/ninja/devcontainer-feature.json | 2 +- features/src/nvhpc/devcontainer-feature.json | 2 +- features/src/nvhpc/install.sh | 2 +- features/src/oneapi/devcontainer-feature.json | 2 +- features/src/oneapi/install.sh | 2 +- features/src/openmpi/devcontainer-feature.json | 2 +- features/src/openmpi/install.sh | 2 +- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- features/src/sccache/devcontainer-feature.json | 2 +- features/src/sccache/install.sh | 2 +- features/src/ucx/devcontainer-feature.json | 2 +- features/src/ucx/install.sh | 6 +++--- features/src/utils/devcontainer-feature.json | 2 +- features/src/utils/install.sh | 2 +- 26 files changed, 31 insertions(+), 31 deletions(-) diff --git a/features/src/cccl-dev/devcontainer-feature.json b/features/src/cccl-dev/devcontainer-feature.json index 95b16301..6fcfbb7c 100644 --- a/features/src/cccl-dev/devcontainer-feature.json +++ b/features/src/cccl-dev/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA CCCL development utilities", "id": "cccl-dev", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install NVIDIA CCCL development utilities", "options": { "litVersion": { diff --git a/features/src/cccl-dev/install.sh b/features/src/cccl-dev/install.sh index e0442d32..7d7d1d4d 100644 --- a/features/src/cccl-dev/install.sh +++ b/features/src/cccl-dev/install.sh @@ -65,7 +65,7 @@ DOXYGEN_URL="https://github.com/doxygen/doxygen/releases/download/Release_${DOXY ( TEMPDEST=$(mktemp -d) - wget --no-hsts -q "$DOXYGEN_URL" -O "$TEMPDEST/doxygen.tar.gz" + wget --no-hsts -q --tries=3 --timeout=30 "$DOXYGEN_URL" -O "$TEMPDEST/doxygen.tar.gz" cd "$TEMPDEST" tar --strip-components=1 -xf doxygen.tar.gz make install -j diff --git a/features/src/cmake/devcontainer-feature.json b/features/src/cmake/devcontainer-feature.json index ea1ae546..3313fa99 100644 --- a/features/src/cmake/devcontainer-feature.json +++ b/features/src/cmake/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "CMake", "id": "cmake", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install CMake", "options": { "version": { diff --git a/features/src/cuda/devcontainer-feature.json b/features/src/cuda/devcontainer-feature.json index 7a1abbc7..08be993f 100644 --- a/features/src/cuda/devcontainer-feature.json +++ b/features/src/cuda/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "CUDA Toolkit", "id": "cuda", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install the NVIDIA CUDA Toolkit", "options": { "version": { diff --git a/features/src/cuda/install.sh b/features/src/cuda/install.sh index 7e0b218d..c419daba 100644 --- a/features/src/cuda/install.sh +++ b/features/src/cuda/install.sh @@ -38,7 +38,7 @@ fi get_cuda_deb() { local deb="$( \ - wget --no-hsts -q -O- "${1}/Packages" \ + wget --no-hsts -q --tries=3 --timeout=30 -O- "${1}/Packages" \ | grep -P "^Filename: \./${2}(.*)\.deb$" \ | sort -Vr | head -n1 | cut -d' ' -f2 \ )"; @@ -46,7 +46,7 @@ get_cuda_deb() { echo "Error: No matching .deb found for '${1}' and '${2}'" >&2 return 1 fi - wget --no-hsts -q -O "/tmp/${deb#./}" "${1}/${deb#./}"; + wget --no-hsts -q --tries=3 --timeout=30 -O "/tmp/${deb#./}" "${1}/${deb#./}"; echo -n "/tmp/${deb#./}"; } diff --git a/features/src/gcc/devcontainer-feature.json b/features/src/gcc/devcontainer-feature.json index be692326..962e1e74 100644 --- a/features/src/gcc/devcontainer-feature.json +++ b/features/src/gcc/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "GCC", "id": "gcc", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install gcc", "options": { "version": { diff --git a/features/src/gitlab-cli/devcontainer-feature.json b/features/src/gitlab-cli/devcontainer-feature.json index 2363261c..3f56c37d 100644 --- a/features/src/gitlab-cli/devcontainer-feature.json +++ b/features/src/gitlab-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "gitlab-cli", - "version": "26.2.0", + "version": "26.2.1", "name": "GitLab CLI", "documentationURL": "https://github.com/rapidsai/devcontainers/features/tree/main/src/gitlab-cli", "description": "Installs the GitLab CLI. Auto-detects latest version and installs needed dependencies.", diff --git a/features/src/gitlab-cli/install.sh b/features/src/gitlab-cli/install.sh index eeec4101..a0ac2dd7 100755 --- a/features/src/gitlab-cli/install.sh +++ b/features/src/gitlab-cli/install.sh @@ -33,7 +33,7 @@ gitlab_cli_file_name() { download_gitlab_cli_deb() { local -; set -euo pipefail; - wget --no-hsts -q -O /tmp/gitlab-cli.deb \ + wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/gitlab-cli.deb \ "https://gitlab.com/gitlab-org/cli/-/releases/v${CLI_VERSION}/downloads/$(gitlab_cli_file_name)"; } diff --git a/features/src/llvm/devcontainer-feature.json b/features/src/llvm/devcontainer-feature.json index aadf20fa..44f9a567 100644 --- a/features/src/llvm/devcontainer-feature.json +++ b/features/src/llvm/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "LLVM compilers and tools", "id": "llvm", - "version": "26.2.1", + "version": "26.2.2", "description": "A feature to install LLVM compilers and tools", "options": { "version": { diff --git a/features/src/llvm/llvm.sh b/features/src/llvm/llvm.sh index 4b0e1bd3..0a90028d 100755 --- a/features/src/llvm/llvm.sh +++ b/features/src/llvm/llvm.sh @@ -131,7 +131,7 @@ if [[ -n "${CODENAME}" ]]; then retry_count=0; - while ! wget --no-hsts -q --method=HEAD \ + while ! wget --no-hsts -q --tries=3 --timeout=30 --method=HEAD \ "${BASE_URL}/${CODENAME}/dists/llvm-toolchain${LINKNAME}${LLVM_VERSION_PATTERNS[$LLVM_VERSION]}/Release" >/dev/null 2>&1; do if test $(( retry_count++ )) -ge 5; then LLVM_VERSION_STRING= @@ -145,7 +145,7 @@ if [[ -n "${CODENAME}" ]]; then REPO_NAME="deb ${BASE_URL}/${CODENAME}/ llvm-toolchain${LINKNAME}${LLVM_VERSION_STRING} main" # check if the repository exists for the distro and version - if ! wget --no-hsts -q --method=HEAD "${BASE_URL}/${CODENAME}" >/dev/null 2>&1; then + if ! wget --no-hsts -q --tries=3 --timeout=30 --method=HEAD "${BASE_URL}/${CODENAME}" >/dev/null 2>&1; then if [[ -n "${CODENAME_FROM_ARGUMENTS}" ]]; then echo "Specified codename '${CODENAME}' is not supported by this script." else @@ -160,7 +160,7 @@ fi if [ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]; then # download GPG key once - wget --no-hsts -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + wget --no-hsts -q --tries=3 --timeout=30 -O- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc fi if [[ -z "`apt-key list 2> /dev/null | grep -i llvm`" ]]; then diff --git a/features/src/mambaforge/devcontainer-feature.json b/features/src/mambaforge/devcontainer-feature.json index 2973e5af..5c1b9829 100644 --- a/features/src/mambaforge/devcontainer-feature.json +++ b/features/src/mambaforge/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Mambaforge", "id": "mambaforge", - "version": "26.2.1", + "version": "26.2.2", "description": "A feature to install mambaforge", "options": { "version": { diff --git a/features/src/mambaforge/install.sh b/features/src/mambaforge/install.sh index aa0048c1..4aec8505 100644 --- a/features/src/mambaforge/install.sh +++ b/features/src/mambaforge/install.sh @@ -18,7 +18,7 @@ if [[ "$MINIFORGE_VERSION" == latest ]]; then find_version_from_git_tags MINIFORGE_VERSION https://github.com/conda-forge/miniforge "tags/" "." "-[0-9]+" "true"; fi -wget --no-hsts -q -O /tmp/miniforge.sh \ +wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/miniforge.sh \ "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-$(uname -p).sh"; echo "Installing Miniforge..."; diff --git a/features/src/ninja/devcontainer-feature.json b/features/src/ninja/devcontainer-feature.json index 1dae9f05..01fccd5c 100644 --- a/features/src/ninja/devcontainer-feature.json +++ b/features/src/ninja/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Ninja build", "id": "ninja", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install ninja-build", "options": { "version": { diff --git a/features/src/nvhpc/devcontainer-feature.json b/features/src/nvhpc/devcontainer-feature.json index 4180e4f3..88409d2d 100644 --- a/features/src/nvhpc/devcontainer-feature.json +++ b/features/src/nvhpc/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVHPC SDK", "id": "nvhpc", - "version": "26.2.1", + "version": "26.2.2", "description": "A feature to install the NVHPC SDK", "options": { "version": { diff --git a/features/src/nvhpc/install.sh b/features/src/nvhpc/install.sh index be96cb78..096a52ad 100644 --- a/features/src/nvhpc/install.sh +++ b/features/src/nvhpc/install.sh @@ -28,7 +28,7 @@ fi echo "Downloading NVHPC gpg key..."; -wget --no-hsts -q -O- https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK \ +wget --no-hsts -q --tries=3 --timeout=30 -O- https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK \ | gpg --dearmor -o /etc/apt/trusted.gpg.d/nvidia-hpcsdk-archive-keyring.gpg; chmod 0644 /etc/apt/trusted.gpg.d/*.gpg || true; diff --git a/features/src/oneapi/devcontainer-feature.json b/features/src/oneapi/devcontainer-feature.json index 10b20df7..72649b80 100644 --- a/features/src/oneapi/devcontainer-feature.json +++ b/features/src/oneapi/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Intel oneapi toolchain", "id": "oneapi", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install the Intel oneapi toolchain", "options": { "version": { diff --git a/features/src/oneapi/install.sh b/features/src/oneapi/install.sh index 5a692135..4a8c369c 100644 --- a/features/src/oneapi/install.sh +++ b/features/src/oneapi/install.sh @@ -30,7 +30,7 @@ if [ -f /etc/profile.d/lmod.sh ]; then fi # Add Intel repo signing key -wget --no-hsts -q -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \ +wget --no-hsts -q --tries=3 --timeout=30 -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \ | gpg --dearmor -o /etc/apt/trusted.gpg.d/oneapi-archive-keyring.gpg; chmod 0644 /etc/apt/trusted.gpg.d/*.gpg || true; diff --git a/features/src/openmpi/devcontainer-feature.json b/features/src/openmpi/devcontainer-feature.json index e124eef4..85c1669d 100644 --- a/features/src/openmpi/devcontainer-feature.json +++ b/features/src/openmpi/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "OpenMPI", "id": "openmpi", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install OpenMPI with optional CUDA and UCX support", "options": { "version": { diff --git a/features/src/openmpi/install.sh b/features/src/openmpi/install.sh index b79cd25e..3ce3089d 100755 --- a/features/src/openmpi/install.sh +++ b/features/src/openmpi/install.sh @@ -96,7 +96,7 @@ build_and_install_openmpi() { local -r major_minor="$(grep -o '^[0-9]*.[0-9]*' <<< "${OPENMPI_VERSION}")"; - wget --no-hsts -q -O- "https://download.open-mpi.org/release/open-mpi/v${major_minor}/openmpi-${OPENMPI_VERSION}.tar.gz" \ + wget --no-hsts -q --tries=3 --timeout=30 -O- "https://download.open-mpi.org/release/open-mpi/v${major_minor}/openmpi-${OPENMPI_VERSION}.tar.gz" \ | tar -C /tmp/ompi -zf - --strip-components=1 -x; ( diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 3ae91b3e..76160822 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "26.2.3", + "version": "26.2.4", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/sccache/devcontainer-feature.json b/features/src/sccache/devcontainer-feature.json index e24ca35b..767a7ec2 100644 --- a/features/src/sccache/devcontainer-feature.json +++ b/features/src/sccache/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "sccache", "id": "sccache", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install sccache", "options": { "repository": { diff --git a/features/src/sccache/install.sh b/features/src/sccache/install.sh index c73559f6..1ab2772a 100644 --- a/features/src/sccache/install.sh +++ b/features/src/sccache/install.sh @@ -25,7 +25,7 @@ elif test "$SCCACHE_VERSION" = dev; then fi # Install sccache -wget --no-hsts -q -O- \ +wget --no-hsts -q --tries=3 --timeout=30 -O- \ "https://github.com/$SCCACHE_REPOSITORY/releases/download/v$SCCACHE_VERSION/sccache-v$SCCACHE_VERSION-$(uname -m)-unknown-linux-musl.tar.gz" \ | tar -C /usr/bin -zf - --wildcards --strip-components=1 -x '*/sccache' \ && chmod +x /usr/bin/sccache; diff --git a/features/src/ucx/devcontainer-feature.json b/features/src/ucx/devcontainer-feature.json index 56c374ca..06d3a64e 100644 --- a/features/src/ucx/devcontainer-feature.json +++ b/features/src/ucx/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "UCX", "id": "ucx", - "version": "26.2.0", + "version": "26.2.1", "description": "A feature to install UCX", "options": { "version": { diff --git a/features/src/ucx/install.sh b/features/src/ucx/install.sh index b7478179..d805d948 100755 --- a/features/src/ucx/install.sh +++ b/features/src/ucx/install.sh @@ -38,12 +38,12 @@ download_ucx_release() { if [ "$(uname -p)" = "x86_64" ]; then # https://github.com/openucx/ucx/releases/download/v1.15.0-rc3/ucx-1.15.0-rc3-ubuntu22.04-mofed5-cuda12-x86_64.tar.bz2 - if ! wget --no-hsts -q -O /tmp/ucx.tar.bz2 "https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/${slug}-$(uname -p).tar.bz2"; then + if ! wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/ucx.tar.bz2 "https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/${slug}-$(uname -p).tar.bz2"; then # https://github.com/openucx/ucx/releases/download/v1.14.1/ucx-1.14.1-ubuntu22.04-mofed5-cuda12.tar.bz2 - wget --no-hsts -q -O /tmp/ucx.tar.bz2 "https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/${slug}.tar.bz2"; + wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/ucx.tar.bz2 "https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/${slug}.tar.bz2"; fi else - wget --no-hsts -q -O /tmp/ucx.tar.bz2 "https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/${slug}-$(uname -p).tar.bz2"; + wget --no-hsts -q --tries=3 --timeout=30 -O /tmp/ucx.tar.bz2 "https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/${slug}-$(uname -p).tar.bz2"; fi } diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index 3fbc857d..98a1bc49 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "26.2.4", + "version": "26.2.5", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 901093d7..8b47694f 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -53,7 +53,7 @@ if ! command -V yq >/dev/null 2>&1; then YQ_VERSION=4.46.1; find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; - while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do + while ! wget --no-hsts -q --tries=3 --timeout=30 -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry..."; find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; done