Use sccache-dist build cluster for conda and wheel builds

.devcontainer/Dockerfile

@@ -42,6 +42,7 @@ ENV HISTFILE="/home/coder/.cache/._bash_history"
 ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
+ENV SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
 ENV SCCACHE_IDLE_TIMEOUT=0
 
 ###

.github/workflows/build.yaml

@@ -39,8 +39,10 @@ jobs:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
+      node_type: cpu16
       script: ci/build_cpp.sh
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
 
   rocky8-clib-standalone-build:
     secrets: inherit
@@ -63,6 +65,7 @@ jobs:
       artifact-name: "libcuvs_c_${{ matrix.cuda_version }}.tar.gz"
       file_to_upload: "libcuvs_c.tar.gz"
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   rust-build:
     needs: cpp-build
     secrets: inherit
@@ -84,6 +87,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       script: "ci/build_rust.sh"
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   go-build:
     needs: cpp-build
     secrets: inherit
@@ -137,6 +141,7 @@ jobs:
       date: ${{ inputs.date }}
       script: ci/build_python.sh
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   upload-conda:
     needs: [cpp-build, python-build]
     secrets: inherit
@@ -173,11 +178,13 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
+      node_type: cpu16
       script: ci/build_wheel_libcuvs.sh
       # build for every combination of arch and CUDA version, but only for the latest Python
       matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       package-name: libcuvs
       package-type: cpp
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   wheel-publish-libcuvs:
     needs: wheel-build-libcuvs
     secrets: inherit
@@ -198,8 +205,10 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
+      node_type: cpu8
       script: ci/build_wheel_cuvs.sh
       package-name: cuvs
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
       package-type: python
   wheel-publish-cuvs:
     needs: wheel-build-cuvs

.github/workflows/pr.yaml

@@ -144,6 +144,7 @@ jobs:
       build_type: pull-request
       node_type: cpu16
       script: ci/build_cpp.sh
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   conda-cpp-tests:
     needs: [conda-cpp-build, changed-files]
     secrets: inherit
@@ -152,6 +153,7 @@ jobs:
     with:
       build_type: pull-request
       script: ci/test_cpp.sh
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   conda-cpp-checks:
     needs: conda-cpp-build
     secrets: inherit
@@ -174,6 +176,7 @@ jobs:
     with:
       build_type: pull-request
       script: ci/test_python.sh
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   rocky8-clib-standalone-build:
     needs: [checks]
     secrets: inherit
@@ -195,6 +198,7 @@ jobs:
       artifact-name: "libcuvs_c_${{ matrix.cuda_version }}.tar.gz"
       file_to_upload: "libcuvs_c.tar.gz"
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   rocky8-clib-tests:
     needs: [rocky8-clib-standalone-build, changed-files]
     secrets: inherit
@@ -254,6 +258,7 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:25.12-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13"
       script: "ci/build_rust.sh"
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   go-build:
     needs: [conda-cpp-build, changed-files]
     secrets: inherit
@@ -289,20 +294,24 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/25.12
     with:
       build_type: pull-request
+      node_type: cpu16
       script: ci/build_wheel_libcuvs.sh
       # build for every combination of arch and CUDA version, but only for the latest Python
       matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       package-name: libcuvs
       package-type: cpp
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   wheel-build-cuvs:
     needs: wheel-build-libcuvs
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/25.12
     with:
       build_type: pull-request
+      node_type: cpu8
       script: ci/build_wheel_cuvs.sh
       package-name: cuvs
       package-type: python
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   wheel-tests-cuvs:
     needs: [wheel-build-cuvs, changed-files]
     secrets: inherit
@@ -311,6 +320,7 @@ jobs:
     with:
       build_type: pull-request
       script: ci/test_wheel_cuvs.sh
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   devcontainer:
     secrets: inherit
     needs: telemetry-setup

.github/workflows/test.yaml

@@ -41,6 +41,7 @@ jobs:
       date: ${{ inputs.date }}
       script: ci/test_cpp.sh
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   conda-python-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@release/25.12
@@ -50,6 +51,7 @@ jobs:
       date: ${{ inputs.date }}
       script: ci/test_python.sh
       sha: ${{ inputs.sha }}
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
   conda-java-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/25.12
@@ -79,3 +81,4 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: ci/test_wheel_cuvs.sh
+      sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN

ci/build_cpp.sh

@@ -5,7 +5,6 @@
 set -euo pipefail
 
 source rapids-configure-sccache
-
 source rapids-date-string
 
 export CMAKE_GENERATOR=Ninja
@@ -14,8 +13,7 @@ rapids-print-env
 
 rapids-logger "Begin cpp build"
 
-sccache --zero-stats
-
+sccache --stop-server 2>/dev/null || true
 
 RAPIDS_PACKAGE_VERSION=$(rapids-generate-version)
 export RAPIDS_PACKAGE_VERSION
@@ -35,6 +33,7 @@ rattler-build build --recipe conda/recipes/libcuvs \
                     "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
 
 # remove build_cache directory
 rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache

ci/build_python.sh

@@ -5,7 +5,6 @@
 set -euo pipefail
 
 source rapids-configure-sccache
-
 source rapids-date-string
 
 export CMAKE_GENERATOR=Ninja
@@ -20,20 +19,17 @@ version=$(rapids-generate-version)
 export RAPIDS_PACKAGE_VERSION=${version}
 echo "${version}" > VERSION
 
-sccache --zero-stats
-
-
 # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array
 source rapids-rattler-channel-string
 
 rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS"
 
 RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}")
 
-sccache --zero-stats
-
 rapids-logger "Building cuvs"
 
+sccache --stop-server 2>/dev/null || true
+
 # --no-build-id allows for caching with `sccache`
 # more info is available at
 # https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build
@@ -42,15 +38,15 @@ rattler-build build --recipe conda/recipes/cuvs \
                     "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
-sccache --zero-stats
+sccache --stop-server >/dev/null 2>&1 || true
 
 rattler-build build --recipe conda/recipes/cuvs-bench \
                     --test skip \
                     "${RATTLER_ARGS[@]}" \
                     "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
-sccache --zero-stats
+sccache --stop-server >/dev/null 2>&1 || true
 
 # Build cuvs-bench-cpu only in one CUDA major version since it only depends on
 # python version
@@ -60,4 +56,5 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "13" ]]; then
                       "${RATTLER_ARGS[@]}" \
                       "${RATTLER_CHANNELS[@]}"
   sccache --show-adv-stats
+  sccache --stop-server >/dev/null 2>&1 || true
 fi

ci/build_rust.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION.
 # SPDX-License-Identifier: Apache-2.0
 
 set -euo pipefail
@@ -25,8 +25,20 @@ set +eu
 conda activate rust
 set -eu
 
+source rapids-configure-sccache
+
+# Don't use the build cluster because conda rust toolchains are too large
+export SCCACHE_NO_DIST_COMPILE=1
+export SCCACHE_S3_KEY_PREFIX="cuvs-rs/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}"
+export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="cuvs-rs/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache"
+export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
+
 rapids-print-env
 
+rapids-logger "Begin rust build"
+
+sccache --stop-server 2>/dev/null || true
+
 # we need to set up LIBCLANG_PATH to allow rust bindgen to work,
 # grab it from the conda env
 LIBCLANG_PATH=$(dirname "$(find /opt/conda -name libclang.so | head -n 1)")
@@ -35,6 +47,9 @@ echo "LIBCLANG_PATH=$LIBCLANG_PATH"
 
 bash ./build.sh rust
 
+sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
+
 # Also test out that we can publish cuvs-sys via a dry-run
 pushd ./rust/cuvs-sys
 cargo publish --dry-run

ci/build_standalone_c.sh

@@ -16,7 +16,26 @@ fi
 dnf install -y \
       patch \
       tar \
-      make
+      unzip \
+      wget
+
+if ! command -V ninja >/dev/null 2>&1; then
+    case "$(uname -m)" in
+        x86_64)
+            wget --no-hsts -q -O /tmp/ninja-linux.zip "https://github.com/ninja-build/ninja/releases/download/v1.13.1/ninja-linux.zip";
+            ;;
+        aarch64)
+            wget --no-hsts -q -O /tmp/ninja-linux.zip "https://github.com/ninja-build/ninja/releases/download/v1.13.1/ninja-linux-aarch64.zip";
+            ;;
+        *)
+            echo "Unrecognized platform '$(uname -m)'" >&2
+            exit 1
+            ;;
+    esac
+    unzip -d /usr/bin /tmp/ninja-linux.zip
+    chmod +x /usr/bin/ninja
+    rm /tmp/ninja-linux.zip
+fi
 
 # Fetch and install CMake.
 if [ ! -e "/usr/local/bin/cmake" ]; then
@@ -29,15 +48,13 @@ if [ ! -e "/usr/local/bin/cmake" ]; then
 fi
 
 source rapids-configure-sccache
-
 source rapids-date-string
 
 rapids-print-env
 
 rapids-logger "Begin cpp build"
 
-sccache --zero-stats
-
+sccache --stop-server 2>/dev/null || true
 
 RAPIDS_PACKAGE_VERSION=$(rapids-generate-version)
 export RAPIDS_PACKAGE_VERSION
@@ -47,7 +64,7 @@ mkdir -p "${RAPIDS_ARTIFACTS_DIR}"
 export RAPIDS_ARTIFACTS_DIR
 
 scl enable gcc-toolset-${TOOLSET_VERSION} -- \
-      cmake -S cpp -B cpp/build/ \
+      cmake -S cpp -B cpp/build/ -GNinja \
             -DCMAKE_CUDA_HOST_COMPILER=/opt/rh/gcc-toolset-${TOOLSET_VERSION}/root/usr/bin/gcc \
             -DCMAKE_CUDA_ARCHITECTURES=RAPIDS \
             -DBUILD_SHARED_LIBS=OFF \
@@ -58,16 +75,22 @@ scl enable gcc-toolset-${TOOLSET_VERSION} -- \
             -DCUVS_STATIC_RAPIDS_LIBRARIES=ON
 cmake --build cpp/build "-j${PARALLEL_LEVEL}"
 
+sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
+
 rapids-logger "Begin c build"
 
 scl enable gcc-toolset-${TOOLSET_VERSION} -- \
-      cmake -S c -B c/build \
+      cmake -S c -B c/build -GNinja \
             -DCMAKE_CUDA_HOST_COMPILER=/opt/rh/gcc-toolset-${TOOLSET_VERSION}/root/usr/bin/gcc \
             -DCUVSC_STATIC_CUVS_LIBRARY=ON \
             -DCMAKE_PREFIX_PATH="$PWD/cpp/build/" \
             -DBUILD_TESTS=${BUILD_C_LIB_TESTS}
 cmake --build c/build "-j${PARALLEL_LEVEL}"
 
+sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
+
 rapids-logger "Begin c install"
 cmake --install c/build --prefix c/build/install
 
@@ -86,5 +109,3 @@ fi
 rapids-logger "Begin c tarball creation"
 tar czf libcuvs_c.tar.gz -C c/build/install/ .
 ls -lh libcuvs_c.tar.gz
-
-sccache --show-adv-stats

ci/build_wheel.sh

@@ -11,6 +11,9 @@ source rapids-configure-sccache
 source rapids-date-string
 source rapids-init-pip
 
+export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache"
+export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
+
 rapids-generate-version > ./VERSION
 
 cd "${package_dir}"
@@ -38,9 +41,11 @@ fi
 SKBUILD_CMAKE_ARGS="-DUSE_NCCL_RUNTIME_WHEEL=ON"
 export SKBUILD_CMAKE_ARGS
 
+rapids-print-env
+
 rapids-logger "Building '${package_name}' wheel"
 
-sccache --zero-stats
+sccache --stop-server 2>/dev/null || true
 
 rapids-pip-retry wheel \
     -w dist \
@@ -50,6 +55,7 @@ rapids-pip-retry wheel \
     .
 
 sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
 
 # repair wheels and write to the location that artifact-uploading code expects to find them
 python -m auditwheel repair -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" "${EXCLUDE_ARGS[@]}" dist/*