Skip to content

Commit 143bc11

Browse files
trxcllnttrxcllnt
authored
Use sccache-dist build cluster for conda and wheel builds (#865)
RAPIDS has deployed an autoscaling cloud build cluster that can be used to accelerate building large RAPIDS projects. This PR updates the conda and wheel builds to use the build cluster. This contributes to rapidsai/build-planning#228. Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Nate Rock (https://github.com/rockhowse) - Bradley Dice (https://github.com/bdice) URL: #865
1 parent c9fb696 commit 143bc11

File tree

10 files changed

+72
-17
lines changed

10 files changed

+72
-17
lines changed

‎.devcontainer/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ ENV HISTFILE="/home/coder/.cache/._bash_history"
3434
ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
3535
ENV SCCACHE_REGION="us-east-2"
3636
ENV SCCACHE_BUCKET="rapids-sccache-devs"
37+
ENV SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
3738
ENV SCCACHE_IDLE_TIMEOUT=0
3839

3940
###

‎.github/workflows/build.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,11 @@ jobs:
3838
with:
3939
build_type: ${{ inputs.build_type || 'branch' }}
4040
branch: ${{ inputs.branch }}
41+
node_type: cpu8
4142
date: ${{ inputs.date }}
4243
script: ci/build_cpp.sh
4344
sha: ${{ inputs.sha }}
45+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4446
python-build:
4547
needs: [cpp-build]
4648
secrets: inherit
@@ -51,6 +53,7 @@ jobs:
5153
date: ${{ inputs.date }}
5254
script: ci/build_python.sh
5355
sha: ${{ inputs.sha }}
56+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
5457
upload-conda:
5558
needs: [cpp-build, python-build]
5659
secrets: inherit
@@ -83,9 +86,11 @@ jobs:
8386
branch: ${{ inputs.branch }}
8487
sha: ${{ inputs.sha }}
8588
date: ${{ inputs.date }}
89+
node_type: cpu8
8690
script: ci/build_wheel_cpp.sh
8791
package-name: libkvikio
8892
package-type: cpp
93+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
8994
wheel-build-python:
9095
needs: wheel-build-cpp
9196
secrets: inherit
@@ -95,8 +100,10 @@ jobs:
95100
branch: ${{ inputs.branch }}
96101
sha: ${{ inputs.sha }}
97102
date: ${{ inputs.date }}
103+
node_type: cpu8
98104
script: ci/build_wheel_python.sh
99105
package-name: kvikio
106+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
100107
package-type: python
101108
wheel-publish-cpp:
102109
needs: wheel-build-cpp

‎.github/workflows/pr.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ jobs:
9595
uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@release/25.12
9696
with:
9797
build_type: pull-request
98+
node_type: cpu8
9899
script: ci/build_cpp.sh
100+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
99101
conda-cpp-tests:
100102
needs: [conda-cpp-build, changed-files]
101103
secrets: inherit
@@ -104,6 +106,7 @@ jobs:
104106
with:
105107
build_type: pull-request
106108
script: ci/test_cpp.sh
109+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
107110
conda-java-tests:
108111
needs: conda-cpp-build
109112
secrets: inherit
@@ -121,6 +124,7 @@ jobs:
121124
with:
122125
build_type: pull-request
123126
script: ci/build_python.sh
127+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
124128
conda-python-tests:
125129
needs: [conda-python-build, changed-files]
126130
secrets: inherit
@@ -129,6 +133,7 @@ jobs:
129133
with:
130134
build_type: pull-request
131135
script: ci/test_python.sh
136+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
132137
docs-build:
133138
needs: conda-python-build
134139
secrets: inherit
@@ -164,18 +169,22 @@ jobs:
164169
with:
165170
matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
166171
build_type: pull-request
172+
node_type: cpu8
167173
script: ci/build_wheel_cpp.sh
168174
package-name: libkvikio
169175
package-type: cpp
176+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
170177
wheel-python-build:
171178
needs: wheel-cpp-build
172179
secrets: inherit
173180
uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/25.12
174181
with:
175182
build_type: pull-request
183+
node_type: cpu8
176184
script: ci/build_wheel_python.sh
177185
package-name: kvikio
178186
package-type: python
187+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
179188
wheel-python-tests:
180189
needs: [wheel-python-build, changed-files]
181190
secrets: inherit
@@ -184,6 +193,7 @@ jobs:
184193
with:
185194
build_type: pull-request
186195
script: ci/test_wheel.sh
196+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
187197
telemetry-summarize:
188198
# This job must use a self-hosted runner to record telemetry traces.
189199
runs-on: linux-amd64-cpu4

‎.github/workflows/test.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
date: ${{ inputs.date }}
3333
script: ci/test_cpp.sh
3434
sha: ${{ inputs.sha }}
35+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
3536
python-tests:
3637
secrets: inherit
3738
uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@release/25.12
@@ -41,6 +42,7 @@ jobs:
4142
date: ${{ inputs.date }}
4243
script: ci/test_python.sh
4344
sha: ${{ inputs.sha }}
45+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4446
conda-java-tests:
4547
secrets: inherit
4648
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/25.12

‎ci/build_cpp.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
set -euo pipefail
66

77
source rapids-configure-sccache
8-
98
source rapids-date-string
109

1110
export CMAKE_GENERATOR=Ninja
@@ -14,7 +13,7 @@ rapids-print-env
1413

1514
rapids-logger "Begin cpp build"
1615

17-
sccache --zero-stats
16+
sccache --stop-server 2>/dev/null || true
1817

1918
RAPIDS_PACKAGE_VERSION=$(rapids-generate-version)
2019
export RAPIDS_PACKAGE_VERSION
@@ -66,6 +65,7 @@ rattler-build build --recipe conda/recipes/libkvikio \
6665
"${RATTLER_CHANNELS[@]}"
6766

6867
sccache --show-adv-stats
68+
sccache --stop-server >/dev/null 2>&1 || true
6969

7070
# remove build_cache directory to avoid uploading the entire source tree
7171
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424

‎ci/build_python.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
set -euo pipefail
66

77
source rapids-configure-sccache
8-
98
source rapids-date-string
109

1110
export CMAKE_GENERATOR=Ninja
@@ -26,7 +25,7 @@ rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS"
2625

2726
RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}")
2827

29-
sccache --zero-stats
28+
sccache --stop-server 2>/dev/null || true
3029

3130
rapids-logger "Building kvikio"
3231

@@ -38,6 +37,7 @@ rattler-build build --recipe conda/recipes/kvikio \
3837
"${RATTLER_CHANNELS[@]}"
3938

4039
sccache --show-adv-stats
40+
sccache --stop-server >/dev/null 2>&1 || true
4141

4242
# remove build_cache directory to avoid uploading the entire source tree
4343
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424

‎ci/build_wheel.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ source rapids-configure-sccache
1111
source rapids-date-string
1212
source rapids-init-pip
1313

14+
export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache"
15+
export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
16+
1417
rapids-generate-version > ./VERSION
1518

1619
cd "${package_dir}"
1720

18-
sccache --zero-stats
21+
sccache --stop-server 2>/dev/null || true
1922

2023
rapids-logger "Building '${package_name}' wheel"
2124
rapids-pip-retry wheel \
@@ -26,3 +29,4 @@ rapids-pip-retry wheel \
2629
.
2730

2831
sccache --show-adv-stats
32+
sccache --stop-server >/dev/null 2>&1 || true

‎cmake/rapids_config.cmake‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,6 @@ if(NOT rapids-cmake-branch)
3535
set(rapids-cmake-branch "${_rapids_branch}")
3636
endif()
3737
include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake")
38+
39+
# Don't use sccache-dist for CMake's compiler tests
40+
set(ENV{SCCACHE_NO_DIST_COMPILE} "1")

‎conda/recipes/kvikio/recipe.yaml‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,32 @@ build:
4141
- AWS_ACCESS_KEY_ID
4242
- AWS_SECRET_ACCESS_KEY
4343
- AWS_SESSION_TOKEN
44+
- SCCACHE_DIST_AUTH_TOKEN
4445
env:
4546
CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
4647
CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }}
4748
CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
4849
CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
49-
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
50-
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
51-
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
52-
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
53-
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
50+
NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
51+
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
52+
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }}
53+
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
54+
SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
55+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
56+
SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }}
57+
SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }}
58+
SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }}
59+
SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }}
60+
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }}
61+
SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }}
62+
SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }}
63+
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
5464
SCCACHE_S3_KEY_PREFIX: kvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}
65+
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
66+
SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: kvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
67+
SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
68+
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
69+
SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
5570

5671
requirements:
5772
build:

‎conda/recipes/libkvikio/recipe.yaml‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,19 +42,32 @@ cache:
4242
- AWS_ACCESS_KEY_ID
4343
- AWS_SECRET_ACCESS_KEY
4444
- AWS_SESSION_TOKEN
45+
- SCCACHE_DIST_AUTH_TOKEN
4546
env:
4647
CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
4748
CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }}
4849
CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
4950
CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
50-
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }}
51-
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }}
52-
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
53-
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
54-
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
55-
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
56-
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
51+
NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
52+
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
53+
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }}
54+
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
55+
SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
56+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
57+
SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }}
58+
SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }}
59+
SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }}
60+
SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }}
61+
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }}
62+
SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }}
63+
SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }}
64+
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
5765
SCCACHE_S3_KEY_PREFIX: libkvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_key_string }}
66+
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
67+
SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libkvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
68+
SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
69+
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
70+
SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
5871
requirements:
5972
build:
6073
- ${{ compiler("c") }}

0 commit comments

Comments
 (0)