Skip to content

Commit 135ad88

Browse files
GPUtesterGPUtester
authored
Merge pull request #875 from rapidsai/release/25.12
Forward-merge release/25.12 into main
2 parents 13accea + 143bc11 commit 135ad88

File tree

10 files changed

+72
-17
lines changed

10 files changed

+72
-17
lines changed

‎.devcontainer/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ ENV HISTFILE="/home/coder/.cache/._bash_history"
3434
ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
3535
ENV SCCACHE_REGION="us-east-2"
3636
ENV SCCACHE_BUCKET="rapids-sccache-devs"
37+
ENV SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
3738
ENV SCCACHE_IDLE_TIMEOUT=0
3839

3940
###

‎.github/workflows/build.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,11 @@ jobs:
3838
with:
3939
build_type: ${{ inputs.build_type || 'branch' }}
4040
branch: ${{ inputs.branch }}
41+
node_type: cpu8
4142
date: ${{ inputs.date }}
4243
script: ci/build_cpp.sh
4344
sha: ${{ inputs.sha }}
45+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4446
python-build:
4547
needs: [cpp-build]
4648
secrets: inherit
@@ -51,6 +53,7 @@ jobs:
5153
date: ${{ inputs.date }}
5254
script: ci/build_python.sh
5355
sha: ${{ inputs.sha }}
56+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
5457
upload-conda:
5558
needs: [cpp-build, python-build]
5659
secrets: inherit
@@ -83,9 +86,11 @@ jobs:
8386
branch: ${{ inputs.branch }}
8487
sha: ${{ inputs.sha }}
8588
date: ${{ inputs.date }}
89+
node_type: cpu8
8690
script: ci/build_wheel_cpp.sh
8791
package-name: libkvikio
8892
package-type: cpp
93+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
8994
wheel-build-python:
9095
needs: wheel-build-cpp
9196
secrets: inherit
@@ -95,8 +100,10 @@ jobs:
95100
branch: ${{ inputs.branch }}
96101
sha: ${{ inputs.sha }}
97102
date: ${{ inputs.date }}
103+
node_type: cpu8
98104
script: ci/build_wheel_python.sh
99105
package-name: kvikio
106+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
100107
package-type: python
101108
wheel-publish-cpp:
102109
needs: wheel-build-cpp

‎.github/workflows/pr.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ jobs:
9595
uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main
9696
with:
9797
build_type: pull-request
98+
node_type: cpu8
9899
script: ci/build_cpp.sh
100+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
99101
conda-cpp-tests:
100102
needs: [conda-cpp-build, changed-files]
101103
secrets: inherit
@@ -104,6 +106,7 @@ jobs:
104106
with:
105107
build_type: pull-request
106108
script: ci/test_cpp.sh
109+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
107110
conda-java-tests:
108111
needs: conda-cpp-build
109112
secrets: inherit
@@ -121,6 +124,7 @@ jobs:
121124
with:
122125
build_type: pull-request
123126
script: ci/build_python.sh
127+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
124128
conda-python-tests:
125129
needs: [conda-python-build, changed-files]
126130
secrets: inherit
@@ -129,6 +133,7 @@ jobs:
129133
with:
130134
build_type: pull-request
131135
script: ci/test_python.sh
136+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
132137
docs-build:
133138
needs: conda-python-build
134139
secrets: inherit
@@ -164,18 +169,22 @@ jobs:
164169
with:
165170
matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
166171
build_type: pull-request
172+
node_type: cpu8
167173
script: ci/build_wheel_cpp.sh
168174
package-name: libkvikio
169175
package-type: cpp
176+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
170177
wheel-python-build:
171178
needs: wheel-cpp-build
172179
secrets: inherit
173180
uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
174181
with:
175182
build_type: pull-request
183+
node_type: cpu8
176184
script: ci/build_wheel_python.sh
177185
package-name: kvikio
178186
package-type: python
187+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
179188
wheel-python-tests:
180189
needs: [wheel-python-build, changed-files]
181190
secrets: inherit
@@ -184,6 +193,7 @@ jobs:
184193
with:
185194
build_type: pull-request
186195
script: ci/test_wheel.sh
196+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
187197
telemetry-summarize:
188198
# This job must use a self-hosted runner to record telemetry traces.
189199
runs-on: linux-amd64-cpu4

‎.github/workflows/test.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
date: ${{ inputs.date }}
3333
script: ci/test_cpp.sh
3434
sha: ${{ inputs.sha }}
35+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
3536
python-tests:
3637
secrets: inherit
3738
uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main
@@ -41,6 +42,7 @@ jobs:
4142
date: ${{ inputs.date }}
4243
script: ci/test_python.sh
4344
sha: ${{ inputs.sha }}
45+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4446
conda-java-tests:
4547
secrets: inherit
4648
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main

‎ci/build_cpp.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
set -euo pipefail
66

77
source rapids-configure-sccache
8-
98
source rapids-date-string
109

1110
export CMAKE_GENERATOR=Ninja
@@ -14,7 +13,7 @@ rapids-print-env
1413

1514
rapids-logger "Begin cpp build"
1615

17-
sccache --zero-stats
16+
sccache --stop-server 2>/dev/null || true
1817

1918
RAPIDS_PACKAGE_VERSION=$(rapids-generate-version)
2019
export RAPIDS_PACKAGE_VERSION
@@ -66,6 +65,7 @@ rattler-build build --recipe conda/recipes/libkvikio \
6665
"${RATTLER_CHANNELS[@]}"
6766

6867
sccache --show-adv-stats
68+
sccache --stop-server >/dev/null 2>&1 || true
6969

7070
# remove build_cache directory to avoid uploading the entire source tree
7171
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424

‎ci/build_python.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
set -euo pipefail
66

77
source rapids-configure-sccache
8-
98
source rapids-date-string
109

1110
export CMAKE_GENERATOR=Ninja
@@ -26,7 +25,7 @@ rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS"
2625

2726
RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}")
2827

29-
sccache --zero-stats
28+
sccache --stop-server 2>/dev/null || true
3029

3130
rapids-logger "Building kvikio"
3231

@@ -38,6 +37,7 @@ rattler-build build --recipe conda/recipes/kvikio \
3837
"${RATTLER_CHANNELS[@]}"
3938

4039
sccache --show-adv-stats
40+
sccache --stop-server >/dev/null 2>&1 || true
4141

4242
# remove build_cache directory to avoid uploading the entire source tree
4343
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424

‎ci/build_wheel.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ source rapids-configure-sccache
1111
source rapids-date-string
1212
source rapids-init-pip
1313

14+
export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache"
15+
export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
16+
1417
rapids-generate-version > ./VERSION
1518

1619
cd "${package_dir}"
1720

18-
sccache --zero-stats
21+
sccache --stop-server 2>/dev/null || true
1922

2023
rapids-logger "Building '${package_name}' wheel"
2124
rapids-pip-retry wheel \
@@ -26,3 +29,4 @@ rapids-pip-retry wheel \
2629
.
2730

2831
sccache --show-adv-stats
32+
sccache --stop-server >/dev/null 2>&1 || true

‎cmake/rapids_config.cmake‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,6 @@ if(NOT rapids-cmake-branch)
3535
set(rapids-cmake-branch "${_rapids_branch}")
3636
endif()
3737
include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake")
38+
39+
# Don't use sccache-dist for CMake's compiler tests
40+
set(ENV{SCCACHE_NO_DIST_COMPILE} "1")

‎conda/recipes/kvikio/recipe.yaml‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,32 @@ build:
4141
- AWS_ACCESS_KEY_ID
4242
- AWS_SECRET_ACCESS_KEY
4343
- AWS_SESSION_TOKEN
44+
- SCCACHE_DIST_AUTH_TOKEN
4445
env:
4546
CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
4647
CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }}
4748
CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
4849
CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
49-
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
50-
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
51-
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
52-
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
53-
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
50+
NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
51+
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
52+
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }}
53+
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
54+
SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
55+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
56+
SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }}
57+
SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }}
58+
SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }}
59+
SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }}
60+
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }}
61+
SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }}
62+
SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }}
63+
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
5464
SCCACHE_S3_KEY_PREFIX: kvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}
65+
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
66+
SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: kvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
67+
SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
68+
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
69+
SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
5570

5671
requirements:
5772
build:

‎conda/recipes/libkvikio/recipe.yaml‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,19 +42,32 @@ cache:
4242
- AWS_ACCESS_KEY_ID
4343
- AWS_SECRET_ACCESS_KEY
4444
- AWS_SESSION_TOKEN
45+
- SCCACHE_DIST_AUTH_TOKEN
4546
env:
4647
CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
4748
CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }}
4849
CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
4950
CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
50-
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }}
51-
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }}
52-
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
53-
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
54-
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
55-
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
56-
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
51+
NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
52+
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
53+
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }}
54+
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
55+
SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
56+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
57+
SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }}
58+
SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }}
59+
SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }}
60+
SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }}
61+
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }}
62+
SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }}
63+
SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }}
64+
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
5765
SCCACHE_S3_KEY_PREFIX: libkvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_key_string }}
66+
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
67+
SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libkvikio/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
68+
SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
69+
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
70+
SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
5871
requirements:
5972
build:
6073
- ${{ compiler("c") }}

0 commit comments

Comments
 (0)