From 96149968fb3b9b62acad90c07b50449a94ff4b4b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 4 Nov 2025 17:40:56 +0000 Subject: [PATCH 01/19] Update conda and wheel builds to use `sccache-dist` --- .github/workflows/build.yaml | 9 +++++++++ .github/workflows/pr.yaml | 13 +++++++++++++ .github/workflows/test.yaml | 4 ++++ cmake/rapids_config.cmake | 3 +++ conda/recipes/libwholegraph/recipe.yaml | 1 + conda/recipes/pylibwholegraph/recipe.yaml | 1 + 6 files changed, 31 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b3e56709..ad9c52c9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -37,10 +37,12 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} + node_type: cpu8 branch: ${{ inputs.branch }} date: ${{ inputs.date }} script: ci/build_cpp.sh sha: ${{ inputs.sha }} + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN python-build: needs: [cpp-build] secrets: inherit @@ -51,6 +53,7 @@ jobs: date: ${{ inputs.date }} script: ci/build_python.sh sha: ${{ inputs.sha }} + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN docs-build: needs: cpp-build secrets: inherit @@ -79,6 +82,7 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: ${{ inputs.build_type || 'branch' }} + node_type: cpu8 branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} @@ -86,6 +90,7 @@ jobs: package-name: cugraph-pyg package-type: python pure-wheel: true + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-publish-cugraph-pyg: needs: wheel-build-cugraph-pyg secrets: inherit @@ -102,6 +107,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} + node_type: cpu8 branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} @@ -110,18 +116,21 @@ jobs: package-type: cpp # build for every combination of arch and CUDA version, but only for the latest Python matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-build-pylibwholegraph: needs: wheel-build-libwholegraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} + node_type: cpu8 branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-publish-libwholegraph: needs: wheel-build-libwholegraph secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 91486e7f..4b9d12eb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,7 +107,9 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: pull-request + node_type: cpu8 script: ci/build_cpp.sh + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit @@ -116,6 +118,7 @@ jobs: with: build_type: pull-request script: ci/test_cpp.sh + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-python-build: needs: conda-cpp-build secrets: inherit @@ -123,6 +126,7 @@ jobs: with: build_type: pull-request script: ci/build_python.sh + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-notebook-tests: needs: [conda-python-build, changed-files] secrets: inherit @@ -145,6 +149,7 @@ jobs: # ref: https://github.com/pytorch/pytorch/issues/159779 matrix_filter: map(select(.ARCH == "amd64" and (.CUDA_VER | startswith("12")))) script: ci/test_python.sh + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN docs-build: needs: conda-cpp-build secrets: inherit @@ -161,19 +166,23 @@ jobs: with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh + node_type: cpu8 package-name: libwholegraph package-type: cpp # build for every combination of arch and CUDA version, but only for the latest Python matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-build-pylibwholegraph: needs: [checks, wheel-build-libwholegraph] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: pull-request + node_type: cpu8 script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-pylibwholegraph: needs: [wheel-build-pylibwholegraph, changed-files] secrets: inherit @@ -183,6 +192,7 @@ jobs: build_type: pull-request script: ci/test_wheel_pylibwholegraph.sh matrix_filter: map(select(.ARCH == "amd64")) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-build-cugraph-pyg: needs: checks secrets: inherit @@ -191,10 +201,12 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request + node_type: cpu8 script: ci/build_wheel_cugraph-pyg.sh package-name: cugraph-pyg package-type: python pure-wheel: true + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-cugraph-pyg: needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files] secrets: inherit @@ -204,3 +216,4 @@ jobs: build_type: pull-request script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d895b577..f0278471 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,6 +32,7 @@ jobs: date: ${{ inputs.date }} script: ci/test_cpp.sh sha: ${{ inputs.sha }} + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-notebook-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main @@ -56,6 +57,7 @@ jobs: # TODO: remove the CUDA 13 exclusion here once there are pytorch CUDA 13 packages # ref: https://github.com/pytorch/pytorch/issues/159779 matrix_filter: map(select(.ARCH == "amd64" and (.CUDA_VER | startswith("12")))) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-pylibwholegraph: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main @@ -66,6 +68,7 @@ jobs: sha: ${{ inputs.sha }} script: ci/test_wheel_pylibwholegraph.sh matrix_filter: map(select(.ARCH == "amd64")) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-cugraph-pyg: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main @@ -76,3 +79,4 @@ jobs: sha: ${{ inputs.sha }} script: ci/test_wheel_cugraph-pyg.sh matrix_filter: map(select(.ARCH == "amd64")) + sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake index 768e2487..9cda1f26 100644 --- a/cmake/rapids_config.cmake +++ b/cmake/rapids_config.cmake @@ -35,3 +35,6 @@ if(NOT rapids-cmake-branch) set(rapids-cmake-branch "${_rapids_branch}") endif() include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake") + +# Don't use sccache-dist for CMake's compiler tests +set(ENV{SCCACHE_NO_DIST_COMPILE} "1") diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index d0ca3b7b..ba02ed95 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -45,6 +45,7 @@ cache: SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS") }} requirements: build: diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index aaff2f6d..7f236e64 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -39,6 +39,7 @@ build: SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS") }} requirements: build: From d00f5f60f1d452eaff63662abb1a931ae1f8d5de Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 4 Nov 2025 18:00:39 +0000 Subject: [PATCH 02/19] use default value for NVCC_APPEND_FLAGS if unset --- conda/recipes/libwholegraph/recipe.yaml | 2 +- conda/recipes/pylibwholegraph/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index ba02ed95..71f72ac0 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -45,7 +45,7 @@ cache: SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} - NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS") }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} requirements: build: diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index 7f236e64..73cc3ad8 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -39,7 +39,7 @@ build: SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} - NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS") }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} requirements: build: From cd389f97be5cd353a2e8b306a77c8b59c4d3675d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 6 Nov 2025 04:08:59 +0000 Subject: [PATCH 03/19] disable build cache for stress tests --- ci/build_cpp.sh | 2 ++ ci/build_python.sh | 2 ++ ci/build_wheel.sh | 2 ++ conda/recipes/libwholegraph/recipe.yaml | 3 ++- conda/recipes/pylibwholegraph/recipe.yaml | 2 ++ 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 4f739b0f..81e5a5c6 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -5,6 +5,8 @@ set -euo pipefail source rapids-configure-sccache +export SCCACHE_NO_CACHE=1 +sccache --stop-server source rapids-date-string diff --git a/ci/build_python.sh b/ci/build_python.sh index 320b0c86..ad560de3 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -5,6 +5,8 @@ set -euo pipefail source rapids-configure-sccache +export SCCACHE_NO_CACHE=1 +sccache --stop-server source rapids-date-string diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9531de3e..af30e1da 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -28,6 +28,8 @@ if [[ "${package_name}" != "libwholegraph" ]]; then fi source rapids-configure-sccache +export SCCACHE_NO_CACHE=1 +sccache --stop-server source rapids-date-string rapids-generate-version > ./VERSION diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index 71f72ac0..52612432 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -38,7 +38,7 @@ cache: CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} @@ -46,6 +46,7 @@ cache: SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} + SCCACHE_NO_CACHE: 1 requirements: build: diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index 73cc3ad8..83135fc9 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -33,6 +33,7 @@ build: CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} @@ -40,6 +41,7 @@ build: SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} + SCCACHE_NO_CACHE: 1 requirements: build: From 811f5f2ca7aa7ab871d332dc15765b81ae1c8e03 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 7 Nov 2025 05:45:42 +0000 Subject: [PATCH 04/19] don't stop the server --- ci/build_cpp.sh | 1 - ci/build_python.sh | 1 - ci/build_wheel.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 81e5a5c6..f56dd9d7 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -6,7 +6,6 @@ set -euo pipefail source rapids-configure-sccache export SCCACHE_NO_CACHE=1 -sccache --stop-server source rapids-date-string diff --git a/ci/build_python.sh b/ci/build_python.sh index ad560de3..698bbbeb 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -6,7 +6,6 @@ set -euo pipefail source rapids-configure-sccache export SCCACHE_NO_CACHE=1 -sccache --stop-server source rapids-date-string diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index af30e1da..de094f05 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -29,7 +29,6 @@ fi source rapids-configure-sccache export SCCACHE_NO_CACHE=1 -sccache --stop-server source rapids-date-string rapids-generate-version > ./VERSION From baf64a3e4d1dfb2a62d1f8f37944d54118093524 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 7 Nov 2025 21:25:50 +0000 Subject: [PATCH 05/19] use cpu16 instances for C++ builds --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ad9c52c9..3adbb84e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -37,9 +37,9 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} - node_type: cpu8 branch: ${{ inputs.branch }} date: ${{ inputs.date }} + node_type: cpu16 script: ci/build_cpp.sh sha: ${{ inputs.sha }} sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN @@ -82,10 +82,10 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: ${{ inputs.build_type || 'branch' }} - node_type: cpu8 branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} + node_type: cpu8 script: ci/build_wheel_cugraph-pyg.sh package-name: cugraph-pyg package-type: python @@ -107,10 +107,10 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} - node_type: cpu8 branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} + node_type: cpu16 script: ci/build_wheel_libwholegraph.sh package-name: libwholegraph package-type: cpp @@ -123,10 +123,10 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} - node_type: cpu8 branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} + node_type: cpu8 script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4b9d12eb..a2ecac9d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,7 +107,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: pull-request - node_type: cpu8 + node_type: cpu16 script: ci/build_cpp.sh sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-cpp-tests: @@ -166,7 +166,7 @@ jobs: with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh - node_type: cpu8 + node_type: cpu16 package-name: libwholegraph package-type: cpp # build for every combination of arch and CUDA version, but only for the latest Python From aebb26ceed97f14069c44d9b5e94456459e1685a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Nov 2025 22:42:14 +0000 Subject: [PATCH 06/19] remove SCCACHE_NO_CACHE=1 --- ci/build_cpp.sh | 1 - ci/build_python.sh | 1 - ci/build_wheel.sh | 1 - conda/recipes/libwholegraph/recipe.yaml | 1 - conda/recipes/pylibwholegraph/recipe.yaml | 1 - 5 files changed, 5 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index f56dd9d7..4f739b0f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -5,7 +5,6 @@ set -euo pipefail source rapids-configure-sccache -export SCCACHE_NO_CACHE=1 source rapids-date-string diff --git a/ci/build_python.sh b/ci/build_python.sh index 698bbbeb..320b0c86 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -5,7 +5,6 @@ set -euo pipefail source rapids-configure-sccache -export SCCACHE_NO_CACHE=1 source rapids-date-string diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index de094f05..9531de3e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -28,7 +28,6 @@ if [[ "${package_name}" != "libwholegraph" ]]; then fi source rapids-configure-sccache -export SCCACHE_NO_CACHE=1 source rapids-date-string rapids-generate-version > ./VERSION diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index 52612432..6a08c3c3 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -46,7 +46,6 @@ cache: SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} - SCCACHE_NO_CACHE: 1 requirements: build: diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index 83135fc9..f15dc9d1 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -41,7 +41,6 @@ build: SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} - SCCACHE_NO_CACHE: 1 requirements: build: From c77a936bc829349038cc9433efc2f5c9748747a0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Nov 2025 22:46:44 +0000 Subject: [PATCH 07/19] use cpu8 GHA runners --- .github/workflows/build.yaml | 4 ++-- .github/workflows/pr.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3adbb84e..ae0405e2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,7 @@ jobs: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} - node_type: cpu16 + node_type: cpu8 script: ci/build_cpp.sh sha: ${{ inputs.sha }} sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN @@ -110,7 +110,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu16 + node_type: cpu8 script: ci/build_wheel_libwholegraph.sh package-name: libwholegraph package-type: cpp diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a2ecac9d..4b9d12eb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,7 +107,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: pull-request - node_type: cpu16 + node_type: cpu8 script: ci/build_cpp.sh sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-cpp-tests: @@ -166,7 +166,7 @@ jobs: with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh - node_type: cpu16 + node_type: cpu8 package-name: libwholegraph package-type: cpp # build for every combination of arch and CUDA version, but only for the latest Python From 8d00dff489f06e0910c608d27339091cd1af9425 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 11 Nov 2025 02:18:10 +0000 Subject: [PATCH 08/19] lazily start the sccache daemon --- .github/workflows/build.yaml | 12 ++++++------ .github/workflows/pr.yaml | 22 +++++++++++----------- .github/workflows/test.yaml | 10 +++++----- conda/recipes/libwholegraph/recipe.yaml | 21 +++++++++++++++------ conda/recipes/pylibwholegraph/recipe.yaml | 21 +++++++++++++++------ 5 files changed, 52 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ae0405e2..8759086d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: docs-build: needs: cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs with: arch: "amd64" branch: ${{ inputs.branch }} @@ -77,7 +77,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -104,7 +104,7 @@ jobs: package-type: python wheel-build-libwholegraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -120,7 +120,7 @@ jobs: wheel-build-pylibwholegraph: needs: wheel-build-libwholegraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4b9d12eb..ef31d123 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -104,7 +104,7 @@ jobs: conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request node_type: cpu8 @@ -113,7 +113,7 @@ jobs: conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@fea/sccache-dist-in-custom-jobs if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request @@ -122,7 +122,7 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request script: ci/build_python.sh @@ -130,7 +130,7 @@ jobs: conda-notebook-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks with: build_type: pull-request @@ -141,7 +141,7 @@ jobs: conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@fea/sccache-dist-in-custom-jobs if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request @@ -153,7 +153,7 @@ jobs: docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs with: arch: "amd64" build_type: pull-request @@ -162,7 +162,7 @@ jobs: wheel-build-libwholegraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh @@ -175,7 +175,7 @@ jobs: wheel-build-pylibwholegraph: needs: [checks, wheel-build-libwholegraph] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request node_type: cpu8 @@ -186,7 +186,7 @@ jobs: wheel-tests-pylibwholegraph: needs: [wheel-build-pylibwholegraph, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request @@ -196,7 +196,7 @@ jobs: wheel-build-cugraph-pyg: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -210,7 +210,7 @@ jobs: wheel-tests-cugraph-pyg: needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f0278471..6a833802 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -35,7 +35,7 @@ jobs: sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: script: "ci/test_notebooks.sh" conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-pylibwholegraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -71,7 +71,7 @@ jobs: sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index 6a08c3c3..1a509733 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -38,14 +38,23 @@ cache: CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_DIST_AUTH_TOKEN: ${{ env.get("SCCACHE_DIST_AUTH_TOKEN", default="") }} + SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} + SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} + SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} + SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }} + SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }} + SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} - NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }} + SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }} requirements: build: diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index f15dc9d1..7f56db58 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -33,14 +33,23 @@ build: CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} + SCCACHE_DIST_AUTH_TOKEN: ${{ env.get("SCCACHE_DIST_AUTH_TOKEN", default="") }} + SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} + SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} + SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} + SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }} + SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }} + SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} - NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }} + SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }} requirements: build: From db3539a4929160cddbaf2b6472831fe4e1fb97cf Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 11 Nov 2025 02:30:20 +0000 Subject: [PATCH 09/19] stop the sccache daemon instead of zeroing so that the first invocation in the conda build starts it --- ci/build_cpp.sh | 2 +- ci/build_python.sh | 6 +++--- ci/build_wheel.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 4f739b0f..ed77619e 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -14,7 +14,7 @@ rapids-print-env rapids-logger "Begin cpp build" -sccache --zero-stats +sccache --stop-server >/dev/null 2>&1 || true RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) export RAPIDS_PACKAGE_VERSION diff --git a/ci/build_python.sh b/ci/build_python.sh index 320b0c86..6da57616 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -27,7 +27,7 @@ rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS" RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") # TODO: Remove `--test skip` flags once importing on a CPU node works correctly -sccache --zero-stats +sccache --stop-server >/dev/null 2>&1 || true rapids-logger "Building pylibwholegraph" @@ -40,7 +40,7 @@ rattler-build build --recipe conda/recipes/pylibwholegraph \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --zero-stats +sccache --stop-server >/dev/null 2>&1 || true rapids-logger "Building cugraph-pyg" @@ -53,7 +53,7 @@ rattler-build build --recipe conda/recipes/cugraph-pyg \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --zero-stats +sccache --stop-server >/dev/null 2>&1 || true # remove build_cache directory to avoid uploading the entire source tree # tracked in https://github.com/prefix-dev/rattler-build/issues/1424 diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9531de3e..601c61cc 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -36,7 +36,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" cd "${package_dir}" -sccache --zero-stats +sccache --stop-server >/dev/null 2>&1 || true rapids-logger "Building '${package_name}' wheel" rapids-pip-retry wheel \ From f41faf3c8008f26686185c086619782e2569b9ce Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 11 Nov 2025 02:50:05 +0000 Subject: [PATCH 10/19] list SCCACHE_DIST_AUTH_TOKEN as a conda build secret --- conda/recipes/libwholegraph/recipe.yaml | 2 +- conda/recipes/pylibwholegraph/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index 1a509733..9d5f0eca 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -33,6 +33,7 @@ cache: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN + - SCCACHE_DIST_AUTH_TOKEN env: CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} @@ -42,7 +43,6 @@ cache: PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} - SCCACHE_DIST_AUTH_TOKEN: ${{ env.get("SCCACHE_DIST_AUTH_TOKEN", default="") }} SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index 7f56db58..e7ef9066 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -28,6 +28,7 @@ build: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN + - SCCACHE_DIST_AUTH_TOKEN env: CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} @@ -37,7 +38,6 @@ build: PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }} RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }} SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }} - SCCACHE_DIST_AUTH_TOKEN: ${{ env.get("SCCACHE_DIST_AUTH_TOKEN", default="") }} SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }} SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }} SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }} From 59696cd38e3dcf65faaa8683341c1efad4365191 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 11 Nov 2025 21:44:42 +0000 Subject: [PATCH 11/19] test cpu8 -> cpu4 --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8759086d..0937882d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,7 @@ jobs: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} - node_type: cpu8 + node_type: cpu4 script: ci/build_cpp.sh sha: ${{ inputs.sha }} sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN @@ -85,7 +85,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu8 + node_type: cpu4 script: ci/build_wheel_cugraph-pyg.sh package-name: cugraph-pyg package-type: python @@ -110,7 +110,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu8 + node_type: cpu4 script: ci/build_wheel_libwholegraph.sh package-name: libwholegraph package-type: cpp @@ -126,7 +126,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu8 + node_type: cpu4 script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef31d123..a054305a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,7 +107,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request - node_type: cpu8 + node_type: cpu4 script: ci/build_cpp.sh sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-cpp-tests: @@ -166,7 +166,7 @@ jobs: with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh - node_type: cpu8 + node_type: cpu4 package-name: libwholegraph package-type: cpp # build for every combination of arch and CUDA version, but only for the latest Python @@ -178,7 +178,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request - node_type: cpu8 + node_type: cpu4 script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python @@ -201,7 +201,7 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request - node_type: cpu8 + node_type: cpu4 script: ci/build_wheel_cugraph-pyg.sh package-name: cugraph-pyg package-type: python From fa100f38e7ab298f43b9771ad657ed1ff065134d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 12 Nov 2025 00:21:31 +0000 Subject: [PATCH 12/19] Revert "test cpu8 -> cpu4" This reverts commit 59696cd38e3dcf65faaa8683341c1efad4365191. --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0937882d..8759086d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,7 @@ jobs: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} - node_type: cpu4 + node_type: cpu8 script: ci/build_cpp.sh sha: ${{ inputs.sha }} sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN @@ -85,7 +85,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu4 + node_type: cpu8 script: ci/build_wheel_cugraph-pyg.sh package-name: cugraph-pyg package-type: python @@ -110,7 +110,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu4 + node_type: cpu8 script: ci/build_wheel_libwholegraph.sh package-name: libwholegraph package-type: cpp @@ -126,7 +126,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - node_type: cpu4 + node_type: cpu8 script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a054305a..ef31d123 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,7 +107,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request - node_type: cpu4 + node_type: cpu8 script: ci/build_cpp.sh sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-cpp-tests: @@ -166,7 +166,7 @@ jobs: with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh - node_type: cpu4 + node_type: cpu8 package-name: libwholegraph package-type: cpp # build for every combination of arch and CUDA version, but only for the latest Python @@ -178,7 +178,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs with: build_type: pull-request - node_type: cpu4 + node_type: cpu8 script: ci/build_wheel_pylibwholegraph.sh package-name: pylibwholegraph package-type: python @@ -201,7 +201,7 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request - node_type: cpu4 + node_type: cpu8 script: ci/build_wheel_cugraph-pyg.sh package-name: cugraph-pyg package-type: python From 0e1ac94e2135076664d43a7f0aa416c4ca5ce547 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Nov 2025 05:44:18 +0000 Subject: [PATCH 13/19] use rapidsai/shared-workflows@main again --- .github/workflows/build.yaml | 12 ++++++------ .github/workflows/pr.yaml | 22 +++++++++++----------- .github/workflows/test.yaml | 10 +++++----- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8759086d..ae0405e2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: docs-build: needs: cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main with: arch: "amd64" branch: ${{ inputs.branch }} @@ -77,7 +77,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -104,7 +104,7 @@ jobs: package-type: python wheel-build-libwholegraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -120,7 +120,7 @@ jobs: wheel-build-pylibwholegraph: needs: wheel-build-libwholegraph secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef31d123..4b9d12eb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -104,7 +104,7 @@ jobs: conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main with: build_type: pull-request node_type: cpu8 @@ -113,7 +113,7 @@ jobs: conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request @@ -122,7 +122,7 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main with: build_type: pull-request script: ci/build_python.sh @@ -130,7 +130,7 @@ jobs: conda-notebook-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks with: build_type: pull-request @@ -141,7 +141,7 @@ jobs: conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request @@ -153,7 +153,7 @@ jobs: docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main with: arch: "amd64" build_type: pull-request @@ -162,7 +162,7 @@ jobs: wheel-build-libwholegraph: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: pull-request script: ci/build_wheel_libwholegraph.sh @@ -175,7 +175,7 @@ jobs: wheel-build-pylibwholegraph: needs: [checks, wheel-build-libwholegraph] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: build_type: pull-request node_type: cpu8 @@ -186,7 +186,7 @@ jobs: wheel-tests-pylibwholegraph: needs: [wheel-build-pylibwholegraph, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request @@ -196,7 +196,7 @@ jobs: wheel-build-cugraph-pyg: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -210,7 +210,7 @@ jobs: wheel-tests-cugraph-pyg: needs: [wheel-build-pylibwholegraph, wheel-build-cugraph-pyg, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6a833802..f0278471 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -35,7 +35,7 @@ jobs: sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -47,7 +47,7 @@ jobs: script: "ci/test_notebooks.sh" conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-pylibwholegraph: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -71,7 +71,7 @@ jobs: sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN wheel-tests-cugraph-pyg: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@fea/sccache-dist-in-custom-jobs + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} From f5e59a6f1dba939e1348dcb2bb730a400d2cabff Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Nov 2025 07:22:49 +0000 Subject: [PATCH 14/19] pass SCCACHE_NO_CACHE and SCCACHE_RECACHE through to conda-build --- conda/recipes/libwholegraph/recipe.yaml | 2 ++ conda/recipes/pylibwholegraph/recipe.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index 9d5f0eca..3a4cc2b0 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -50,6 +50,8 @@ cache: SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }} SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }} SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }} + SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }} + SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }} SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index e7ef9066..e2f3ed6c 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -45,6 +45,8 @@ build: SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }} SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }} SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }} + SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }} + SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }} SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} From cc012b235e1ca19c7732bd47cd0d330604643c4b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Nov 2025 08:04:55 +0000 Subject: [PATCH 15/19] test SCCACHE_RECACHE=1 --- .github/workflows/pr.yaml | 1 + ci/build_cpp.sh | 1 + ci/build_python.sh | 1 + ci/build_wheel.sh | 2 ++ 4 files changed, 5 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4b9d12eb..f0ae8c96 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -88,6 +88,7 @@ jobs: node_type: "cpu8" rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN env: | + SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index ed77619e..311ff2f3 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -5,6 +5,7 @@ set -euo pipefail source rapids-configure-sccache +export SCCACHE_RECACHE=1 source rapids-date-string diff --git a/ci/build_python.sh b/ci/build_python.sh index 6da57616..150a6c75 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -5,6 +5,7 @@ set -euo pipefail source rapids-configure-sccache +export SCCACHE_RECACHE=1 source rapids-date-string diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 601c61cc..fddc387e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -28,6 +28,8 @@ if [[ "${package_name}" != "libwholegraph" ]]; then fi source rapids-configure-sccache +export SCCACHE_RECACHE=1 + source rapids-date-string rapids-generate-version > ./VERSION From 5d5727f4b2b7c9ec913fc67e3dce482ab77ad931 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 15 Nov 2025 01:09:08 +0000 Subject: [PATCH 16/19] remove SCCACHE_RECACHE=1 --- .github/workflows/pr.yaml | 1 - ci/build_cpp.sh | 2 -- ci/build_python.sh | 2 -- ci/build_wheel.sh | 2 -- 4 files changed, 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f0ae8c96..4b9d12eb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -88,7 +88,6 @@ jobs: node_type: "cpu8" rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN env: | - SCCACHE_RECACHE=1 SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 311ff2f3..cceb934f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -5,8 +5,6 @@ set -euo pipefail source rapids-configure-sccache -export SCCACHE_RECACHE=1 - source rapids-date-string export CMAKE_GENERATOR=Ninja diff --git a/ci/build_python.sh b/ci/build_python.sh index 150a6c75..d13efd78 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -5,8 +5,6 @@ set -euo pipefail source rapids-configure-sccache -export SCCACHE_RECACHE=1 - source rapids-date-string export CMAKE_GENERATOR=Ninja diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index fddc387e..601c61cc 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -28,8 +28,6 @@ if [[ "${package_name}" != "libwholegraph" ]]; then fi source rapids-configure-sccache -export SCCACHE_RECACHE=1 - source rapids-date-string rapids-generate-version > ./VERSION From 37b1bf65819761778f6b3fdedb7f36c6c743b376 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 15 Nov 2025 01:22:01 +0000 Subject: [PATCH 17/19] print stats in CI on server shutdown --- ci/build_cpp.sh | 2 +- ci/build_python.sh | 6 +++--- ci/build_wheel.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index cceb934f..4abd1b96 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -13,7 +13,7 @@ rapids-print-env rapids-logger "Begin cpp build" -sccache --stop-server >/dev/null 2>&1 || true +sccache --stop-server 2>/dev/null || true RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) export RAPIDS_PACKAGE_VERSION diff --git a/ci/build_python.sh b/ci/build_python.sh index d13efd78..eb0c5183 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -26,7 +26,7 @@ rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS" RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") # TODO: Remove `--test skip` flags once importing on a CPU node works correctly -sccache --stop-server >/dev/null 2>&1 || true +sccache --stop-server 2>/dev/null || true rapids-logger "Building pylibwholegraph" @@ -39,7 +39,7 @@ rattler-build build --recipe conda/recipes/pylibwholegraph \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --stop-server >/dev/null 2>&1 || true +sccache --stop-server 2>/dev/null || true rapids-logger "Building cugraph-pyg" @@ -52,7 +52,7 @@ rattler-build build --recipe conda/recipes/cugraph-pyg \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --stop-server >/dev/null 2>&1 || true +sccache --stop-server 2>/dev/null || true # remove build_cache directory to avoid uploading the entire source tree # tracked in https://github.com/prefix-dev/rattler-build/issues/1424 diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 601c61cc..0c08ad7a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -36,7 +36,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" cd "${package_dir}" -sccache --stop-server >/dev/null 2>&1 || true +sccache --stop-server 2>/dev/null || true rapids-logger "Building '${package_name}' wheel" rapids-pip-retry wheel \ From 62d68d1b2cba345c5fb449c5dc3627b117feae3f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 19 Nov 2025 19:04:00 +0000 Subject: [PATCH 18/19] use S3 as the preprocessor cache location --- ci/build_cpp.sh | 1 + ci/build_python.sh | 4 ++-- ci/build_wheel.sh | 5 +++++ conda/recipes/libwholegraph/recipe.yaml | 2 ++ conda/recipes/pylibwholegraph/recipe.yaml | 2 ++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 4abd1b96..2e0dde1b 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -29,3 +29,4 @@ rattler-build build --recipe conda/recipes/libwholegraph \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats +sccache --stop-server >/dev/null 2>&1 || true diff --git a/ci/build_python.sh b/ci/build_python.sh index eb0c5183..c7331969 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -39,7 +39,7 @@ rattler-build build --recipe conda/recipes/pylibwholegraph \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --stop-server 2>/dev/null || true +sccache --stop-server >/dev/null 2>&1 || true rapids-logger "Building cugraph-pyg" @@ -52,7 +52,7 @@ rattler-build build --recipe conda/recipes/cugraph-pyg \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -sccache --stop-server 2>/dev/null || true +sccache --stop-server >/dev/null 2>&1 || true # remove build_cache directory to avoid uploading the entire source tree # tracked in https://github.com/prefix-dev/rattler-build/issues/1424 diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0c08ad7a..29386466 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -30,6 +30,10 @@ fi source rapids-configure-sccache source rapids-date-string +# shellcheck disable=SC2155 +export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache" +export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true + rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" @@ -47,6 +51,7 @@ rapids-pip-retry wheel \ . sccache --show-adv-stats +sccache --stop-server >/dev/null 2>&1 || true # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "cugraph-pyg" ]]; then diff --git a/conda/recipes/libwholegraph/recipe.yaml b/conda/recipes/libwholegraph/recipe.yaml index 3a4cc2b0..abc1b524 100644 --- a/conda/recipes/libwholegraph/recipe.yaml +++ b/conda/recipes/libwholegraph/recipe.yaml @@ -55,6 +55,8 @@ cache: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} + SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache + SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }} SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }} diff --git a/conda/recipes/pylibwholegraph/recipe.yaml b/conda/recipes/pylibwholegraph/recipe.yaml index e2f3ed6c..44043d20 100644 --- a/conda/recipes/pylibwholegraph/recipe.yaml +++ b/conda/recipes/pylibwholegraph/recipe.yaml @@ -50,6 +50,8 @@ build: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }} SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }} + SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache + SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }} SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }} From 2e0b03af47eeb43af229db39bdc489981cd4ca7f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 19 Nov 2025 20:00:21 +0000 Subject: [PATCH 19/19] use S3 as the preprocessor cache location in devcontainers --- .devcontainer/Dockerfile | 1 + ci/build_wheel.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7053dd27..9b070b49 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -40,6 +40,7 @@ ENV HISTFILE="/home/coder/.cache/._bash_history" ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true ENV SCCACHE_IDLE_TIMEOUT=0 ### diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 29386466..47170a7a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -30,7 +30,6 @@ fi source rapids-configure-sccache source rapids-date-string -# shellcheck disable=SC2155 export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache" export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true