use S3 as the preprocessor cache location

Author: trxcllnt

ci/build_cpp.sh

@@ -29,3 +29,4 @@ rattler-build build --recipe conda/recipes/libwholegraph \
                     "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true

ci/build_python.sh

@@ -39,7 +39,7 @@ rattler-build build --recipe conda/recipes/pylibwholegraph \
                     "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
-sccache --stop-server 2>/dev/null || true
+sccache --stop-server >/dev/null 2>&1 || true
 
 rapids-logger "Building cugraph-pyg"
 
@@ -52,7 +52,7 @@ rattler-build build --recipe conda/recipes/cugraph-pyg \
                     "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
-sccache --stop-server 2>/dev/null || true
+sccache --stop-server >/dev/null 2>&1 || true
 
 # remove build_cache directory to avoid uploading the entire source tree
 # tracked in https://github.com/prefix-dev/rattler-build/issues/1424

ci/build_wheel.sh

@@ -30,6 +30,10 @@ fi
 source rapids-configure-sccache
 source rapids-date-string
 
+# shellcheck disable=SC2155
+export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache"
+export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
+
 rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
@@ -47,6 +51,7 @@ rapids-pip-retry wheel \
     .
 
 sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
 
 # pure-python packages should be marked as pure, and not have auditwheel run on them.
 if [[ ${package_name} == "cugraph-pyg" ]]; then

conda/recipes/libwholegraph/recipe.yaml

@@ -55,6 +55,8 @@ cache:
         SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
         SCCACHE_S3_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}
         SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
+        SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
+        SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
         SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
         SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
 

conda/recipes/pylibwholegraph/recipe.yaml

@@ -50,6 +50,8 @@ build:
       SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
       SCCACHE_S3_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}
       SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
+      SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: pylibwholegraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
+      SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
       SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
       SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}