feat(ci): Improve Gradle cache in CI

.github/actions/ci-incr-build-cache-prepare/action.yml

@@ -0,0 +1,82 @@
+# Copyright (C) 2020 Dremio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
+
+name: 'Incremental Gradle build cache prepare'
+description: 'Prepare to save incremental Gradle build cache'
+inputs:
+  cache-read-only:
+    description: 'Gradle cache read only'
+    default: 'true'
+  java-version:
+    description: 'Java version'
+    default: '21'
+  job-id:
+    description: 'Job ID to prefer'
+    default: 'polaris-gradle'
+  job-instance:
+    description: 'Job instance to prefer'
+    default: 'gradle'
+runs:
+  using: "composite"
+  steps:
+    - name: Prep env
+      shell: bash
+      run: |
+        echo "GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT=java-${{ inputs.java-version }}" >> ${GITHUB_ENV}
+        echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB=${{ inputs.job-id }}" >> ${GITHUB_ENV}
+        if [[ -n "${{ inputs.no-daemon }}" ]] ; then
+          echo "G_DAEMON_FLAG=--no-daemon" >> ${GITHUB_ENV}
+        fi
+        if [[ -n "${{ inputs.job-instance }}" ]] ; then
+          echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE=${{ inputs.job-instance }}" >> ${GITHUB_ENV}
+        fi
+
+    - name: Gradle / Setup
+      uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+      with:
+        cache-read-only: ${{ inputs.cache-read-only }}
+        validate-wrappers: false
+
+    - name: Gradle / Init
+      shell: bash
+      run: ./gradlew -h
+
+    - name: Download existing workflow artifacts
+      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      # Just in case, don't know the exact inner workings of Gradle's build cache and whether
+      # the download-action complains about duplicate files.
+      continue-on-error: true
+      with:
+        path: ~/downloaded-artifacts/
+
+    - name: Extract caches
+      shell: bash
+      run: |
+        echo "::group::Gradle build cache / add incremental updates"
+        mkdir -p ~/.gradle/caches/
+
+        if [[ -d ~/downloaded-artifacts/ ]] ; then
+          find ~/downloaded-artifacts/ -type f -name "ci-gradle-caches-*-${{ inputs.java-version }}.tar" | while read arch ; do
+            echo "Adding archive content from $arch ..."
+            (cd ~/.gradle/caches/ ; tar xf $arch)
+          done
+        else
+          echo "No previous build cache artifacts downloaded."
+        fi
+
+        date +%s > ~/caches-prepared-at-epoch
+
+        echo "::endgroup::"

.github/actions/ci-incr-build-cache-save/action.yml

@@ -0,0 +1,70 @@
+# Copyright (C) 2020 Dremio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
+
+name: 'Save incremental Gradle caches'
+description: 'Save incremental Gradle caches'
+inputs:
+  job-name:
+    description: 'job name'
+  java-version:
+    description: 'Java version'
+    default: '21'
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare Gradle caches archive
+      shell: bash
+      run: |
+        if [[ -d ~/.gradle/caches/ ]] ; then
+          echo "::group::Gradle caches / identify updated cache items"
+
+          cd ~/.gradle/caches/
+
+          echo "Gradle caches/ contains $(find . -type f | wc -l) files"
+          # Identify the added and changed files in caches/.
+
+          echo "Identifying changed/added files..."
+          AGE_SECS=$(($(date +%s) - $(cat ~/caches-prepared-at-epoch)))
+          echo "Build started ~ $AGE_SECS seconds ago"
+          AGE_MINS=$(($AGE_SECS / 60 + 1))
+          echo " ... assuming that is ~ $AGE_MINS minutes"
+          # This lists all relevant files that have been created or modified during by the Gradle
+          # runs of the current job.
+          find . -mmin -$AGE_MINS -type f '(' \
+            -path './[0-9]*/kotlin-dsl/*' -or \
+            -path './jars-*/*' -or \
+            -path './modules-*/files-*/*' -or \
+            -path './modules-*/files-*/*' -or \
+            -path './build-cache-*/*' \
+            ')' | grep -v '[.]lock$'  > ~/ci-gradle-caches-diff || true
+          echo "Identified $(wc -l < ~/ci-gradle-caches-diff) changed/added files in caches/"
+
+          # Only call 'tar', if there is some difference
+          # Note: actions/upload-artifact takes care of compressing the artifact, no need to bug the CPU here
+          echo "Creating artifact (if necessary)..."
+          if [[ -s ~/ci-gradle-caches-diff ]] ; then
+            tar --create --ignore-failed-read --file ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar -T ~/ci-gradle-caches-diff
+            ls -al ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
+          fi
+          echo "::endgroup::"
+        fi
+    - name: Archive code-checks incremental
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}
+        path: ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
+        if-no-files-found: ignore
+        retention-days: 1

.github/workflows/gradle.yml

@@ -50,12 +50,19 @@ jobs:
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
         with:
           validate-wrappers: false
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
       - name: Run unit tests
         run: |
           ./gradlew check sourceTarball distTar distZip publishToMavenLocal \
             -x :polaris-runtime-service:test \
             -x :polaris-admin:test \
             -x intTest --continue
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: 'unit-tests'
       - name: Archive test results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
@@ -80,12 +87,19 @@ jobs:
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
         with:
           validate-wrappers: false
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
       - name: Run Quarkus tests
         run: |
           ./gradlew \
             :polaris-runtime-service:test \
             :polaris-admin:test \
             --continue
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: 'quarkus-tests'
       - name: Archive test results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
@@ -110,12 +124,46 @@ jobs:
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
         with:
           validate-wrappers: false
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
       - name: Run integration tests
         run: ./gradlew intTest --continue
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: 'integration-tests'
       - name: Archive test results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
         with:
           name: upload-integration-test-artifacts
           path: |
             **/build/test-results/**
+
+  store-gradle-cache:
+    name: Store Gradle Cache
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs:
+      - unit-tests
+      - quarkus-tests
+      - integration-tests
+    steps:
+      - name: Set up JDK 21
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+      - name: Collect partial Gradle build caches
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+        with:
+          cache-read-only: false
+      - name: Trigger Gradle home cleanup
+        run: ./gradlew --no-daemon :showVersion
+      # Note: the "Post Gradle invocation" archives the updated build cache.

.github/workflows/regtest.yml

@@ -40,6 +40,14 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
 
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+
       - name: Fix permissions
         run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
 
@@ -58,4 +66,4 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
         run: |
-          docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest
+          docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest

.github/workflows/spark_client_regtests.yml

@@ -40,6 +40,14 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
 
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+
       - name: Fix permissions
         run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
 

LICENSE

@@ -263,6 +263,8 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 
 This product includes code from Project Nessie.
 
+* .github/actions/ci-incr-build-cache-prepare/action.yml
+* .github/actions/ci-incr-build-cache-save/action.yml
 * build-logic/src/main/kotlin/LicenseFileValidation.kt
 * build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerPlugin.kt
 * build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt