From c39bd85ba4bdb612e384c427dfb23cc6dc92c8de Mon Sep 17 00:00:00 2001
From: Alexandre Dutra <adutra@users.noreply.github.com>
Date: Tue, 24 Jun 2025 17:44:01 +0200
Subject: [PATCH 1/6] feat(ci): Improve Gradle cache in CI

---
 .../ci-incr-build-cache-prepare/action.yml    | 82 +++++++++++++++++++
 .../ci-incr-build-cache-save/action.yml       | 70 ++++++++++++++++
 .github/workflows/gradle.yml                  | 48 +++++++++++
 3 files changed, 200 insertions(+)
 create mode 100644 .github/actions/ci-incr-build-cache-prepare/action.yml
 create mode 100644 .github/actions/ci-incr-build-cache-save/action.yml

diff --git a/.github/actions/ci-incr-build-cache-prepare/action.yml b/.github/actions/ci-incr-build-cache-prepare/action.yml
new file mode 100644
index 0000000000..cd06fd7049
--- /dev/null
+++ b/.github/actions/ci-incr-build-cache-prepare/action.yml
@@ -0,0 +1,82 @@
+# Copyright (C) 2020 Dremio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
+
+name: 'Incremental Gradle build cache prepare'
+description: 'Prepare to save incremental Gradle build cache'
+inputs:
+  cache-read-only:
+    description: 'Gradle cache read only'
+    default: 'true'
+  java-version:
+    description: 'Java version'
+    default: '21'
+  job-id:
+    description: 'Job name to prefer'
+    default: 'polaris-gradle'
+  job-instance:
+    description: 'Job instance to prefer'
+    default: 'gradle'
+runs:
+  using: "composite"
+  steps:
+    - name: Prep env
+      shell: bash
+      run: |
+        echo "GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT=java-${{ inputs.java-version }}" >> ${GITHUB_ENV}
+        echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB=${{ inputs.job-id }}" >> ${GITHUB_ENV}
+        if [[ -n "${{ inputs.no-daemon }}" ]] ; then
+          echo "G_DAEMON_FLAG=--no-daemon" >> ${GITHUB_ENV}
+        fi
+        if [[ -n "${{ inputs.job-instance }}" ]] ; then
+          echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE=${{ inputs.job-instance }}" >> ${GITHUB_ENV}
+        fi
+
+    - name: Gradle / Setup
+      uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+      with:
+        cache-read-only: ${{ inputs.cache-read-only }}
+        validate-wrappers: false
+
+    - name: Gradle / Init
+      shell: bash
+      run: ./gradlew -h
+
+    - name: Download existing workflow artifacts
+      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      # Just in case, don't know the exact inner workings of Gradle's build cache and whether
+      # the download-action complains about duplicate files.
+      continue-on-error: true
+      with:
+        path: ~/downloaded-artifacts/
+
+    - name: Extract caches
+      shell: bash
+      run: |
+        echo "::group::Gradle build cache / add incremental updates"
+        mkdir -p ~/.gradle/caches/
+
+        if [[ -d ~/downloaded-artifacts/ ]] ; then
+          find ~/downloaded-artifacts/ -type f -name "ci-gradle-caches-*-${{ inputs.java-version }}.tar" | while read arch ; do
+            echo "Adding archive content from $arch ..."
+            (cd ~/.gradle/caches/ ; tar xf $arch)
+          done
+        else
+          echo "No previous build cache artifacts downloaded."
+        fi
+        
+        date +%s > ~/caches-prepared-at-epoch
+
+        echo "::endgroup::"
diff --git a/.github/actions/ci-incr-build-cache-save/action.yml b/.github/actions/ci-incr-build-cache-save/action.yml
new file mode 100644
index 0000000000..4d6085c971
--- /dev/null
+++ b/.github/actions/ci-incr-build-cache-save/action.yml
@@ -0,0 +1,70 @@
+# Copyright (C) 2020 Dremio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
+
+name: 'Save incremental Gradle caches'
+description: 'Save incremental Gradle caches'
+inputs:
+  job-name:
+    description: 'job name'
+  java-version:
+    description: 'Java version'
+    default: '21'
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare Gradle caches archive
+      shell: bash
+      run: |
+        if [[ -d ~/.gradle/caches/ ]] ; then
+          echo "::group::Gradle caches / identify updated cache items"
+
+          cd ~/.gradle/caches/
+
+          echo "Gradle caches/ contains $(find . -type f | wc -l) files"
+          # Identify the added and changed files in caches/.
+
+          echo "Identifying changed/added files..."
+          AGE_SECS=$(($(date +%s) - $(cat ~/caches-prepared-at-epoch)))
+          echo "Build started ~ $AGE_SECS seconds ago"
+          AGE_MINS=$(($AGE_SECS / 60 + 1))
+          echo " ... assuming that is ~ $AGE_MINS minutes"
+          # This lists all relevant files that have been created or modified during by the Gradle
+          # runs of the current job.
+          find . -mmin -$AGE_MINS -type f '(' \
+            -path './[0-9]*/kotlin-dsl/*' -or \
+            -path './jars-*/*' -or \
+            -path './modules-*/files-*/*' -or \
+            -path './modules-*/files-*/*' -or \
+            -path './build-cache-*/*' \
+            ')' | grep -v '[.]lock$'  > ~/ci-gradle-caches-diff || true
+          echo "Identified $(wc -l < ~/ci-gradle-caches-diff) changed/added files in caches/"
+
+          # Only call 'tar', if there is some difference
+          # Note: actions/upload-artifact takes care of compressing the artifact, no need to bug the CPU here
+          echo "Creating artifact (if necessary)..."
+          if [[ -s ~/ci-gradle-caches-diff ]] ; then
+            tar --create --ignore-failed-read --file ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar -T ~/ci-gradle-caches-diff
+            ls -al ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
+          fi
+          echo "::endgroup::"
+        fi
+    - name: Archive code-checks incremental
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}
+        path: ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
+        if-no-files-found: ignore
+        retention-days: 1
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
index ae974984f6..9ffc77288d 100644
--- a/.github/workflows/gradle.yml
+++ b/.github/workflows/gradle.yml
@@ -50,12 +50,19 @@ jobs:
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
         with:
           validate-wrappers: false
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
       - name: Run unit tests
         run: |
           ./gradlew check sourceTarball distTar distZip publishToMavenLocal \
             -x :polaris-runtime-service:test \
             -x :polaris-admin:test \
             -x intTest --continue
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: 'unit-tests'
       - name: Archive test results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
@@ -80,12 +87,19 @@ jobs:
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
         with:
           validate-wrappers: false
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
       - name: Run Quarkus tests
         run: |
           ./gradlew \
             :polaris-runtime-service:test \
             :polaris-admin:test \
             --continue
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: 'quarkus-tests'
       - name: Archive test results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
@@ -110,8 +124,15 @@ jobs:
         uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
         with:
           validate-wrappers: false
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
       - name: Run integration tests
         run: ./gradlew intTest --continue
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: 'integration-tests'
       - name: Archive test results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
@@ -119,3 +140,30 @@ jobs:
           name: upload-integration-test-artifacts
           path: |
             **/build/test-results/**
+
+  store-gradle-cache:
+    name: Store Gradle Cache
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs:
+      - unit-tests
+      - quarkus-tests
+      - integration-tests
+    steps:
+      - name: Set up JDK 21
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+      - name: Collect partial Gradle build caches
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+        with:
+          cache-read-only: false
+      - name: Trigger Gradle home cleanup
+        run: ./gradlew --no-daemon :showVersion
+      # Note: the "Post Gradle invocation" archives the updated build cache.

From 382f317132990cf06c06876d874c3a729706d070 Mon Sep 17 00:00:00 2001
From: Alexandre Dutra <adutra@users.noreply.github.com>
Date: Fri, 27 Jun 2025 16:21:42 +0200
Subject: [PATCH 2/6] add cache mgmt to regtest jobs

---
 .github/workflows/regtest.yml               | 41 ++++++++++++++++++++-
 .github/workflows/spark_client_regtests.yml | 39 ++++++++++++++++++++
 2 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml
index 70e433baf5..89bf8d966f 100644
--- a/.github/workflows/regtest.yml
+++ b/.github/workflows/regtest.yml
@@ -40,6 +40,14 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
 
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+
       - name: Fix permissions
         run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
 
@@ -53,9 +61,40 @@ jobs:
               :polaris-server:quarkusAppPartsBuild --rerun \
               -Dquarkus.container-image.build=true
 
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: regtest
+
       - name: Regression Test
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
         run: |
-          docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest
\ No newline at end of file
+          docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest
+
+  store-gradle-cache:
+    name: Store Gradle Cache
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs:
+      - regtest
+    steps:
+      - name: Set up JDK 21
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+      - name: Collect partial Gradle build caches
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+        with:
+          cache-read-only: false
+      - name: Trigger Gradle home cleanup
+        run: ./gradlew --no-daemon :showVersion
+      # Note: the "Post Gradle invocation" archives the updated build cache.
diff --git a/.github/workflows/spark_client_regtests.yml b/.github/workflows/spark_client_regtests.yml
index b4da3331e6..82fd676d04 100644
--- a/.github/workflows/spark_client_regtests.yml
+++ b/.github/workflows/spark_client_regtests.yml
@@ -40,6 +40,14 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
 
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+
       - name: Fix permissions
         run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
 
@@ -56,6 +64,12 @@ jobs:
               :polaris-server:quarkusAppPartsBuild --rerun \
               -Dquarkus.container-image.build=true
 
+      - name: Save partial Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-save
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          job-name: spark-plugin-regtest
+
       # NOTE: the regression test runs with spark 3.5.5 and scala 2.12 in Java 17. We also have integration
       # tests runs with the existing gradle.yml, which only runs on Java 21. Since spark Java compatibility
       # for 3.5 is 8, 11, and 17, we should run spark client with those compatible java versions.
@@ -63,3 +77,28 @@ jobs:
       - name: Regression Test
         run: |
           docker compose -f plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest
+
+  store-gradle-cache:
+    name: Store Gradle Cache
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs:
+      - spark-plugin-regtest
+    steps:
+      - name: Set up JDK 21
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
+        with:
+          validate-wrappers: false
+      - name: Collect partial Gradle build caches
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+        with:
+          cache-read-only: false
+      - name: Trigger Gradle home cleanup
+        run: ./gradlew --no-daemon :showVersion
+      # Note: the "Post Gradle invocation" archives the updated build cache.

From 744f78c5a91e161843fbe80dd32adbeb11449f23 Mon Sep 17 00:00:00 2001
From: Alexandre Dutra <adutra@users.noreply.github.com>
Date: Fri, 27 Jun 2025 18:09:41 +0200
Subject: [PATCH 3/6] review

---
 .github/actions/ci-incr-build-cache-prepare/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/ci-incr-build-cache-prepare/action.yml b/.github/actions/ci-incr-build-cache-prepare/action.yml
index cd06fd7049..55c80c3ef9 100644
--- a/.github/actions/ci-incr-build-cache-prepare/action.yml
+++ b/.github/actions/ci-incr-build-cache-prepare/action.yml
@@ -24,7 +24,7 @@ inputs:
     description: 'Java version'
     default: '21'
   job-id:
-    description: 'Job name to prefer'
+    description: 'Job ID to prefer'
     default: 'polaris-gradle'
   job-instance:
     description: 'Job instance to prefer'

From 63531b5947f67e0490331bcd8c618409d21689ef Mon Sep 17 00:00:00 2001
From: Alexandre Dutra <adutra@users.noreply.github.com>
Date: Thu, 3 Jul 2025 11:21:45 +0200
Subject: [PATCH 4/6] mention files in LICENSE

---
 LICENSE | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/LICENSE b/LICENSE
index 4330ca1544..86fab5a2e4 100644
--- a/LICENSE
+++ b/LICENSE
@@ -263,6 +263,8 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 
 This product includes code from Project Nessie.
 
+* .github/actions/ci-incr-build-cache-prepare/action.yml
+* .github/actions/ci-incr-build-cache-save/action.yml
 * build-logic/src/main/kotlin/LicenseFileValidation.kt
 * build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerPlugin.kt
 * build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt

From a747e670d011261d9c059f6f15a001f09b614c23 Mon Sep 17 00:00:00 2001
From: Alexandre Dutra <adutra@users.noreply.github.com>
Date: Fri, 11 Jul 2025 16:37:25 +0200
Subject: [PATCH 5/6] review

---
 .github/workflows/regtest.yml | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml
index 89bf8d966f..e6d531b0e2 100644
--- a/.github/workflows/regtest.yml
+++ b/.github/workflows/regtest.yml
@@ -61,40 +61,9 @@ jobs:
               :polaris-server:quarkusAppPartsBuild --rerun \
               -Dquarkus.container-image.build=true
 
-      - name: Save partial Gradle build cache
-        uses: ./.github/actions/ci-incr-build-cache-save
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        with:
-          job-name: regtest
-
       - name: Regression Test
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
         run: |
           docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest
-
-  store-gradle-cache:
-    name: Store Gradle Cache
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    needs:
-      - regtest
-    steps:
-      - name: Set up JDK 21
-        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
-        with:
-          java-version: '21'
-          distribution: 'temurin'
-      - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
-        with:
-          validate-wrappers: false
-      - name: Collect partial Gradle build caches
-        uses: ./.github/actions/ci-incr-build-cache-prepare
-        with:
-          cache-read-only: false
-      - name: Trigger Gradle home cleanup
-        run: ./gradlew --no-daemon :showVersion
-      # Note: the "Post Gradle invocation" archives the updated build cache.

From a144b1e438acd7ca16c4adb7fb8125773f832f77 Mon Sep 17 00:00:00 2001
From: Alexandre Dutra <adutra@apache.org>
Date: Sun, 13 Jul 2025 17:20:35 +0200
Subject: [PATCH 6/6] review

---
 .github/workflows/spark_client_regtests.yml | 31 ---------------------
 1 file changed, 31 deletions(-)

diff --git a/.github/workflows/spark_client_regtests.yml b/.github/workflows/spark_client_regtests.yml
index 82fd676d04..9f606428b1 100644
--- a/.github/workflows/spark_client_regtests.yml
+++ b/.github/workflows/spark_client_regtests.yml
@@ -64,12 +64,6 @@ jobs:
               :polaris-server:quarkusAppPartsBuild --rerun \
               -Dquarkus.container-image.build=true
 
-      - name: Save partial Gradle build cache
-        uses: ./.github/actions/ci-incr-build-cache-save
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        with:
-          job-name: spark-plugin-regtest
-
       # NOTE: the regression test runs with spark 3.5.5 and scala 2.12 in Java 17. We also have integration
       # tests runs with the existing gradle.yml, which only runs on Java 21. Since spark Java compatibility
       # for 3.5 is 8, 11, and 17, we should run spark client with those compatible java versions.
@@ -77,28 +71,3 @@ jobs:
       - name: Regression Test
         run: |
           docker compose -f plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest
-
-  store-gradle-cache:
-    name: Store Gradle Cache
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    needs:
-      - spark-plugin-regtest
-    steps:
-      - name: Set up JDK 21
-        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
-        with:
-          java-version: '21'
-          distribution: 'temurin'
-      - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
-        with:
-          validate-wrappers: false
-      - name: Collect partial Gradle build caches
-        uses: ./.github/actions/ci-incr-build-cache-prepare
-        with:
-          cache-read-only: false
-      - name: Trigger Gradle home cleanup
-        run: ./gradlew --no-daemon :showVersion
-      # Note: the "Post Gradle invocation" archives the updated build cache.