CI: Select Flink PR matrix by changed version

.github/scripts/ci-plan-common.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+gradle_property() {
+  local name="$1"
+  sed -n "s/^systemProp\.${name}=//p" gradle.properties | head -n 1
+}
+
+changed_files=()
+read_changed_files() {
+  local file
+  while IFS= read -r file; do
+    if [[ -n "${file}" ]]; then
+      changed_files+=("${file}")
+    fi
+  done
+}
+
+read_pull_request_files() {
+  changed_files=()
+  if [[ -n "${CHANGED_FILES_FILE:-}" ]]; then
+    read_changed_files < "${CHANGED_FILES_FILE}"
+  elif [[ -n "${BASE_SHA:-}" ]]; then
+    git fetch --no-tags --depth=1 origin "${BASE_SHA}"
+    read_changed_files < <(git diff --name-only "${BASE_SHA}" HEAD)
+  fi
+}
+
+value_in_list() {
+  local value="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ "${candidate}" == "${value}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}

.github/scripts/plan-flink-ci.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -euo pipefail
+
+source .github/scripts/ci-plan-common.sh
+
+known_flink_versions="$(gradle_property knownFlinkVersions)"
+IFS=',' read -r -a all_flink_versions <<< "${known_flink_versions}"
+
+add_selected_flink_version() {
+  local value="$1"
+  if ! value_in_list "${value}" "${selected_flink_versions[@]:-}"; then
+    selected_flink_versions+=("${value}")
+  fi
+}
+
+select_flink_versions() {
+  local file
+  local flink_version
+  full_flink_matrix=false
+  selected_flink_versions=()
+
+  if [[ "${GITHUB_EVENT_NAME:-}" != "pull_request" || "${FULL_CI_LABEL:-false}" == "true" ]]; then
+    full_flink_matrix=true
+    selected_flink_versions=("${all_flink_versions[@]}")
+    return
+  fi
+
+  read_pull_request_files
+  if [[ ${#changed_files[@]} -eq 0 ]]; then
+    full_flink_matrix=true
+    selected_flink_versions=("${all_flink_versions[@]}")
+    return
+  fi
+
+  for file in "${changed_files[@]}"; do
+    if [[ "${file}" =~ ^flink/v([^/]+)/ ]]; then
+      flink_version="${BASH_REMATCH[1]}"
+      if value_in_list "${flink_version}" "${all_flink_versions[@]}"; then
+        add_selected_flink_version "${flink_version}"
+      else
+        full_flink_matrix=true
+        selected_flink_versions=("${all_flink_versions[@]}")
+        return
+      fi
+    else
+      full_flink_matrix=true
+      selected_flink_versions=("${all_flink_versions[@]}")
+      return
+    fi
+  done
+}
+
+jvm_versions() {
+  jq -r '.jvm[]' <<< "${JVM_MATRIX:?JVM_MATRIX must be set by the workflow}"
+}
+
+flink_matrix() {
+  local flink_version
+  local jvm
+  local -a rows=()
+
+  select_flink_versions
+
+  while IFS= read -r jvm; do
+    for flink_version in "${selected_flink_versions[@]}"; do
+      rows+=("${jvm}|${flink_version}")
+    done
+  done < <(jvm_versions)
+
+  printf '%s\n' "${rows[@]}" \
+    | jq -R 'split("|") | {jvm: .[0], flink: .[1]}' \
+    | jq -s -c '{include: .}'
+}
+
+echo "flink_matrix=$(flink_matrix)" >> "${GITHUB_OUTPUT}"

.github/workflows/delta-conversion-ci.yml

@@ -78,7 +78,7 @@ jobs:
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -107,7 +107,7 @@ jobs:
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/flink-ci.yml

@@ -73,16 +73,29 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  plan:
+    runs-on: ubuntu-24.04
+    outputs:
+      flink_matrix: ${{ steps.plan.outputs.flink_matrix }}
+    steps:
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      with:
+        persist-credentials: false
+    - id: plan
+      env:
+        BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        FULL_CI_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') }}
+        JVM_MATRIX: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '{"jvm":[17]}' || '{"jvm":[17,21]}' }}
+      run: bash .github/scripts/plan-flink-ci.sh
 
   # Test all flink versions with scala 2.12 for general validation.
   flink-scala-2-12-tests:
+    needs: plan
     runs-on: ubuntu-24.04
     timeout-minutes: 60
     strategy:
       max-parallel: 15
-      matrix:
-        jvm: [17, 21]
-        flink: ['1.20', '2.0', '2.1']
+      matrix: ${{ fromJson(needs.plan.outputs.flink_matrix) }}
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -98,7 +111,13 @@ jobs:
         # Read-only: java-ci's build-checks (17) is the global canonical writer.
         cache-read-only: true
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DkafkaVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc -DtestParallelism=auto
+    - env:
+        FLINK_VERSION: ${{ matrix.flink }}
+      run: >-
+        ./gradlew -DsparkVersions= -DkafkaVersions= -DflinkVersions="${FLINK_VERSION}"
+        ":iceberg-flink:iceberg-flink-${FLINK_VERSION}:check"
+        ":iceberg-flink:iceberg-flink-runtime-${FLINK_VERSION}:check"
+        -Pquick=true -x javadoc -DtestParallelism=auto
     - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
       if: failure()
       with:

.github/workflows/hive-ci.yml

@@ -79,7 +79,7 @@ jobs:
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/java-ci.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -103,7 +103,7 @@ jobs:
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     steps:
     - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
       with:
@@ -123,7 +123,7 @@ jobs:
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     steps:
     - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
       with:

.github/workflows/kafka-connect-ci.yml

@@ -73,13 +73,12 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-
   kafka-connect-tests:
     runs-on: ubuntu-24.04
     strategy:
       max-parallel: 15
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/spark-ci.yml

@@ -81,7 +81,7 @@ jobs:
       # Keep matrix <= 20 jobs; exceeding it queues a second wave and slows CI.
       max-parallel: 20
       matrix:
-        jvm: [17, 21]
+        jvm: ${{ fromJson((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-ci') == false) && '[17]' || '[17,21]') }}
         spark: ['3.5', '4.0', '4.1']
         scala: ['2.12', '2.13']
         # Split iceberg-spark (core) from iceberg-spark-extensions/-runtime
@@ -110,14 +110,21 @@ jobs:
           cache-read-only: true
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
       - name: Run tests
+        env:
+          SPARK_VERSION: ${{ matrix.spark }}
+          SCALA_VERSION: ${{ matrix.scala }}
+          TEST_GROUP: ${{ matrix.tests }}
         run: |
-          if [[ "${{ matrix.tests }}" == "core" ]]; then
-            projects=":iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check"
+          if [[ "${TEST_GROUP}" == "core" ]]; then
+            projects=(":iceberg-spark:iceberg-spark-${SPARK_VERSION}_${SCALA_VERSION}:check")
           else
-            projects=":iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check"
+            projects=(
+              ":iceberg-spark:iceberg-spark-extensions-${SPARK_VERSION}_${SCALA_VERSION}:check"
+              ":iceberg-spark:iceberg-spark-runtime-${SPARK_VERSION}_${SCALA_VERSION}:check"
+            )
           fi
-          ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DflinkVersions= -DkafkaVersions= \
-            $projects -Pquick=true -x javadoc -DtestParallelism=auto
+          ./gradlew -DsparkVersions="${SPARK_VERSION}" -DscalaVersion="${SCALA_VERSION}" -DflinkVersions= -DkafkaVersions= \
+            "${projects[@]}" -Pquick=true -x javadoc -DtestParallelism=auto
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure()
         with: