[AURON #2015] Add Native Scan Support for Apache Iceberg Copy-On-Write Tables.

.github/workflows/iceberg.yml

@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Iceberg
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+      - branch-*
+  pull_request:
+    branches:
+      - master
+      - branch-*
+
+concurrency:
+  group: iceberg-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-iceberg:
+    name: Test Iceberg (${{ matrix.sparkver }} / JDK${{ matrix.javaver }} / Scala${{ matrix.scalaver }})
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sparkver: "3.4"
+            scalaver: "2.12"
+            javaver: "17"
+          - sparkver: "3.5"
+            scalaver: "2.12"
+            javaver: "17"
+          - sparkver: "3.5"
+            scalaver: "2.12"
+            javaver: "21"
+          - sparkver: "4.0"
+            scalaver: "2.13"
+            javaver: "21"
+
+    steps:
+      - name: Checkout Auron
+        uses: actions/checkout@v6
+
+      - name: Setup Java and Maven cache
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'adopt-hotspot'
+          java-version: ${{ matrix.javaver }}
+          cache: 'maven'
+
+      - name: Build dependencies (skip tests)
+        run: >
+          ./build/mvn -B install
+          -pl thirdparty/auron-iceberg
+          -am
+          -Pscala-${{ matrix.scalaver }}
+          -Pspark-${{ matrix.sparkver }}
+          -Piceberg
+          -DicebergEnabled=true
+          -DicebergVersion=1.10.1
+          -DskipTests
+
+      - name: Test Iceberg Module
+        run: >
+          ./build/mvn -B test
+          -pl thirdparty/auron-iceberg
+          -Pscala-${{ matrix.scalaver }}
+          -Pspark-${{ matrix.sparkver }}
+          -Piceberg
+          -DicebergEnabled=true
+          -DicebergVersion=1.10.1
+
+      - name: Upload reports
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: auron-iceberg-${{ matrix.sparkver }}-jdk${{ matrix.javaver }}-test-report
+          path: thirdparty/auron-iceberg/target/surefire-reports

auron-build.sh

@@ -37,7 +37,7 @@ SUPPORTED_CELEBORN_VERSIONS=("0.5" "0.6")
 SUPPORTED_UNIFFLE_VERSIONS=("0.10")
 SUPPORTED_PAIMON_VERSIONS=("1.2")
 SUPPORTED_FLINK_VERSIONS=("1.18")
-SUPPORTED_ICEBERG_VERSIONS=("1.9")
+SUPPORTED_ICEBERG_VERSIONS=("1.10.1")
 
 # -----------------------------------------------------------------------------
 # Function: print_help
@@ -270,11 +270,11 @@ while [[ $# -gt 0 ]]; do
                   print_invalid_option_error Iceberg "$ICEBERG_VER" "${SUPPORTED_ICEBERG_VERSIONS[@]}"
                 fi
                 if [ -z "$SPARK_VER" ]; then
-                  echo "ERROR: Building iceberg requires spark at the same time, and only Spark versions 3.4 or 3.5 are supported."
+                  echo "ERROR: Building iceberg requires spark at the same time, and only Spark versions 3.4 to 4.0 are supported."
                   exit 1
                 fi
-                if [ "$SPARK_VER" != "3.4" ] && [ "$SPARK_VER" != "3.5" ]; then
-                  echo "ERROR: Building iceberg requires spark versions are 3.4 or 3.5."
+                if [ "$SPARK_VER" != "3.4" ] && [ "$SPARK_VER" != "3.5" ] && [ "$SPARK_VER" != "4.0" ]; then
+                  echo "ERROR: Building iceberg requires spark versions are 3.4, 3.5 or 4.0."
                   exit 1
                 fi
                 shift 2
@@ -393,7 +393,7 @@ if [[ -n "$FLINK_VER" ]]; then
     BUILD_ARGS+=("-Pflink-$FLINK_VER")
 fi
 if [[ -n "$ICEBERG_VER" ]]; then
-    BUILD_ARGS+=("-Piceberg-$ICEBERG_VER")
+    BUILD_ARGS+=("-Piceberg" "-DicebergEnabled=true" "-DicebergVersion=$ICEBERG_VER")
 fi
 
 MVN_ARGS=("${CLEAN_ARGS[@]}" "${BUILD_ARGS[@]}")

dev/reformat

@@ -48,15 +48,15 @@ else
   cargo fmt --all -q --
 fi
 
-# Check or format all code, including third-party code, with spark-3.5
+# Check or format all code, including third-party code, with spark-3.4
 sparkver=spark-3.5
 for celebornver in celeborn-0.5 celeborn-0.6
 do
-  run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10 -Ppaimon,paimon-1.2 -Pflink,flink-1.18 -Piceberg,iceberg-1.9
+  run_maven -P"${sparkver}" -P"${celebornver}" -Puniffle-0.10 -Ppaimon-1.2 -Pflink-1.18 -Piceberg -DicebergEnabled=true -DicebergVersion=1.10.1
 
 done
 
-sparkvers=(spark-3.0 spark-3.1 spark-3.2 spark-3.3 spark-3.4)
+sparkvers=(spark-3.0 spark-3.1 spark-3.2 spark-3.3 spark-3.5)
 for sparkver in "${sparkvers[@]}"
 do
   run_maven -P"${sparkver}"