[KYUUBI #6928] Bump Spark 4.0.0

### Why are the changes needed?

Test Spark 4.0.0 RC1
https://lists.apache.org/thread/3sx86qhnmot1p519lloyprxv9h7nt2xh

### How was this patch tested?

GHA.

### Was this patch authored or co-authored using generative AI tooling?

No

Closes #6928 from pan3793/spark-4.0.0.

Closes #6928

a910169 [Cheng Pan] Bump Spark 4.0.0

Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>

Author: pan3793

.github/workflows/license.yml

@@ -41,10 +41,10 @@ jobs:
           distribution: temurin
           java-version: 8
           cache: 'maven'
-      - run: >-
-          build/mvn org.apache.rat:apache-rat-plugin:check
-          -Ptpcds -Pkubernetes-it
-          -Pspark-3.3 -Pspark-3.4 -Pspark-3.5
+      - run: |
+          build/mvn org.apache.rat:apache-rat-plugin:check \
+            -Ptpcds -Pkubernetes-it \
+            -Pspark-3.3 -Pspark-3.4 -Pspark-3.5 -Pspark-4.0
       - name: Upload rat report
         if: failure()
         uses: actions/upload-artifact@v4

.github/workflows/master.yml

@@ -77,7 +77,7 @@ jobs:
             comment: 'verify-on-spark-3.4-binary'
           - java: 17
             spark: '3.5'
-            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0-preview2 -Dspark.archive.name=spark-4.0.0-preview2-bin-hadoop3.tgz'
+            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0 -Dspark.archive.name=spark-4.0.0-bin-hadoop3.tgz'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-4.0-binary'
     env:

pom.xml

@@ -2064,18 +2064,19 @@
                 <module>extensions/spark/kyuubi-spark-connector-hive</module>
             </modules>
             <properties>
-                <spark.version>4.0.0-preview2</spark.version>
+                <spark.version>4.0.0</spark.version>
                 <spark.binary.version>4.0</spark.binary.version>
                 <antlr4.version>4.13.1</antlr4.version>
-                <!-- TODO: update once Delta support Spark 4.0.0-preview2 -->
+                <!-- TODO: update once Delta support Spark 4.0 -->
                 <delta.version>4.0.0rc1</delta.version>
                 <delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
                 <!-- TODO: update once Hudi support Spark 4.0 -->
                 <hudi.artifact>hudi-spark3.5-bundle_${scala.binary.version}</hudi.artifact>
                 <!-- TODO: update once Iceberg support Spark 4.0 -->
                 <iceberg.artifact>iceberg-spark-runtime-3.5_${scala.binary.version}</iceberg.artifact>
-                <!-- TODO: update once Paimon support Spark 4.0 -->
-                <paimon.artifact>paimon-spark-3.5</paimon.artifact>
+                <!-- TODO: update once Paimon support Spark 4.0.
+                           paimon-spark-3.5 contains Scala 2.12 classes cause conflicts with Scala 2.13 -->
+                <paimon.artifact>paimon-common</paimon.artifact>
                 <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
                 <spark.archive.name>spark-${spark.version}-bin-hadoop3.tgz</spark.archive.name>
             </properties>