elastic · samxbr · Jan 31, 2025 · Jan 21, 2025 · Jan 21, 2025 · Jan 21, 2025
diff --git a/...org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy b/...org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy
@@ -78,7 +78,8 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor {
     @Override
     String artifactName(ServiceConfiguration configuration) {
         Version version = configuration.getVersion()
-        return "spark-$version-bin-hadoop${hadoopVersionCompatibility()}"
+        // artifact name: https://spark.apache.org/downloads.html
+        return "spark-$version-bin-hadoop${hadoopVersionCompatibility()}-scala2.13"
     }
 
     @Override

diff --git a/dist/build.gradle b/dist/build.gradle
@@ -12,7 +12,7 @@ base {
   archivesName = 'elasticsearch-hadoop'
 }
 
-def sparkVariantIncluded = 'spark30scala212'
+def sparkVariantIncluded = 'spark30scala213'
 
 configurations {
     embedded {
@@ -86,25 +86,25 @@ dependencies {
     }
     implementation("org.apache.hive:hive-exec:$hiveVersion")
     implementation("org.apache.hive:hive-metastore:$hiveVersion")
-    implementation("org.apache.spark:spark-core_${project.ext.scala212MajorVersion}:$spark30Version") {
+    implementation("org.apache.spark:spark-core_${project.ext.scala213MajorVersion}:$spark30Version") {
         exclude group: 'javax.servlet'
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.apache.spark:spark-yarn_${project.ext.scala212MajorVersion}:$spark30Version") {
+    implementation("org.apache.spark:spark-yarn_${project.ext.scala213MajorVersion}:$spark30Version") {
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.apache.spark:spark-sql_${project.ext.scala212MajorVersion}:$spark30Version") {
+    implementation("org.apache.spark:spark-sql_${project.ext.scala213MajorVersion}:$spark30Version") {
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.apache.spark:spark-streaming_${project.ext.scala212MajorVersion}:$spark30Version") {
+    implementation("org.apache.spark:spark-streaming_${project.ext.scala213MajorVersion}:$spark30Version") {
         exclude group: 'org.apache.hadoop'
     }
-    implementation("org.scala-lang:scala-library:$scala211Version")
-    implementation("org.scala-lang:scala-reflect:$scala211Version")
+    implementation("org.scala-lang:scala-library:$scala213Version")
+    implementation("org.scala-lang:scala-reflect:$scala213Version")
     implementation(project.ext.hadoopClient)
     implementation("org.apache.hadoop:hadoop-common:${project.ext.hadoopVersion}")
     implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${project.ext.hadoopVersion}")
-    compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala212MajorVersion}:$spark30Version")
+    compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala213MajorVersion}:$spark30Version")
 }
 
 // Configure uber jar

diff --git a/dist/licenses/scala-library-2.11.12.jar.sha1 b/dist/licenses/scala-library-2.11.12.jar.sha1
diff --git a/dist/licenses/scala-library-2.13.13.jar.sha1 b/dist/licenses/scala-library-2.13.13.jar.sha1
@@ -0,0 +1 @@
+3b1434403ddb147b8030f98f246153e4ce622c78
diff --git a/dist/licenses/scala-reflect-2.11.12.jar.sha1 b/dist/licenses/scala-reflect-2.11.12.jar.sha1
diff --git a/dist/licenses/scala-reflect-2.13.13.jar.sha1 b/dist/licenses/scala-reflect-2.13.13.jar.sha1
@@ -0,0 +1 @@
+46866f0e2cf6168322dc11f0f721c2db78d52ac7
diff --git a/dist/licenses/spark-core_2.12-3.4.3.jar.sha1 b/dist/licenses/spark-core_2.12-3.4.3.jar.sha1
diff --git a/dist/licenses/spark-core_2.13-3.4.3.jar.sha1 b/dist/licenses/spark-core_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+a8055d61a13e9af6871e05ecdca141134d102f1f
diff --git a/dist/licenses/spark-sql_2.12-3.4.3.jar.sha1 b/dist/licenses/spark-sql_2.12-3.4.3.jar.sha1
diff --git a/dist/licenses/spark-sql_2.13-3.4.3.jar.sha1 b/dist/licenses/spark-sql_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+b1744080f5e2f81b9a1a088eb375d33b85132b6d
diff --git a/dist/licenses/spark-streaming_2.12-3.4.3.jar.sha1 b/dist/licenses/spark-streaming_2.12-3.4.3.jar.sha1
diff --git a/dist/licenses/spark-streaming_2.13-3.4.3.jar.sha1 b/dist/licenses/spark-streaming_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+64c4e38f3a89428926383f4d9f8a8636a81d499b
diff --git a/dist/licenses/spark-yarn_2.12-3.4.3.jar.sha1 b/dist/licenses/spark-yarn_2.12-3.4.3.jar.sha1
diff --git a/dist/licenses/spark-yarn_2.13-3.4.3.jar.sha1 b/dist/licenses/spark-yarn_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4
diff --git a/licenses/scala-library-2.12.19.jar.sha1 b/licenses/scala-library-2.12.19.jar.sha1
diff --git a/licenses/scala-library-2.13.13.jar.sha1 b/licenses/scala-library-2.13.13.jar.sha1
@@ -0,0 +1 @@
+3b1434403ddb147b8030f98f246153e4ce622c78
diff --git a/licenses/scala-reflect-2.12.19.jar.sha1 b/licenses/scala-reflect-2.12.19.jar.sha1
diff --git a/licenses/scala-reflect-2.13.13.jar.sha1 b/licenses/scala-reflect-2.13.13.jar.sha1
@@ -0,0 +1 @@
+46866f0e2cf6168322dc11f0f721c2db78d52ac7
diff --git a/licenses/spark-catalyst_2.12-3.4.3.jar.sha1 b/licenses/spark-catalyst_2.12-3.4.3.jar.sha1
diff --git a/licenses/spark-catalyst_2.13-3.4.3.jar.sha1 b/licenses/spark-catalyst_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+6a9e9c5e149bd019dd5d781888e1e81ca4d32875
diff --git a/licenses/spark-core_2.12-3.4.3.jar.sha1 b/licenses/spark-core_2.12-3.4.3.jar.sha1
diff --git a/licenses/spark-core_2.13-3.4.3.jar.sha1 b/licenses/spark-core_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+a8055d61a13e9af6871e05ecdca141134d102f1f
diff --git a/licenses/spark-sql_2.12-3.4.3.jar.sha1 b/licenses/spark-sql_2.12-3.4.3.jar.sha1
diff --git a/licenses/spark-sql_2.13-3.4.3.jar.sha1 b/licenses/spark-sql_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+b1744080f5e2f81b9a1a088eb375d33b85132b6d
diff --git a/licenses/spark-streaming_2.12-3.4.3.jar.sha1 b/licenses/spark-streaming_2.12-3.4.3.jar.sha1
diff --git a/licenses/spark-streaming_2.13-3.4.3.jar.sha1 b/licenses/spark-streaming_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+64c4e38f3a89428926383f4d9f8a8636a81d499b
diff --git a/licenses/spark-yarn_2.12-3.4.3.jar.sha1 b/licenses/spark-yarn_2.12-3.4.3.jar.sha1
diff --git a/licenses/spark-yarn_2.13-3.4.3.jar.sha1 b/licenses/spark-yarn_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4
diff --git a/qa/kerberos/build.gradle b/qa/kerberos/build.gradle
@@ -63,22 +63,22 @@ configurations {
 dependencies {
     implementation(project(":elasticsearch-hadoop-mr"))
 
-    implementation("org.scala-lang:scala-library:${scala212Version}")
-    implementation("org.scala-lang:scala-reflect:${scala212Version}")
+    implementation("org.scala-lang:scala-library:${scala213Version}")
+    implementation("org.scala-lang:scala-reflect:${scala213Version}")
     implementation(project(":elasticsearch-spark-30")) {
         capabilities {
-            // Spark 3.x on Scala 2.12
-            requireCapability("org.elasticsearch.spark.sql.variant:spark30scala212:$project.version")
+            // Spark 3.x on Scala 2.13
+            requireCapability("org.elasticsearch.spark.sql.variant:spark30scala213:$project.version")
         }
     }
 
-    compileOnly("com.fasterxml.jackson.module:jackson-module-scala_2.12:2.6.7.1")
-    compileOnly("com.fasterxml.jackson.core:jackson-annotations:2.6.7")
-    compileOnly("org.json4s:json4s-jackson_2.12:3.2.11")
+    compileOnly("com.fasterxml.jackson.module:jackson-module-scala_${scala213MajorVersion}:2.9.10")
+    compileOnly("com.fasterxml.jackson.core:jackson-annotations:2.9.10")
+    compileOnly("org.json4s:json4s-jackson_${scala213MajorVersion}:3.6.7")
     compileOnly("org.slf4j:slf4j-api:${project.ext.slf4jVersion}")
 
     implementation("org.apache.hadoop:hadoop-client:${HadoopClusterConfiguration.HADOOP.defaultVersion()}")
-    implementation("org.apache.spark:spark-sql_2.12:$project.ext.spark30Version")
+    implementation("org.apache.spark:spark-sql_${scala213MajorVersion}:$project.ext.spark30Version")
 
     implementation( project(path: ':elasticsearch-hadoop-mr', configuration: 'itestJarConfig'))
 

diff --git a/spark/core/build.gradle b/spark/core/build.gradle
@@ -12,8 +12,8 @@ sparkVariants {
 
     // Changing the formatting of these lines could break .buildkite/pipeline.py, it uses regex to parse the `spark30scala212` part
     // We should maybe move these to a separate config file that can be read from both this file and the pipeline script in the future if it creates issues
-    setCoreDefaultVariant "spark30scala212", spark30Version, scala212Version
-    addCoreFeatureVariant "spark30scala213", spark30Version, scala213Version
+    setCoreDefaultVariant "spark30scala213", spark30Version, scala213Version
+    addCoreFeatureVariant "spark30scala212", spark30Version, scala212Version
 
     all { SparkVariantPlugin.SparkVariant variant ->
 

diff --git a/spark/core/licenses/scala-library-2.12.19.jar.sha1 b/spark/core/licenses/scala-library-2.12.19.jar.sha1
diff --git a/spark/core/licenses/scala-library-2.13.13.jar.sha1 b/spark/core/licenses/scala-library-2.13.13.jar.sha1
@@ -0,0 +1 @@
+3b1434403ddb147b8030f98f246153e4ce622c78
diff --git a/spark/core/licenses/scala-reflect-2.12.19.jar.sha1 b/spark/core/licenses/scala-reflect-2.12.19.jar.sha1
diff --git a/spark/core/licenses/scala-reflect-2.13.13.jar.sha1 b/spark/core/licenses/scala-reflect-2.13.13.jar.sha1
@@ -0,0 +1 @@
+46866f0e2cf6168322dc11f0f721c2db78d52ac7
diff --git a/spark/core/licenses/spark-core_2.12-3.4.3.jar.sha1 b/spark/core/licenses/spark-core_2.12-3.4.3.jar.sha1
diff --git a/spark/core/licenses/spark-core_2.13-3.4.3.jar.sha1 b/spark/core/licenses/spark-core_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+a8055d61a13e9af6871e05ecdca141134d102f1f
diff --git a/spark/sql-30/build.gradle b/spark/sql-30/build.gradle
@@ -9,8 +9,8 @@ apply plugin: 'spark.variants'
 
 sparkVariants {
     capabilityGroup 'org.elasticsearch.spark.sql.variant'
-    setDefaultVariant "spark30scala212", spark30Version, scala212Version
-    addFeatureVariant "spark30scala213", spark30Version, scala213Version
+    setDefaultVariant "spark30scala213", spark30Version, scala213Version
+    addFeatureVariant "spark30scala212", spark30Version, scala212Version
 
     all { SparkVariantPlugin.SparkVariant variant ->
         String scalaCompileTaskName = project.sourceSets

diff --git a/spark/sql-30/licenses/scala-library-2.12.19.jar.sha1 b/spark/sql-30/licenses/scala-library-2.12.19.jar.sha1
diff --git a/spark/sql-30/licenses/scala-library-2.13.13.jar.sha1 b/spark/sql-30/licenses/scala-library-2.13.13.jar.sha1
@@ -0,0 +1 @@
+3b1434403ddb147b8030f98f246153e4ce622c78
diff --git a/spark/sql-30/licenses/scala-reflect-2.12.19.jar.sha1 b/spark/sql-30/licenses/scala-reflect-2.12.19.jar.sha1
diff --git a/spark/sql-30/licenses/scala-reflect-2.13.13.jar.sha1 b/spark/sql-30/licenses/scala-reflect-2.13.13.jar.sha1
@@ -0,0 +1 @@
+46866f0e2cf6168322dc11f0f721c2db78d52ac7
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.4.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-catalyst_2.13-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+6a9e9c5e149bd019dd5d781888e1e81ca4d32875
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.4.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-core_2.13-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-core_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+a8055d61a13e9af6871e05ecdca141134d102f1f
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.4.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-sql_2.13-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+b1744080f5e2f81b9a1a088eb375d33b85132b6d
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.4.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-streaming_2.13-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+64c4e38f3a89428926383f4d9f8a8636a81d499b
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.4.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-yarn_2.13-3.4.3.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.13-3.4.3.jar.sha1
@@ -0,0 +1 @@
+d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4