Feature/remove spark 2 9.0

dist/build.gradle

@@ -12,8 +12,6 @@ base {
   archivesName = 'elasticsearch-hadoop'
 }
 
-def sparkVariantIncluded = 'spark20scala211'
-
 configurations {
     embedded {
         canBeResolved = true
@@ -47,28 +45,20 @@ configurations {
 
 BuildPlugin.disableTransitiveDependencies(project, project.configurations.thirdPartyShaded)
 
-def distProjects = [":elasticsearch-hadoop-mr", ":elasticsearch-hadoop-hive",
-                    ":elasticsearch-spark-20"]
+def distProjects = [":elasticsearch-hadoop-mr", ":elasticsearch-hadoop-hive"]
 
 distProjects.each { distProject ->
-    def configureDistDependency = { Dependency dependency ->
-        if (distProject == ":elasticsearch-spark-20") {
-            dependency.capabilities {
-                requireCapability("org.elasticsearch.spark.sql.variant:$sparkVariantIncluded:$project.version")
-            }
-        }
-    }
     dependencies {
         // This is only going to pull in each project's regular jar to create the project-wide uberjar.
-        add('embedded', project(distProject), configureDistDependency)
+        add('embedded', project(distProject))
         // To squash Javadoc warnings.
-        add('javadocDependencies', project(distProject), configureDistDependency)
+        add('javadocDependencies', project(distProject))
         // This will pull all java sources (including generated) for the project-wide javadoc.
-        add('javadocSources', project(distProject), configureDistDependency)
+        add('javadocSources', project(distProject))
         // This will pull all non-generated sources for the project-wide source jar.
-        add('additionalSources', project(distProject), configureDistDependency)
+        add('additionalSources', project(distProject))
         // This will pull in the regular jar, javadoc jar, and source jar to be packaged in the distribution.
-        add('dist', project(distProject), configureDistDependency)
+        add('dist', project(distProject))
     }
 }
 

docs/src/reference/asciidoc/appendix/breaking.adoc

@@ -8,6 +8,17 @@ For clarity, we always list any breaking changes at the top of the
 //NOTE: The notable-breaking-changes tagged regions are re-used in the
 //Installation and Upgrade Guide
 
+=== Deprecations in 8.18
+
+The following functionality has been deprecated in {eh} 8.18 and will be removed
+in a future version. While this won’t have an immediate impact on your
+applications, we strongly encourage you take the described steps to update your
+code after upgrading to 8.18.
+
+==== Spark 2.x support is deprecated
+
+Spark 2.x is no longer maintained. Spark 3 are still supported.
+
 [[breaking-changes-8.9]]
 === Breaking Changes in 8.9
 

docs/src/reference/asciidoc/core/intro/download.adoc

@@ -69,8 +69,8 @@ The Spark connector framework is the most sensitive to version incompatibilities
 | Spark Version | Scala Version | ES-Hadoop Artifact ID
 | 1.0 - 2.x | 2.10 | <unsupported>
 | 1.0 - 1.6 | 2.11 | <unsupported>
-| 2.x | 2.11 | elasticsearch-spark-20_2.11
-| 2.x | 2.12 | elasticsearch-spark-20_2.12
+| 2.x | 2.11 | <unsupported>
+| 2.x | 2.12 | <unsupported>
 | 3.0+ | 2.12 | elasticsearch-spark-30_2.12
 | 3.2+ | 2.13 | elasticsearch-spark-30_2.13
 |==========================================================

docs/src/reference/asciidoc/core/intro/requirements.adoc

@@ -81,6 +81,8 @@ Hive version {hv-v}
 [[requirements-spark]]
 === Apache Spark
 
+deprecated::[9.0,Support for Spark 2.x in {eh} is deprecated.]
+
 Spark 2.0 or higher. We recommend using the latest release of Spark (currently {sp-v}). As {eh} provides native integration (which is recommended) with {sp}, it does not matter what binary one is using.
 The same applies when using the Hadoop layer to integrate the two as {eh} supports the majority of
 Hadoop distributions out there.

settings.gradle

@@ -17,10 +17,6 @@ include 'spark-core'
 project(":spark-core").projectDir = new File(settingsDir, "spark/core")
 project(":spark-core").name = "elasticsearch-spark"
 
-include 'sql-20'
-project(":sql-20").projectDir = new File(settingsDir, "spark/sql-20")
-project(":sql-20").name = "elasticsearch-spark-20"
-
 include 'sql-30'
 project(":sql-30").projectDir = new File(settingsDir, "spark/sql-30")
 project(":sql-30").name = "elasticsearch-spark-30"

spark/core/build.gradle

@@ -10,12 +10,10 @@ apply plugin: 'spark.variants'
 sparkVariants {
     capabilityGroup 'org.elasticsearch.spark.variant'
 
-    // Changing the formatting of these lines could break .buildkite/pipeline.py, it uses regex to parse the `spark20scala212` part
+    // Changing the formatting of these lines could break .buildkite/pipeline.py, it uses regex to parse the `spark30scala212` part
     // We should maybe move these to a separate config file that can be read from both this file and the pipeline script in the future if it creates issues
-    setCoreDefaultVariant "spark20scala212", spark24Version, scala212Version
+    setCoreDefaultVariant "spark30scala212", spark30Version, scala212Version
     addCoreFeatureVariant "spark30scala213", spark30Version, scala213Version
-    addCoreFeatureVariant "spark30scala212", spark30Version, scala212Version
-    addCoreFeatureVariant "spark20scala211", spark24Version, scala211Version
 
     all { SparkVariantPlugin.SparkVariant variant ->
 

spark/core/src/itest/scala/org/elasticsearch/spark/integration/AbstractScalaEsSpark.scala

@@ -854,10 +854,10 @@ class AbstractScalaEsScalaSpark(prefix: String, readMetadata: jl.Boolean) extend
     val target = resource(index, typename, version)
 
     val rawCore = List( Map("colint" -> 1, "colstr" -> "s"),
-                         Map("colint" -> null, "colstr" -> null) )
+                         Map("colint" -> 9, "colstr" -> null) )
     sc.parallelize(rawCore, 1).saveToEs(target)
     val qjson =
-      """{"query":{"range":{"colint":{"from":null,"to":"9","include_lower":true,"include_upper":true}}}}"""
+      """{"query":{"range":{"colint":{"lte":"9"}}}}"""
 
     val esRDD = EsSpark.esRDD(sc, target, qjson)
     val scRDD = sc.esRDD(target, qjson)