Deprecating old technology (#1845) (#1876)

This commit deprecates spark 1.x, scala 2.10 on spark 2.x, hadoop 1, pig, and storm. Support will be removed for these things at a later date.

Author: masseyke

docs/src/reference/asciidoc/appendix/breaking.adoc

@@ -23,10 +23,40 @@ We've increased the `es.scroll.size` default from `50` to `1000`. If you deploy
 ES-hadoop in a low memory environment, consider lowering `es.scroll.size` to
 avoid issues.
 
-==== Support for Scala 2.10 has been removed
+// end::notable-v8-breaking-changes[]
+
+[[deprecations-8.0]]
+=== Deprecations in 8.0
+
+The following functionality has been deprecated in {eh} 8.0 and will be removed
+in a future version. While this won’t have an immediate impact on your
+applications, we strongly encourage you take the described steps to update your
+code after upgrading to 8.0.
+
+// tag::notable-v8-breaking-changes[]
+
+==== Spark 1.x support is deprecated
+
+Spark 1.x is no longer maintained. Spark 2 and Spark 3 are still supported.
+
+==== Scala 2.10 support for Spark 2.x is deprecated
+
+Spark deprecated support for Scala 2.10 in the Spark 2.0.0 release and removed it in the 2.3.0 release. Scala 2.11 and 2.12 are supported
+for Spark 2.x.
+
+==== Hadoop 1.x support is deprecated
+
+Hadoop 1.x is no longer maintained. Support for it has not been tested since {eh} 6.0. Support is now formally deprecated. Hadoop 2 and
+Hadoop 3 are supported.
+
+==== Apache Pig support is deprecated
+
+Apache Pig is no longer maintained.
+
+==== Apache Storm support is deprecated
+
+Apache Storm has not been a popular {es} integration.
 
-Our compatibility artifacts no longer support Scala 2.10. Spark deprecated
-support for Scala 2.10 in the Spark 2.0.0 release.
 // end::notable-v8-breaking-changes[]
 
 [[breaking-changes-7.0]]

docs/src/reference/asciidoc/core/intro/requirements.adoc

@@ -85,8 +85,9 @@ Hive version {hv-v}
 [[requirements-pig]]
 === Apache Pig
 
-Pig 0.10.0 or higher. We recommend using the latest release of Pig (currently {pg-v}).
+deprecated::[8.0,Support for Apache Pig in {eh} is deprecated.]
 
+Pig 0.10.0 or higher. We recommend using the latest release of Pig (currently {pg-v}).
 In a similar fashion, Pig version can be discovered from its folder path or through the command-line:
 
 ["source","bash",subs="attributes"]
@@ -98,8 +99,9 @@ Apache Pig version {pg-v}
 [[requirements-spark]]
 === Apache Spark
 
-Spark 1.3.0 or higher. We recommend using the latest release of Spark (currently {sp-v}). As {eh} provides
-native integration (which is recommended) with {sp} it does not matter what binary one is using.
+deprecated::[8.0,Support for Spark 1.x in {eh} is deprecated.]
+
+Spark 1.3.0 or higher. We recommend using the latest release of Spark (currently {sp-v}). As {eh} provides native integration (which is recommended) with {sp}, it does not matter what binary one is using.
 The same applies when using the Hadoop layer to integrate the two as {eh} supports the majority of
 Hadoop distributions out there.
 
@@ -143,9 +145,10 @@ Note that Spark 1.0-1.2 are no longer supported (again due to backwards incompat
 [[requirements-storm]]
 === Apache Storm
 
+deprecated::[8.0,Support for Apache Storm in {eh} is deprecated.]
+
 Storm 1.0.0 or higher. Do note that Storm 1.0.0 broke backwards compatibility with the previous versions (by changing the package name)
 however upgrading is easy and recommended. We recommend using the latest release of Storm (currently {st-v}).
-
 One can discover the Storm version by looking at its folder or by invoking the command:
 
 ["source","bash",subs="attributes"]

mr/src/main/java/org/elasticsearch/hadoop/mr/EsInputFormat.java

@@ -408,6 +408,7 @@ public EsInputRecordReader<K, V> createRecordReader(InputSplit split, TaskAttemp
 
     // Note: data written to the JobConf will be silently discarded
     @Override
+    @Deprecated // Hadoop 1 support is deprecated
     public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
 
         Settings settings = HadoopSettingsManager.loadFrom(job);
@@ -424,6 +425,7 @@ public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplit
 
     @SuppressWarnings("unchecked")
     @Override
+    @Deprecated // Hadoop 1 support is deprecated
     public EsInputRecordReader<K, V> getRecordReader(org.apache.hadoop.mapred.InputSplit split, JobConf job, Reporter reporter) {
         return (EsInputRecordReader<K, V>) (isOutputAsJson(job) ? new JsonWritableEsInputRecordReader(split, job, reporter) : new WritableEsInputRecordReader(split, job, reporter));
     }

pig/src/main/java/org/elasticsearch/hadoop/pig/EsStorage.java

@@ -118,6 +118,7 @@ public EsStorage(String... configuration) {
                 throw new EsHadoopIllegalArgumentException("Cannot parse options " + Arrays.toString(configuration), ex);
             }
         }
+        log.warn("Support for Apache Pig has been deprecated and will be removed in a future release.");
     }
 
     @Override

spark/sql-13/src/main/scala/org/elasticsearch/spark/sql/EsSparkSQL.scala

@@ -34,6 +34,7 @@ import org.elasticsearch.hadoop.util.ObjectUtils
 import org.elasticsearch.spark.cfg.SparkSettingsManager
 import org.elasticsearch.hadoop.rest.InitializationUtils
 
+@deprecated(message="Support for Apache Spark 1 is deprecated. Use Spark 2 or 3.")
 object EsSparkSQL {
 
   private val init = { ObjectUtils.loadClass("org.elasticsearch.spark.rdd.CompatUtils", classOf[ObjectUtils].getClassLoader) }

spark/sql-13/src/main/scala/org/elasticsearch/spark/sql/package.scala

@@ -26,8 +26,10 @@ import org.elasticsearch.spark.sql.EsSparkSQL
 
 package object sql {
 
+  @deprecated(message="Support for Apache Spark 1 is deprecated. Use Spark 2 or 3.")
   implicit def sqlContextFunctions(sc: SQLContext)= new SQLContextFunctions(sc)
 
+  @deprecated(message="Support for Apache Spark 1 is deprecated. Use Spark 2 or 3.")
   class SQLContextFunctions(sc: SQLContext) extends Serializable {
     def esDF() = EsSparkSQL.esDF(sc)
     def esDF(resource: String) = EsSparkSQL.esDF(sc, resource)
@@ -37,8 +39,10 @@ package object sql {
     def esDF(resource: String, query: String, cfg: scala.collection.Map[String, String]) = EsSparkSQL.esDF(sc, resource, query, cfg)
   }
 
+  @deprecated(message="Support for Apache Spark 1 is deprecated. Use Spark 2 or 3.")
   implicit def sparkDataFrameFunctions(df: DataFrame) = new SparkDataFrameFunctions(df)
 
+  @deprecated(message="Support for Apache Spark 1 is deprecated. Use Spark 2 or 3.")
   class SparkDataFrameFunctions(df: DataFrame) extends Serializable {
     def saveToEs(resource: String): Unit = { EsSparkSQL.saveToEs(df, resource) }
     def saveToEs(resource: String, cfg: scala.collection.Map[String, String]): Unit = { EsSparkSQL.saveToEs(df, resource, cfg) }

spark/sql-13/src/main/scala/org/elasticsearch/spark/streaming/EsSparkStreaming.scala

@@ -27,6 +27,7 @@ import org.elasticsearch.spark.rdd.EsSpark
 
 import scala.collection.Map
 
+@deprecated(message="Support for Apache Spark 1 is deprecated. Use Spark 2 or 3.")
 object EsSparkStreaming {
 
   // Save methods

storm/src/main/java/org/elasticsearch/storm/EsBolt.java

@@ -46,7 +46,11 @@
 import static org.elasticsearch.hadoop.cfg.ConfigurationOptions.*;
 import static org.elasticsearch.storm.cfg.StormConfigurationOptions.ES_STORM_BOLT_ACK;
 
+/**
+ * @deprecated Support for Apache Storm is deprecated and will be removed in the future. Consider moving to Spark or Mapreduce.
+ */
 @SuppressWarnings({ "rawtypes", "unchecked" })
+@Deprecated
 public class EsBolt implements IRichBolt {
 
     private transient static Log log = LogFactory.getLog(EsBolt.class);
@@ -74,6 +78,7 @@ public EsBolt(String target, Map configuration) {
     }
 
     private EsBolt(String target, Boolean writeAck, Map configuration) {
+        log.warn("Support for Apache Storm has been deprecated and will be removed in a future release.");
         boltConfig.put(ES_RESOURCE_WRITE, target);
 
         if (writeAck != null) {

storm/src/main/java/org/elasticsearch/storm/EsSpout.java

@@ -48,7 +48,11 @@
 
 import static org.elasticsearch.hadoop.cfg.ConfigurationOptions.*;
 
+/**
+ * @deprecated Support for Apache Storm is deprecated and will be removed in the future. Consider moving to Spark or Mapreduce.
+ */
 @SuppressWarnings({ "rawtypes", "unchecked" })
+@Deprecated
 public class EsSpout implements IRichSpout {
 
     private transient static Log log = LogFactory.getLog(EsSpout.class);
@@ -82,6 +86,7 @@ public EsSpout(String target, Map configuration) {
     }
 
     public EsSpout(String target, String query, Map configuration) {
+        log.warn("Support for Apache Storm has been deprecated and will be removed in a future release.");
         if (configuration != null) {
             spoutConfig.putAll(configuration);
         }