Builds with java 17 (#1802)

This commit adds support for building on java 17, while leaving support for building java 11 untouched.

Author: masseyke

hive/build.gradle

@@ -58,6 +58,12 @@ dependencies {
     javadocSources(project(":elasticsearch-hadoop-mr"))
 }
 
+tasks.named("test").configure {
+    onlyIf { JavaVersion.current() >= JavaVersion.VERSION_16 }
+    jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+    jvmArgs "--add-opens=java.base/java.net=ALL-UNNAMED" // Needed for org.apache.hadoop.hive.common.StringInternUtils
+}
+
 jar {
     dependsOn(project.configurations.embedded)
     from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {

pig/build.gradle

@@ -47,6 +47,11 @@ dependencies {
     javadocSources(project(":elasticsearch-hadoop-mr"))
 }
 
+tasks.named("test").configure {
+    onlyIf { JavaVersion.current() >= JavaVersion.VERSION_16 }
+    jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+}
+
 jar {
     dependsOn(project.configurations.embedded)
     from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {

spark/core/build.gradle

@@ -26,7 +26,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -143,8 +142,14 @@ configurations.matching{ it.name.contains('CompilerPlugin') == false }.all { Con
     conf.exclude group: "org.mortbay.jetty"
 }
 
-tasks.named("test").configure {
-    jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+if (JavaVersion.current() >= JavaVersion.VERSION_17) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test")) 
+            task.configure {
+                jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+                jvmArgs "--add-opens=java.base/java.nio=ALL-UNNAMED" // Needed for org.apache.spark.SparkConf, which indirectly uses java.nio.DirectByteBuffer
+            }
+    }
 }
 
 // Set minimum compatibility and java home for compiler task

spark/sql-13/build.gradle

@@ -36,7 +36,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -174,6 +173,14 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
 }
 
+if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test"))
+        task.configure {
+            jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+        }}
+}
+
 // Embed the embedded dependencies in the final jar after all configuration is complete
 sparkVariants {
     all { SparkVariantPlugin.SparkVariant variant ->

spark/sql-20/build.gradle

@@ -32,7 +32,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -194,6 +193,14 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
 }
 
+if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test"))
+        task.configure {
+            jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+        }}
+}
+
 // Embed the embedded dependencies in the final jar after all configuration is complete
 sparkVariants {
     all { SparkVariantPlugin.SparkVariant variant ->

spark/sql-30/build.gradle

@@ -30,7 +30,6 @@ sparkVariants {
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {
                 memoryMaximumSize = '1g'
-                jvmArgs = ['-XX:MaxPermSize=512m']
             }
             compileScala.scalaCompileOptions.additionalParameters = [
                     "-feature",
@@ -177,6 +176,15 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
 }
 
+if (JavaVersion.current() >= JavaVersion.VERSION_16) {
+    tasks.withType(Test) { Test task ->
+        if (task.getName().startsWith("test"))
+        task.configure {
+            jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+            jvmArgs "--add-opens=java.base/java.nio=ALL-UNNAMED" // Needed for org.apache.spark.SparkConf, which indirectly uses java.nio.DirectByteBuffer
+        }}
+}
+
 // Embed the embedded dependencies in the final jar after all configuration is complete
 sparkVariants {
     all { SparkVariantPlugin.SparkVariant variant ->

test/shared/build.gradle

@@ -58,6 +58,11 @@ configurations {
     }
 }
 
+tasks.named("test").configure {
+    onlyIf { JavaVersion.current() >= JavaVersion.VERSION_16 }
+    jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED" // Needed for IOUtils's BYTE_ARRAY_BUFFER reflection
+}
+
 dependencies {
     api(project(':elasticsearch-hadoop-mr'))
     api(project.ext.hadoopClient) {