Require JDK 9 for compilation (elastic#28071)

This commit modifies the build to require JDK 9 for
compilation. Henceforth, we will compile with a JDK 9 compiler targeting
JDK 8 as the class file format. Optionally, RUNTIME_JAVA_HOME can be set
as the runtime JDK used for running tests. To enable this change, we
separate the meaning of the compiler Java home versus the runtime Java
home. If the runtime Java home is not set (via RUNTIME_JAVA_HOME) then
we fallback to using JAVA_HOME as the runtime Java home. This enables:
 - developers only have to set one Java home (JAVA_HOME)
 - developers can set an optional Java home (RUNTIME_JAVA_HOME) to test
   on the minimum supported runtime
 - we can test compiling with JDK 9 running on JDK 8 and compiling with
   JDK 9 running on JDK 9 in CI

Author: jasontedor

CONTRIBUTING.md

@@ -92,6 +92,14 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
+JDK 9 is required to build Elasticsearch. You must have a JDK 9 installation
+with the environment variable `JAVA_HOME` referencing the path to Java home for
+your JDK 9 installation. By default, tests use the same runtime as `JAVA_HOME`.
+However, since Elasticsearch, supports JDK 8 the build supports compiling with
+JDK 9 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
+pointing to the Java home of a JDK 8 installation. Note that this mechanism can
+be used to test against other JDKs as well, this is not only limited to JDK 8.
+
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script in the root of the repository.
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -56,7 +56,8 @@ import java.time.ZonedDateTime
  */
 class BuildPlugin implements Plugin<Project> {
 
-    static final JavaVersion minimumJava = JavaVersion.VERSION_1_8
+    static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
+    static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_9
 
     @Override
     void apply(Project project) {
@@ -93,20 +94,26 @@ class BuildPlugin implements Plugin<Project> {
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
-            String javaHome = findJavaHome()
+            String compilerJavaHome = findCompilerJavaHome()
+            String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
             String javaVendor = System.getProperty('java.vendor')
             String javaVersion = System.getProperty('java.version')
             String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
                 " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"
 
-            String javaVersionDetails = gradleJavaVersionDetails
-            JavaVersion javaVersionEnum = JavaVersion.current()
-            if (new File(javaHome).canonicalPath != gradleJavaHome.canonicalPath) {
-                javaVersionDetails = findJavaVersionDetails(project, javaHome)
-                javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
-                javaVendor = findJavaVendor(project, javaHome)
-                javaVersion = findJavaVersion(project, javaHome)
+            String compilerJavaVersionDetails = gradleJavaVersionDetails
+            JavaVersion compilerJavaVersionEnum = JavaVersion.current()
+            if (new File(compilerJavaHome).canonicalPath != gradleJavaHome.canonicalPath) {
+                compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome)
+                compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome))
+            }
+
+            String runtimeJavaVersionDetails = gradleJavaVersionDetails
+            JavaVersion runtimeJavaVersionEnum = JavaVersion.current()
+            if (new File(runtimeJavaHome).canonicalPath != gradleJavaHome.canonicalPath) {
+                runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome)
+                runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
             }
 
             // Build debugging info
@@ -115,11 +122,13 @@ class BuildPlugin implements Plugin<Project> {
             println '======================================='
             println "  Gradle Version        : ${project.gradle.gradleVersion}"
             println "  OS Info               : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
-            if (gradleJavaVersionDetails != javaVersionDetails) {
+            if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
                 println "  JDK Version (gradle)  : ${gradleJavaVersionDetails}"
                 println "  JAVA_HOME (gradle)    : ${gradleJavaHome}"
-                println "  JDK Version (compile) : ${javaVersionDetails}"
-                println "  JAVA_HOME (compile)   : ${javaHome}"
+                println "  JDK Version (compile) : ${compilerJavaVersionDetails}"
+                println "  JAVA_HOME (compile)   : ${compilerJavaHome}"
+                println "  JDK Version (runtime) : ${runtimeJavaVersionDetails}"
+                println "  JAVA_HOME (runtime)   : ${runtimeJavaHome}"
             } else {
                 println "  JDK Version           : ${gradleJavaVersionDetails}"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
@@ -135,54 +144,47 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             // enforce Java version
-            if (javaVersionEnum < minimumJava) {
-                throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch")
+            if (compilerJavaVersionEnum < minimumCompilerVersion) {
+                throw new GradleException("Java ${minimumCompilerVersion} or above is required to build Elasticsearch")
             }
 
-            // this block of code detecting buggy JDK 8 compiler versions can be removed when minimum Java version is incremented
-            assert minimumJava == JavaVersion.VERSION_1_8 : "Remove JDK compiler bug detection only applicable to JDK 8"
-            if (javaVersionEnum == JavaVersion.VERSION_1_8) {
-                if (Objects.equals("Oracle Corporation", javaVendor)) {
-                    def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/
-                    if (matcher.matches()) {
-                        int update;
-                        if (matcher.group(1) == null) {
-                            update = 0
-                        } else {
-                            update = matcher.group(1).toInteger()
-                        }
-                        if (update < 40) {
-                            throw new GradleException("JDK ${javaVendor} ${javaVersion} has compiler bug JDK-8052388, update your JDK to at least 8u40")
-                        }
-                    }
-                }
+            if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
+                throw new GradleException("Java ${minimumRuntimeVersion} or above is required to run Elasticsearch")
             }
 
-            project.rootProject.ext.javaHome = javaHome
-            project.rootProject.ext.javaVersion = javaVersionEnum
+            project.rootProject.ext.compilerJavaHome = compilerJavaHome
+            project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
+            project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
+            project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
             project.rootProject.ext.buildChecksDone = true
         }
-        project.targetCompatibility = minimumJava
-        project.sourceCompatibility = minimumJava
+        project.targetCompatibility = minimumRuntimeVersion
+        project.sourceCompatibility = minimumRuntimeVersion
         // set java home for each project, so they dont have to find it in the root project
-        project.ext.javaHome = project.rootProject.ext.javaHome
-        project.ext.javaVersion = project.rootProject.ext.javaVersion
+        project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome
+        project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
+        project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
+        project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
     }
 
-    /** Finds and enforces JAVA_HOME is set */
-    private static String findJavaHome() {
-        String javaHome = System.getenv('JAVA_HOME')
+    private static String findCompilerJavaHome() {
+        final String javaHome = System.getenv('JAVA_HOME')
         if (javaHome == null) {
             if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
-                // intellij doesn't set JAVA_HOME, so we use the jdk gradle was run with
-                javaHome = Jvm.current().javaHome
+                // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
+                return Jvm.current().javaHome
             } else {
-                throw new GradleException('JAVA_HOME must be set to build Elasticsearch')
+                throw new GradleException("JAVA_HOME must be set to build Elasticsearch")
             }
         }
         return javaHome
     }
 
+    private static String findRuntimeJavaHome(final String compilerJavaHome) {
+        assert compilerJavaHome != null
+        return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
+    }
+
     /** Finds printable java version of the given JAVA_HOME */
     private static String findJavaVersionDetails(Project project, String javaHome) {
         String versionInfoScript = 'print(' +
@@ -412,7 +414,7 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Adds compiler settings to the project */
     static void configureCompile(Project project) {
-        if (project.javaVersion < JavaVersion.VERSION_1_10) {
+        if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
             project.ext.compactProfile = 'compact3'
         } else {
             project.ext.compactProfile = 'full'
@@ -422,7 +424,7 @@ class BuildPlugin implements Plugin<Project> {
                 File gradleJavaHome = Jvm.current().javaHome
                 // we fork because compiling lots of different classes in a shared jvm can eventually trigger GC overhead limitations
                 options.fork = true
-                options.forkOptions.executable = new File(project.javaHome, 'bin/javac')
+                options.forkOptions.javaHome = new File(project.compilerJavaHome)
                 options.forkOptions.memoryMaximumSize = "1g"
                 if (project.targetCompatibility >= JavaVersion.VERSION_1_8) {
                     // compile with compact 3 profile by default
@@ -447,22 +449,18 @@ class BuildPlugin implements Plugin<Project> {
 
                 options.encoding = 'UTF-8'
                 options.incremental = true
-
-                if (project.javaVersion == JavaVersion.VERSION_1_9) {
-                    // hack until gradle supports java 9's new "--release" arg
-                    assert minimumJava == JavaVersion.VERSION_1_8
-                    options.compilerArgs << '--release' << '8'
-                }
+                // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510)
+                options.compilerArgs << '--release' << project.targetCompatibility.majorVersion
             }
         }
     }
 
     static void configureJavadoc(Project project) {
         project.tasks.withType(Javadoc) {
-            executable = new File(project.javaHome, 'bin/javadoc')
+            executable = new File(project.compilerJavaHome, 'bin/javadoc')
         }
         configureJavadocJar(project)
-        if (project.javaVersion == JavaVersion.VERSION_1_10) {
+        if (project.compilerJavaVersion == JavaVersion.VERSION_1_10) {
             project.tasks.withType(Javadoc) { it.enabled = false }
             project.tasks.getByName('javadocJar').each { it.enabled = false }
         }
@@ -508,7 +506,7 @@ class BuildPlugin implements Plugin<Project> {
                         'X-Compile-Lucene-Version': VersionProperties.lucene,
                         'X-Compile-Elasticsearch-Snapshot': isSnapshot,
                         'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
-                        'Build-Java-Version': project.javaVersion)
+                        'Build-Java-Version': project.compilerJavaVersion)
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {
                     logger.warn('Building without git revision id.')
                     jarTask.manifest.attributes('Change': 'Unknown')
@@ -545,7 +543,7 @@ class BuildPlugin implements Plugin<Project> {
     /** Returns a closure of common configuration shared by unit and integration tests. */
     static Closure commonTestConfig(Project project) {
         return {
-            jvm "${project.javaHome}/bin/java"
+            jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
             ifNoTests 'fail'
             onNonEmptyWorkDirectory 'wipe'

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -169,7 +169,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             Files.copy(jarFile.resolveSibling(sourcesFileName), jarFile.resolveSibling(clientSourcesFileName),
                     StandardCopyOption.REPLACE_EXISTING)
 
-            if (project.javaVersion < JavaVersion.VERSION_1_10) {
+            if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
                 String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
                 String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
                 Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy

@@ -42,7 +42,7 @@ public class JarHellTask extends LoggedExec {
             inputs.files(classpath)
             dependsOn(classpath)
             description = "Runs CheckJarHell on ${classpath}"
-            executable = new File(project.javaHome, 'bin/java')
+            executable = new File(project.runtimeJavaHome, 'bin/java')
             doFirst({
                 /* JarHell doesn't like getting directories that don't exist but
                   gradle isn't especially careful about that. So we have to do it

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LoggerUsageTask.groovy

@@ -44,7 +44,7 @@ public class LoggerUsageTask extends LoggedExec {
         project.afterEvaluate {
             dependsOn(classpath)
             description = "Runs LoggerUsageCheck on ${classDirectories}"
-            executable = new File(project.javaHome, 'bin/java')
+            executable = new File(project.runtimeJavaHome, 'bin/java')
             if (classDirectories == null) {
                 // Default to main and test class files
                 List files = []

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy

@@ -80,7 +80,7 @@ public class NamingConventionsTask extends LoggedExec {
         FileCollection classpath = project.sourceSets.test.runtimeClasspath
         inputs.files(classpath)
         description = "Tests that test classes aren't misnamed or misplaced"
-        executable = new File(project.javaHome, 'bin/java')
+        executable = new File(project.runtimeJavaHome, 'bin/java')
         if (false == checkForTestsInMain) {
             /* This task is created by default for all subprojects with this
              * setting and there is no point in running it if the files don't

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -655,7 +655,7 @@ class ClusterFormationTasks {
                 String pid = node.pidFile.getText('UTF-8')
                 ByteArrayOutputStream output = new ByteArrayOutputStream()
                 project.exec {
-                    commandLine = ["${project.javaHome}/bin/jstack", pid]
+                    commandLine = ["${project.runtimeJavaHome}/bin/jstack", pid]
                     standardOutput = output
                 }
                 output.toString('UTF-8').eachLine { line -> logger.error("|    ${line}") }
@@ -699,7 +699,7 @@ class ClusterFormationTasks {
     }
 
     private static File getJpsExecutableByName(Project project, String jpsExecutableName) {
-        return Paths.get(project.javaHome.toString(), "bin/" + jpsExecutableName).toFile()
+        return Paths.get(project.runtimeJavaHome.toString(), "bin/" + jpsExecutableName).toFile()
     }
 
     /** Adds a task to kill an elasticsearch node with the given pidfile */

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -162,7 +162,7 @@ class NodeInfo {
             args.add("${esScript}")
         }
 
-        env = ['JAVA_HOME': project.javaHome]
+        env = ['JAVA_HOME': project.runtimeJavaHome]
         args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs

distribution/bwc/build.gradle

@@ -20,6 +20,7 @@
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+
 import java.util.regex.Matcher
 
 /**
@@ -118,29 +119,31 @@ if (project.hasProperty('bwcVersion')) {
   task buildBwcVersion(type: Exec) {
     dependsOn checkoutBwcBranch, writeBuildMetadata
     workingDir = checkoutDir
+    if (project.rootProject.ext.runtimeJavaVersion == JavaVersion.VERSION_1_8 && ["5.6", "6.0", "6.1"].contains(bwcBranch)) {
+      /*
+       * If runtime Java home is set to JDK 8 and we are building branches that are officially built with JDK 8, push this to JAVA_HOME for
+       * these builds.
+       */
+      environment('JAVA_HOME', System.getenv('RUNTIME_JAVA_HOME'))
+    }
     if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       executable 'cmd'
       args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
     } else {
-      executable = new File(checkoutDir, 'gradlew').toString()
+      executable new File(checkoutDir, 'gradlew').toString()
     }
-    final ArrayList<String> commandLineArgs = [
-            ":distribution:deb:assemble",
-            ":distribution:rpm:assemble",
-            ":distribution:zip:assemble",
-            "-Dbuild.snapshot=${System.getProperty('build.snapshot') ?: 'true'}"]
+    args ":distribution:deb:assemble", ":distribution:rpm:assemble", ":distribution:zip:assemble", "-Dbuild.snapshot=${System.getProperty('build.snapshot') ?: 'true'}"
     final LogLevel logLevel = gradle.startParameter.logLevel
     if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
-      commandLineArgs << "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
+      args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
     }
     final String showStacktraceName = gradle.startParameter.showStacktrace.name()
     assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
     if (showStacktraceName.equals("ALWAYS")) {
-      commandLineArgs << "--stacktrace"
+      args "--stacktrace"
     } else if (showStacktraceName.equals("ALWAYS_FULL")) {
-      commandLineArgs << "--full-stacktrace"
+      args "--full-stacktrace"
     }
-    args commandLineArgs
     doLast {
       List missing = [bwcDeb, bwcRpm, bwcZip].grep { file ->
         false == file.exists()

plugins/discovery-azure-classic/build.gradle

@@ -67,7 +67,7 @@ task createKey(type: LoggedExec) {
     project.delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
   }
-  executable = new File(project.javaHome, 'bin/keytool')
+  executable = new File(project.runtimeJavaHome, 'bin/keytool')
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
           '-alias', 'test-node',

plugins/discovery-ec2/build.gradle

@@ -75,13 +75,11 @@ thirdPartyAudit.excludes = [
   'software.amazon.ion.system.IonSystemBuilder',
   'software.amazon.ion.system.IonTextWriterBuilder',
   'software.amazon.ion.system.IonWriterBuilder',
+  'javax.xml.bind.DatatypeConverter',
+  'javax.xml.bind.JAXBContext',
   'javax.servlet.ServletContextEvent', 
   'javax.servlet.ServletContextListener', 
   'org.apache.avalon.framework.logger.Logger', 
   'org.apache.log.Hierarchy', 
   'org.apache.log.Logger',
 ]
-
-if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
-  thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
-}

plugins/discovery-gce/build.gradle

@@ -35,7 +35,7 @@ task createKey(type: LoggedExec) {
     project.delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
   }
-  executable = new File(project.javaHome, 'bin/keytool')
+  executable = new File(project.runtimeJavaHome, 'bin/keytool')
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
           '-alias', 'test-node',