Merge branch 'master' into ccr

* master:
  Fix third-party audit tasks on JDK 8
  Remove duplicated javadoc `fieldType` param
  Handle 5.6.6 and 6.1.2 release
  Introduce multi-release JAR
  Move the multi-get response tests to server
  Require JDK 9 for compilation (#28071)
  Revert "[Docs] Fix Java Api index administration usage (#28133)"
  Revert "[Docs] Fix base directory to include for put_mapping.asciidoc"
  Added multi get api to the high level rest client.
  [Docs] Clarify numeric datatype ranges (#28240)
  [Docs] Fix base directory to include for put_mapping.asciidoc
  Open engine should keep only starting commit (#28228)

Author: jasontedor

CONTRIBUTING.md

@@ -92,6 +92,14 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
+JDK 9 is required to build Elasticsearch. You must have a JDK 9 installation
+with the environment variable `JAVA_HOME` referencing the path to Java home for
+your JDK 9 installation. By default, tests use the same runtime as `JAVA_HOME`.
+However, since Elasticsearch, supports JDK 8 the build supports compiling with
+JDK 9 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
+pointing to the Java home of a JDK 8 installation. Note that this mechanism can
+be used to test against other JDKs as well, this is not only limited to JDK 8.
+
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script in the root of the repository.
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -56,7 +56,8 @@ import java.time.ZonedDateTime
  */
 class BuildPlugin implements Plugin<Project> {
 
-    static final JavaVersion minimumJava = JavaVersion.VERSION_1_8
+    static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
+    static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_9
 
     @Override
     void apply(Project project) {
@@ -93,20 +94,26 @@ class BuildPlugin implements Plugin<Project> {
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
-            String javaHome = findJavaHome()
+            String compilerJavaHome = findCompilerJavaHome()
+            String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
             String javaVendor = System.getProperty('java.vendor')
             String javaVersion = System.getProperty('java.version')
             String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
                 " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"
 
-            String javaVersionDetails = gradleJavaVersionDetails
-            JavaVersion javaVersionEnum = JavaVersion.current()
-            if (new File(javaHome).canonicalPath != gradleJavaHome.canonicalPath) {
-                javaVersionDetails = findJavaVersionDetails(project, javaHome)
-                javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
-                javaVendor = findJavaVendor(project, javaHome)
-                javaVersion = findJavaVersion(project, javaHome)
+            String compilerJavaVersionDetails = gradleJavaVersionDetails
+            JavaVersion compilerJavaVersionEnum = JavaVersion.current()
+            if (new File(compilerJavaHome).canonicalPath != gradleJavaHome.canonicalPath) {
+                compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome)
+                compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome))
+            }
+
+            String runtimeJavaVersionDetails = gradleJavaVersionDetails
+            JavaVersion runtimeJavaVersionEnum = JavaVersion.current()
+            if (new File(runtimeJavaHome).canonicalPath != gradleJavaHome.canonicalPath) {
+                runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome)
+                runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
             }
 
             // Build debugging info
@@ -115,11 +122,13 @@ class BuildPlugin implements Plugin<Project> {
             println '======================================='
             println "  Gradle Version        : ${project.gradle.gradleVersion}"
             println "  OS Info               : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
-            if (gradleJavaVersionDetails != javaVersionDetails) {
+            if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
                 println "  JDK Version (gradle)  : ${gradleJavaVersionDetails}"
                 println "  JAVA_HOME (gradle)    : ${gradleJavaHome}"
-                println "  JDK Version (compile) : ${javaVersionDetails}"
-                println "  JAVA_HOME (compile)   : ${javaHome}"
+                println "  JDK Version (compile) : ${compilerJavaVersionDetails}"
+                println "  JAVA_HOME (compile)   : ${compilerJavaHome}"
+                println "  JDK Version (runtime) : ${runtimeJavaVersionDetails}"
+                println "  JAVA_HOME (runtime)   : ${runtimeJavaHome}"
             } else {
                 println "  JDK Version           : ${gradleJavaVersionDetails}"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
@@ -135,54 +144,49 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             // enforce Java version
-            if (javaVersionEnum < minimumJava) {
-                throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch")
+            if (compilerJavaVersionEnum < minimumCompilerVersion) {
+                throw new GradleException("Java ${minimumCompilerVersion} or above is required to build Elasticsearch")
             }
 
-            // this block of code detecting buggy JDK 8 compiler versions can be removed when minimum Java version is incremented
-            assert minimumJava == JavaVersion.VERSION_1_8 : "Remove JDK compiler bug detection only applicable to JDK 8"
-            if (javaVersionEnum == JavaVersion.VERSION_1_8) {
-                if (Objects.equals("Oracle Corporation", javaVendor)) {
-                    def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/
-                    if (matcher.matches()) {
-                        int update;
-                        if (matcher.group(1) == null) {
-                            update = 0
-                        } else {
-                            update = matcher.group(1).toInteger()
-                        }
-                        if (update < 40) {
-                            throw new GradleException("JDK ${javaVendor} ${javaVersion} has compiler bug JDK-8052388, update your JDK to at least 8u40")
-                        }
-                    }
-                }
+            if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
+                throw new GradleException("Java ${minimumRuntimeVersion} or above is required to run Elasticsearch")
             }
 
-            project.rootProject.ext.javaHome = javaHome
-            project.rootProject.ext.javaVersion = javaVersionEnum
+            project.rootProject.ext.compilerJavaHome = compilerJavaHome
+            project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
+            project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
+            project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
             project.rootProject.ext.buildChecksDone = true
         }
-        project.targetCompatibility = minimumJava
-        project.sourceCompatibility = minimumJava
+
+        project.targetCompatibility = minimumRuntimeVersion
+        project.sourceCompatibility = minimumRuntimeVersion
+
         // set java home for each project, so they dont have to find it in the root project
-        project.ext.javaHome = project.rootProject.ext.javaHome
-        project.ext.javaVersion = project.rootProject.ext.javaVersion
+        project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome
+        project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
+        project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
+        project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
     }
 
-    /** Finds and enforces JAVA_HOME is set */
-    private static String findJavaHome() {
-        String javaHome = System.getenv('JAVA_HOME')
+    private static String findCompilerJavaHome() {
+        final String javaHome = System.getenv('JAVA_HOME')
         if (javaHome == null) {
             if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
-                // intellij doesn't set JAVA_HOME, so we use the jdk gradle was run with
-                javaHome = Jvm.current().javaHome
+                // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
+                return Jvm.current().javaHome
             } else {
-                throw new GradleException('JAVA_HOME must be set to build Elasticsearch')
+                throw new GradleException("JAVA_HOME must be set to build Elasticsearch")
             }
         }
         return javaHome
     }
 
+    private static String findRuntimeJavaHome(final String compilerJavaHome) {
+        assert compilerJavaHome != null
+        return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
+    }
+
     /** Finds printable java version of the given JAVA_HOME */
     private static String findJavaVersionDetails(Project project, String javaHome) {
         String versionInfoScript = 'print(' +
@@ -412,19 +416,19 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Adds compiler settings to the project */
     static void configureCompile(Project project) {
-        if (project.javaVersion < JavaVersion.VERSION_1_10) {
+        if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
             project.ext.compactProfile = 'compact3'
         } else {
             project.ext.compactProfile = 'full'
         }
         project.afterEvaluate {
             project.tasks.withType(JavaCompile) {
-                File gradleJavaHome = Jvm.current().javaHome
+                final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(it.targetCompatibility)
                 // we fork because compiling lots of different classes in a shared jvm can eventually trigger GC overhead limitations
                 options.fork = true
-                options.forkOptions.executable = new File(project.javaHome, 'bin/javac')
+                options.forkOptions.javaHome = new File(project.compilerJavaHome)
                 options.forkOptions.memoryMaximumSize = "1g"
-                if (project.targetCompatibility >= JavaVersion.VERSION_1_8) {
+                if (targetCompatibilityVersion == JavaVersion.VERSION_1_8) {
                     // compile with compact 3 profile by default
                     // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
                     if (project.compactProfile != 'full') {
@@ -448,21 +452,18 @@ class BuildPlugin implements Plugin<Project> {
                 options.encoding = 'UTF-8'
                 options.incremental = true
 
-                if (project.javaVersion == JavaVersion.VERSION_1_9) {
-                    // hack until gradle supports java 9's new "--release" arg
-                    assert minimumJava == JavaVersion.VERSION_1_8
-                    options.compilerArgs << '--release' << '8'
-                }
+                // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510)
+                options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion
             }
         }
     }
 
     static void configureJavadoc(Project project) {
         project.tasks.withType(Javadoc) {
-            executable = new File(project.javaHome, 'bin/javadoc')
+            executable = new File(project.compilerJavaHome, 'bin/javadoc')
         }
         configureJavadocJar(project)
-        if (project.javaVersion == JavaVersion.VERSION_1_10) {
+        if (project.compilerJavaVersion == JavaVersion.VERSION_1_10) {
             project.tasks.withType(Javadoc) { it.enabled = false }
             project.tasks.getByName('javadocJar').each { it.enabled = false }
         }
@@ -508,7 +509,7 @@ class BuildPlugin implements Plugin<Project> {
                         'X-Compile-Lucene-Version': VersionProperties.lucene,
                         'X-Compile-Elasticsearch-Snapshot': isSnapshot,
                         'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
-                        'Build-Java-Version': project.javaVersion)
+                        'Build-Java-Version': project.compilerJavaVersion)
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {
                     logger.warn('Building without git revision id.')
                     jarTask.manifest.attributes('Change': 'Unknown')
@@ -545,7 +546,7 @@ class BuildPlugin implements Plugin<Project> {
     /** Returns a closure of common configuration shared by unit and integration tests. */
     static Closure commonTestConfig(Project project) {
         return {
-            jvm "${project.javaHome}/bin/java"
+            jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
             ifNoTests 'fail'
             onNonEmptyWorkDirectory 'wipe'

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -169,7 +169,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             Files.copy(jarFile.resolveSibling(sourcesFileName), jarFile.resolveSibling(clientSourcesFileName),
                     StandardCopyOption.REPLACE_EXISTING)
 
-            if (project.javaVersion < JavaVersion.VERSION_1_10) {
+            if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
                 String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
                 String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
                 Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy

@@ -42,7 +42,7 @@ public class JarHellTask extends LoggedExec {
             inputs.files(classpath)
             dependsOn(classpath)
             description = "Runs CheckJarHell on ${classpath}"
-            executable = new File(project.javaHome, 'bin/java')
+            executable = new File(project.runtimeJavaHome, 'bin/java')
             doFirst({
                 /* JarHell doesn't like getting directories that don't exist but
                   gradle isn't especially careful about that. So we have to do it

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LoggerUsageTask.groovy

@@ -44,7 +44,7 @@ public class LoggerUsageTask extends LoggedExec {
         project.afterEvaluate {
             dependsOn(classpath)
             description = "Runs LoggerUsageCheck on ${classDirectories}"
-            executable = new File(project.javaHome, 'bin/java')
+            executable = new File(project.runtimeJavaHome, 'bin/java')
             if (classDirectories == null) {
                 // Default to main and test class files
                 List files = []

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy

@@ -80,7 +80,7 @@ public class NamingConventionsTask extends LoggedExec {
         FileCollection classpath = project.sourceSets.test.runtimeClasspath
         inputs.files(classpath)
         description = "Tests that test classes aren't misnamed or misplaced"
-        executable = new File(project.javaHome, 'bin/java')
+        executable = new File(project.runtimeJavaHome, 'bin/java')
         if (false == checkForTestsInMain) {
             /* This task is created by default for all subprojects with this
              * setting and there is no point in running it if the files don't

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -655,7 +655,7 @@ class ClusterFormationTasks {
                 String pid = node.pidFile.getText('UTF-8')
                 ByteArrayOutputStream output = new ByteArrayOutputStream()
                 project.exec {
-                    commandLine = ["${project.javaHome}/bin/jstack", pid]
+                    commandLine = ["${project.runtimeJavaHome}/bin/jstack", pid]
                     standardOutput = output
                 }
                 output.toString('UTF-8').eachLine { line -> logger.error("|    ${line}") }
@@ -699,7 +699,7 @@ class ClusterFormationTasks {
     }
 
     private static File getJpsExecutableByName(Project project, String jpsExecutableName) {
-        return Paths.get(project.javaHome.toString(), "bin/" + jpsExecutableName).toFile()
+        return Paths.get(project.runtimeJavaHome.toString(), "bin/" + jpsExecutableName).toFile()
     }
 
     /** Adds a task to kill an elasticsearch node with the given pidfile */

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -162,7 +162,7 @@ class NodeInfo {
             args.add("${esScript}")
         }
 
-        env = ['JAVA_HOME': project.javaHome]
+        env = ['JAVA_HOME': project.runtimeJavaHome]
         args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.MultiSearchRequest;
@@ -312,6 +313,15 @@ static Request get(GetRequest getRequest) {
         return new Request(HttpGet.METHOD_NAME, endpoint, parameters.getParams(), null);
     }
 
+    static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
+        Params parameters = Params.builder();
+        parameters.withPreference(multiGetRequest.preference());
+        parameters.withRealtime(multiGetRequest.realtime());
+        parameters.withRefresh(multiGetRequest.refresh());
+        HttpEntity entity = createEntity(multiGetRequest, REQUEST_BODY_CONTENT_TYPE);
+        return new Request(HttpGet.METHOD_NAME, "/_mget", parameters.getParams(), entity);
+    }
+
     static Request index(IndexRequest indexRequest) {
         String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME;
 

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -34,6 +34,8 @@
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetRequest;
+import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.main.MainRequest;
@@ -289,6 +291,25 @@ public final void getAsync(GetRequest getRequest, ActionListener<GetResponse> li
         performRequestAsyncAndParseEntity(getRequest, Request::get, GetResponse::fromXContent, listener, singleton(404), headers);
     }
 
+    /**
+     * Retrieves multiple documents by id using the Multi Get API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
+     */
+    public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(multiGetRequest, Request::multiGet, MultiGetResponse::fromXContent, singleton(404), headers);
+    }
+
+    /**
+     * Asynchronously retrieves multiple documents by id using the Multi Get API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
+     */
+    public void multiGetAsync(MultiGetRequest multiGetRequest, ActionListener<MultiGetResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(multiGetRequest, Request::multiGet, MultiGetResponse::fromXContent, listener,
+                singleton(404), headers);
+    }
+
     /**
      * Checks for the existence of a document. Returns true if it exists, false otherwise
      *