Merge branch '6.x' into ccr-6.x

* 6.x: (96 commits)
  Introduce global checkpoint listeners (#32696)
  Use JDK 10 for 6.4 BWC builds (#32866)
  Remove unused imports - follow up to removal of test in issue 32855
  Removed flaky test. Looks like randomisation makes these assertions unreliable. This test is superfluous - it was added to address #32770 but it later turned out there was an existing test that just required a fix to provide the missing test coverage.
  [test] mute IndexShardTests.testDocStats
  Test: Fix forbidden uses in test framework (#32824)
  Security: remove password hash bootstrap check (#32440)
  Move validation to server for put user requests (#32471)
  [ML] Add high level REST client docs for ML put job endpoint (#32843)
  Painless: Change fqn_only to no_import (#32817)
  [test] mute testSearchWithSignificantTermsAgg
  Backport: CompletableContext class to avoid throwable (#32829)
  [TEST] Select free port for Minio (#32837)
  SCRIPTING: Support BucketAggScript return null (#32811) (#32833)
  HLRC: Add Delete License API (#32586)
  Aggregations/HL Rest client fix: missing scores (#32774)
  HLRC: migration get assistance API (#32744)
  Fix NOOP bulk updates (#32819)
  Increase logging testRetentionPolicyChangeDuringRecovery
  AwaitsFix case-functions.sql-spec
  ...

Author: jasontedor

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -87,6 +87,8 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('nebula.info-scm')
         project.pluginManager.apply('nebula.info-jar')
 
+        project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
+
         globalBuildInfo(project)
         configureRepositories(project)
         configureConfigurations(project)
@@ -101,6 +103,7 @@ class BuildPlugin implements Plugin<Project> {
         configureDependenciesInfo(project)
     }
 
+
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
@@ -116,12 +119,14 @@ class BuildPlugin implements Plugin<Project> {
 
             final Map<Integer, String> javaVersions = [:]
             for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
-                javaVersions.put(version, findJavaHome(version));
+                if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
+                    javaVersions.put(version, findJavaHome(version.toString()));
+                }
             }
 
             String javaVendor = System.getProperty('java.vendor')
-            String javaVersion = System.getProperty('java.version')
-            String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
+            String gradleJavaVersion = System.getProperty('java.version')
+            String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" +
                 " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"
 
             String compilerJavaVersionDetails = gradleJavaVersionDetails
@@ -144,33 +149,33 @@ class BuildPlugin implements Plugin<Project> {
             // Build debugging info
             println '======================================='
             println 'Elasticsearch Build Hamster says Hello!'
-            println '======================================='
             println "  Gradle Version        : ${project.gradle.gradleVersion}"
             println "  OS Info               : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
             if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
-                println "  JDK Version (gradle)  : ${gradleJavaVersionDetails}"
-                println "  JAVA_HOME (gradle)    : ${gradleJavaHome}"
-                println "  JDK Version (compile) : ${compilerJavaVersionDetails}"
-                println "  JAVA_HOME (compile)   : ${compilerJavaHome}"
-                println "  JDK Version (runtime) : ${runtimeJavaVersionDetails}"
-                println "  JAVA_HOME (runtime)   : ${runtimeJavaHome}"
+                println "  Compiler JDK Version  : ${getPaddedMajorVersion(compilerJavaVersionEnum)} (${compilerJavaVersionDetails})"
+                println "  Compiler java.home    : ${compilerJavaHome}"
+                println "  Runtime JDK Version   : ${getPaddedMajorVersion(runtimeJavaVersionEnum)} (${runtimeJavaVersionDetails})"
+                println "  Runtime java.home     : ${runtimeJavaHome}"
+                println "  Gradle JDK Version    : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
+                println "  Gradle java.home      : ${gradleJavaHome}"
             } else {
-                println "  JDK Version           : ${gradleJavaVersionDetails}"
+                println "  JDK Version           : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
             }
             println "  Random Testing Seed   : ${project.testSeed}"
+            println '======================================='
 
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
                 final String message =
-                        "the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
+                        "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
                                 " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
                 throw new GradleException(message)
             }
 
             if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
                 final String message =
-                        "the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
+                        "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
                                 " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
                 throw new GradleException(message)
             }
@@ -205,6 +210,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
             project.rootProject.ext.inFipsJvm = inFipsJvm
+            project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -217,11 +223,20 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
         project.ext.javaVersions = project.rootProject.ext.javaVersions
         project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
+        project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion
+    }
+
+    private static String getPaddedMajorVersion(JavaVersion compilerJavaVersionEnum) {
+        compilerJavaVersionEnum.getMajorVersion().toString().padLeft(2)
     }
 
     private static String findCompilerJavaHome() {
-        final String javaHome = System.getenv('JAVA_HOME')
-        if (javaHome == null) {
+        final String compilerJavaHome = System.getenv('JAVA_HOME')
+        final String compilerJavaProperty = System.getProperty('compiler.java')
+        if (compilerJavaProperty != null) {
+            compilerJavaHome = findJavaHome(compilerJavaProperty)
+        }
+        if (compilerJavaHome == null) {
             if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
                 // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
                 return Jvm.current().javaHome
@@ -233,11 +248,24 @@ class BuildPlugin implements Plugin<Project> {
                 )
             }
         }
-        return javaHome
+        return compilerJavaHome
     }
 
-    private static String findJavaHome(int version) {
-        return System.getenv('JAVA' + version + '_HOME')
+    private static String findJavaHome(String version) {
+        String versionedVarName = getJavaHomeEnvVarName(version)
+        String versionedJavaHome = System.getenv(versionedVarName);
+        if (versionedJavaHome == null) {
+            throw new GradleException(
+                    "$versionedVarName must be set to build Elasticsearch. " +
+                            "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
+                            "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
+            )
+        }
+        return versionedJavaHome
+    }
+
+    private static String getJavaHomeEnvVarName(String version) {
+        return 'JAVA' + version + '_HOME'
     }
 
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
@@ -271,7 +299,10 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     private static String findRuntimeJavaHome(final String compilerJavaHome) {
-        assert compilerJavaHome != null
+        String runtimeJavaProperty = System.getProperty("runtime.java")
+        if (runtimeJavaProperty != null) {
+            return findJavaHome(runtimeJavaProperty)
+        }
         return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
     }
 
@@ -769,6 +800,12 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
             systemProperty 'es.scripting.exception_for_missing_value', 'true'
+            systemProperty 'compiler.java', project.ext.compilerJavaVersion.getMajorVersion()
+            if (project.ext.inFipsJvm) {
+                systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS"
+            } else {
+                systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion()
+            }
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
@@ -783,9 +820,12 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
 
-            // TODO: remove this once joda time is removed from scriptin in 7.0
+            // TODO: remove this once joda time is removed from scripting in 7.0
             systemProperty 'es.scripting.use_java_time', 'true'
 
+            // TODO: remove this once ctx isn't added to update script params in 7.0
+            systemProperty 'es.scripting.update.ctx_in_params', 'false'
+
             // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
             if (project.inFipsJvm) {
                 systemProperty 'javax.net.ssl.trustStorePassword', 'password'

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -284,6 +284,10 @@ public class SnippetsTask extends DefaultTask {
                     contents.append(line).append('\n')
                     return
                 }
+                // Allow line continuations for console snippets within lists
+                if (snippet != null && line.trim() == '+') {
+                    return
+                }
                 // Just finished
                 emit()
             }

buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.StopExecutionException;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Export Elasticsearch build resources to configurable paths
+ * <p>
+ * Wil overwrite existing files and create missing directories.
+ * Useful for resources that that need to be passed to other processes trough the filesystem or otherwise can't be
+ * consumed from the classpath.
+ */
+public class ExportElasticsearchBuildResourcesTask extends DefaultTask {
+
+    private final Logger logger = Logging.getLogger(ExportElasticsearchBuildResourcesTask.class);
+
+    private final Set<String> resources = new HashSet<>();
+
+    private DirectoryProperty outputDir;
+
+    public ExportElasticsearchBuildResourcesTask() {
+        outputDir = getProject().getLayout().directoryProperty(
+            getProject().getLayout().getBuildDirectory().dir("build-tools-exported")
+        );
+    }
+
+    @OutputDirectory
+    public DirectoryProperty getOutputDir() {
+        return outputDir;
+    }
+
+    @Input
+    @SkipWhenEmpty
+    public Set<String> getResources() {
+        return Collections.unmodifiableSet(resources);
+    }
+
+    @Classpath
+    public String getResourcesClasspath() {
+        // This will make sure the task is not considered up to date if the resources are changed.
+        logger.info("Classpath: {}", System.getProperty("java.class.path"));
+        return System.getProperty("java.class.path");
+    }
+
+    public void setOutputDir(DirectoryProperty outputDir) {
+        this.outputDir = outputDir;
+    }
+
+    public RegularFileProperty take(String resource) {
+        if (getState().getExecuted() || getState().getExecuting()) {
+            throw new GradleException("buildResources can't be configured after the task ran. " +
+                "Make sure task is not used after configuration time"
+            );
+        }
+        resources.add(resource);
+        return getProject().getLayout().fileProperty(outputDir.file(resource));
+    }
+
+    @TaskAction
+    public void doExport() {
+        if (resources.isEmpty()) {
+            throw new StopExecutionException();
+        }
+        resources.stream().parallel()
+            .forEach(resourcePath -> {
+                Path destination = outputDir.get().file(resourcePath).getAsFile().toPath();
+                try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) {
+                    if (is == null) {
+                        throw new GradleException("Can't export `" + resourcePath + "` from build-tools: not found");
+                    }
+                    Files.copy(is, destination);
+                } catch (IOException e) {
+                    throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination);
+                }
+            });
+    }
+
+}

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -720,6 +720,7 @@
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngine.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]StoredExpressionTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
   <suppress files="modules[/\\]reindex[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]reindex[/\\]TransportUpdateByQueryAction.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuCollationTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuFoldingTokenFilterFactory.java" checks="LineLength" />