Merge branch 'master' into feature/extensible-values-source

 Conflicts:
	server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
	server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java

Author: not-napoleon

.ci/bwcVersions

@@ -16,4 +16,5 @@ BWC_VERSION:
   - "7.5.2"
   - "7.5.3"
   - "7.6.0"
+  - "7.7.0"
   - "8.0.0"

.ci/matrix-runtime-javas.yml

@@ -9,6 +9,7 @@ ES_RUNTIME_JAVA:
   - java11
   - openjdk13
   - openjdk14
+  - openjdk15
   - zulu11
   - corretto11
   - adoptopenjdk11

.eclipseformat.xml

@@ -51,7 +51,7 @@
         <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator" value="true"/>
-        <setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
+        <setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="true"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
@@ -101,7 +101,7 @@
         <setting id="org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line" value="one_line_if_empty"/>
         <setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="do not insert"/>
-        <setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
+        <setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="true"/>
         <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause" value="common_lines"/>
         <setting id="org.eclipse.jdt.core.formatter.alignment_for_additive_operator" value="16"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
@@ -286,7 +286,7 @@
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.comment.indent_tag_description" value="false"/>
-        <setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
+        <setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="true"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="48"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>

Vagrantfile

@@ -475,6 +475,7 @@ JAVA
     ensure curl
     ensure unzip
     ensure rsync
+    ensure expect
 
     installed bats || {
       # Bats lives in a git repository....

buildSrc/build.gradle

@@ -273,8 +273,17 @@ class VersionPropertiesLoader {
       }
       elasticsearch += "-" + qualifier
     }
-    if ("true".equals(systemProperties.getProperty("build.snapshot", "true"))) {
-      elasticsearch += "-SNAPSHOT"
+    final String buildSnapshotSystemProperty = systemProperties.getProperty("build.snapshot", "true");
+    switch (buildSnapshotSystemProperty) {
+      case "true":
+        elasticsearch += "-SNAPSHOT"
+        break;
+      case "false":
+        // do nothing
+        break;
+      default:
+        throw new IllegalArgumentException(
+          "build.snapshot was set to [" + buildSnapshotSystemProperty + "] but can only be unset or [true|false]");
     }
     loadedProps.put("elasticsearch", elasticsearch)
   }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -32,6 +32,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.elasticsearch.gradle.testclusters.TestDistribution
 import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.Action
 import org.gradle.api.GradleException
@@ -424,7 +425,7 @@ class BuildPlugin implements Plugin<Project> {
                             dependencyNode.appendNode('groupId', dependency.group)
                             dependencyNode.appendNode('artifactId', dependency.getDependencyProject().convention.getPlugin(BasePluginConvention).archivesBaseName)
                             dependencyNode.appendNode('version', dependency.version)
-                            dependencyNode.appendNode('scope', 'runtime')
+                            dependencyNode.appendNode('scope', 'compile')
                         }
                     }
                 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -45,12 +45,8 @@ class RestIntegTestTask extends DefaultTask {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
 
-        project.testClusters {
-                "$name" {
-                    javaHome = BuildParams.runtimeJavaHome
-                }
-        }
-        runner.useCluster project.testClusters."$name"
+        ElasticsearchCluster cluster = project.testClusters.create(name)
+        runner.useCluster cluster
 
         runner.include('**/*IT.class')
         runner.systemProperty('tests.rest.load_packaged', 'false')
@@ -60,7 +56,6 @@ class RestIntegTestTask extends DefaultTask {
                 throw new IllegalArgumentException("tests.rest.cluster, tests.cluster, and tests.clustername must all be null or non-null")
             }
 
-            ElasticsearchCluster cluster = project.testClusters."${name}"
             runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",")}")
             runner.nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI}")
             runner.nonInputProperties.systemProperty('tests.clustername', "${-> cluster.getName()}")

buildSrc/src/main/java/org/elasticsearch/GradleServicesAdapter.java

@@ -48,6 +48,8 @@
 import java.util.concurrent.Callable;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.gradle.Util.capitalize;
+
 /**
  * A plugin to manage getting and extracting distributions of Elasticsearch.
  *
@@ -309,10 +311,6 @@ private static String configName(String prefix, ElasticsearchDistribution distri
         );
     }
 
-    private static String capitalize(String s) {
-        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
-    }
-
     private static String extractTaskName(ElasticsearchDistribution distribution) {
         String taskName = "extractElasticsearch";
         if (distribution.getType() != Type.INTEG_TEST_ZIP) {

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -20,37 +20,41 @@
 package org.elasticsearch.gradle;
 
 import org.gradle.api.Buildable;
-import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.TaskDependency;
 
 import java.io.File;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class Jdk implements Buildable, Iterable<File> {
 
     private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
-    static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
-    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac"));
+    private static final List<String> ALLOWED_PLATFORMS = List.of("darwin", "linux", "windows", "mac");
+    private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
+    private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final Configuration configuration;
 
     private final Property<String> vendor;
     private final Property<String> version;
     private final Property<String> platform;
+    private String baseVersion;
+    private String major;
+    private String build;
+    private String hash;
 
-    Jdk(String name, Project project) {
+    Jdk(String name, Configuration configuration, ObjectFactory objectFactory) {
         this.name = name;
-        this.configuration = project.getConfigurations().create("jdk_" + name);
-        this.vendor = project.getObjects().property(String.class);
-        this.version = project.getObjects().property(String.class);
-        this.platform = project.getObjects().property(String.class);
+        this.configuration = configuration;
+        this.vendor = objectFactory.property(String.class);
+        this.version = objectFactory.property(String.class);
+        this.platform = objectFactory.property(String.class);
     }
 
     public String getName() {
@@ -73,9 +77,10 @@ public String getVersion() {
     }
 
     public void setVersion(String version) {
-        if (VERSION_PATTERN.matcher(version).matches() == false) {
+        if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
             throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
         }
+        parseVersion(version);
         this.version.set(version);
     }
 
@@ -92,15 +97,30 @@ public void setPlatform(String platform) {
         this.platform.set(platform);
     }
 
-    // pkg private, for internal use
-    Configuration getConfiguration() {
-        return configuration;
+    public String getBaseVersion() {
+        return baseVersion;
+    }
+
+    public String getMajor() {
+        return major;
+    }
+
+    public String getBuild() {
+        return build;
+    }
+
+    public String getHash() {
+        return hash;
     }
 
     public String getPath() {
         return configuration.getSingleFile().toString();
     }
 
+    public String getConfigurationName() {
+        return configuration.getName();
+    }
+
     @Override
     public String toString() {
         return getPath();
@@ -143,4 +163,23 @@ public Iterator<File> iterator() {
         return configuration.iterator();
     }
 
+    private void parseVersion(String version) {
+        // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
+        // Note the "patch" version is not yet handled here, as it has not yet been used by java.
+        Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version);
+        if (jdkVersionMatcher.matches() == false) {
+            // Try again with the pre-Java9 version format
+            jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
+
+            if (jdkVersionMatcher.matches() == false) {
+                throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+            }
+        }
+
+        baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
+        major = jdkVersionMatcher.group(1);
+        build = jdkVersionMatcher.group(3);
+        hash = jdkVersionMatcher.group(5);
+    }
+
 }