Merge remote-tracking branch 'elastic/master' into geoshape-doc-values

Author: talevy

.ci/dockerOnLinuxExclusions

@@ -7,9 +7,11 @@
 debian-8
 opensuse-15-1
 ol-7.7
-sles-12
+sles-12.3 # older version used in Vagrant image
+sles-12.4
+sles-15.1
 
-# These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded 
+# These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded
 # for PR checks
 centos-6
 ol-6.10

.github/workflows/gradle-wrapper-validation.yml

@@ -0,0 +1,10 @@
+name: "Validate Gradle Wrapper"
+on: [push]
+
+jobs:
+  validation:
+    name: "Validation"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: gradle/wrapper-validation-action@v1

TESTING.asciidoc

@@ -270,34 +270,49 @@ code 137 (128 + 9, where 9 means SIGKILL).
 
 == Testing the REST layer
 
-The available integration tests make use of the java API to communicate with
-the elasticsearch nodes, using the internal binary transport (port 9300 by
-default).
-The REST layer is tested through specific tests that are shared between all
-the elasticsearch official clients and consist of YAML files that describe the
+The REST layer is tested through specific tests that are executed against
+a cluster that is configured and initialized via Gradle. The tests
+themselves can be written in either Java or with a YAML based DSL.
+
+YAML based REST tests should be preferred since these are shared between all
+the elasticsearch official clients. The YAML based tests describe the
 operations to be executed and the obtained results that need to be tested.
 
-The YAML files support various operators defined in the link:/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc[rest-api-spec] and adhere to the link:/rest-api-spec/README.markdown[Elasticsearch REST API JSON specification]
+The YAML tests support various operators defined in the link:/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc[rest-api-spec] and adhere to the link:/rest-api-spec/README.markdown[Elasticsearch REST API JSON specification]
+In order to run the the YAML tests, the relevant API specification needs
+to be on the test classpath. Any gradle project that has support for REST
+tests will get the primary API on it's class path. However, to better support
+Gradle incremental builds, it is recommended to explicitly declare which
+parts of the API the tests depend upon.
+
+For example:
+---------------------------------------------------------------------------
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest'
+  }
+}
+---------------------------------------------------------------------------
+
+YAML tests that include x-pack specific APIs need to explicitly declare
+which APIs are required through a similar `includeXpack` configuration.
 
 The REST tests are run automatically when executing the "./gradlew check" command. To run only the
 REST tests use the following command:
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:archives:integ-test-zip:integTest   \
-  -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT"
+./gradlew :distribution:archives:integ-test-zip:integTestRunner \
+ --tests "org.elasticsearch.test.rest.IntegTestZipClientYamlTestSuiteIT"
 ---------------------------------------------------------------------------
 
-A specific test case can be run with
+A specific test case can be run with the following command:
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:archives:integ-test-zip:integTest \
-  -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT" \
-  -Dtests.method="test {p0=cat.shards/10_basic/Help}"
+./gradlew ':distribution:archives:integ-test-zip:integTestRunner' \
+  --tests "org.elasticsearch.test.rest.IntegTestZipClientYamlTestSuiteIT" \
+  -Dtests.method="test {p0=cat.segments/10_basic/Help}"
 ---------------------------------------------------------------------------
 
-`*Yaml*IT` are the executable test classes that runs all the
-yaml suites available within the `rest-api-spec` folder.
-
 The REST tests support all the options provided by the randomized runner, plus the following:
 
 * `tests.rest[true|false]`: determines whether the REST tests need to be run (default) or not.
@@ -459,6 +474,14 @@ version 5.3.2 run:
 ./gradlew v5.3.2#bwcTest
 -------------------------------------------------
 
+Use -Dtest.class and -Dtests.method to run a specific bwcTest test.
+For example to run a specific tests from the x-pack rolling upgrade from 7.7.0:
+-------------------------------------------------
+./gradlew :x-pack:qa:rolling-upgrade:v7.7.0#bwcTest \
+ -Dtests.class=org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT \
+ -Dtests.method="test {p0=*/40_ml_datafeed_crud/*}"
+-------------------------------------------------
+
 Tests are ran for versions that are not yet released but with which the current version will be compatible with.
 These are automatically checked out and built from source.
 See link:./buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java[VersionCollection]
@@ -512,8 +535,9 @@ There are multiple base classes for tests:
 * **`ESIntegTestCase`**: An integration test case that creates a cluster that
   might have multiple nodes.
 * **`ESRestTestCase`**: An integration tests that interacts with an external
-  cluster via the REST API. For instance, YAML tests run via sub classes of
-  `ESRestTestCase`.
+  cluster via the REST API. This is used for Java based REST tests.
+* **`ESClientYamlSuiteTestCase` **: A subclass of `ESRestTestCase` used to run
+  YAML based REST tests.
 
 === Good practices
 
@@ -643,4 +667,4 @@ please see https://esrally.readthedocs.io/en/stable/[Rally's documentation].
 
 The Elasticsearch docs are in AsciiDoc format. You can test and build the docs
 locally using the Elasticsearch documentation build process. See
-https://github.com/elastic/docs.
+https://github.com/elastic/docs.

benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java

@@ -55,13 +55,14 @@ public class AvailableIndexFoldersBenchmark {
     @Setup
     public void setup() throws IOException {
         Path path = Files.createTempDirectory("test");
-        String[] paths = new String[] {path.toString()};
+        String[] paths = new String[] { path.toString() };
         nodePath = new NodeEnvironment.NodePath(path);
 
         LogConfigurator.setNodeName("test");
         Settings settings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), path)
-            .putList(Environment.PATH_DATA_SETTING.getKey(), paths).build();
+            .putList(Environment.PATH_DATA_SETTING.getKey(), paths)
+            .build();
         nodeEnv = new NodeEnvironment(settings, new Environment(settings, null));
 
         Files.createDirectories(nodePath.indicesPath);
@@ -80,7 +81,6 @@ public void setup() throws IOException {
         }
     }
 
-
     @Benchmark
     public Set<String> availableIndexFolderNaive() throws IOException {
         return nodeEnv.availableIndexFoldersForPath(nodePath);

benchmarks/src/main/java/org/elasticsearch/benchmark/indices/breaker/MemoryStatsBenchmark.java

@@ -41,11 +41,11 @@
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.MICROSECONDS)
 @State(Scope.Benchmark)
-@SuppressWarnings("unused") //invoked by benchmarking framework
+@SuppressWarnings("unused") // invoked by benchmarking framework
 public class MemoryStatsBenchmark {
     private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean();
 
-    @Param({"0", "16", "256", "4096"})
+    @Param({ "0", "16", "256", "4096" })
     private int tokens;
 
     @Benchmark
@@ -102,4 +102,3 @@ public long getMemoryStats_64() {
         return MEMORY_MX_BEAN.getHeapMemoryUsage().getUsed();
     }
 }
-

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -49,7 +49,7 @@
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Benchmark)
-@SuppressWarnings("unused") //invoked by benchmarking framework
+@SuppressWarnings("unused") // invoked by benchmarking framework
 public class AllocationBenchmark {
     // Do NOT make any field final (even if it is not annotated with @Param)! See also
     // http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java
@@ -106,8 +106,7 @@ public class AllocationBenchmark {
         "       10|     10|        2|    50",
         "      100|      1|        2|    50",
         "      100|      3|        2|    50",
-        "      100|     10|        2|    50"
-    })
+        "      100|     10|        2|    50" })
     public String indicesShardsReplicasNodes = "10|1|0|1";
 
     public int numTags = 2;
@@ -124,13 +123,14 @@ public void setUp() throws Exception {
         int numReplicas = toInt(params[2]);
         int numNodes = toInt(params[3]);
 
-        strategy = Allocators.createAllocationService(Settings.builder()
-                .put("cluster.routing.allocation.awareness.attributes", "tag")
-                .build());
+        strategy = Allocators.createAllocationService(
+            Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build()
+        );
 
         MetaData.Builder mb = MetaData.builder();
         for (int i = 1; i <= numIndices; i++) {
-            mb.put(IndexMetaData.builder("test_" + i)
+            mb.put(
+                IndexMetaData.builder("test_" + i)
                     .settings(Settings.builder().put("index.version.created", Version.CURRENT))
                     .numberOfShards(numShards)
                     .numberOfReplicas(numReplicas)
@@ -147,8 +147,10 @@ public void setUp() throws Exception {
             nb.add(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
         }
         initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData).routingTable(routingTable).nodes
-                (nb).build();
+            .metaData(metaData)
+            .routingTable(routingTable)
+            .nodes(nb)
+            .build();
     }
 
     private int toInt(String v) {
@@ -159,8 +161,10 @@ private int toInt(String v) {
     public ClusterState measureAllocation() {
         ClusterState clusterState = initialClusterState;
         while (clusterState.getRoutingNodes().hasUnassignedShards()) {
-            clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
-                    .shardsWithState(ShardRoutingState.INITIALIZING));
+            clusterState = strategy.applyStartedShards(
+                clusterState,
+                clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING)
+            );
             clusterState = strategy.reroute(clusterState, "reroute");
         }
         return clusterState;

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -36,7 +36,6 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
 
-import java.lang.reflect.InvocationTargetException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -67,33 +66,34 @@ private Allocators() {
         throw new AssertionError("Do not instantiate");
     }
 
-
-    public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException,
-        IllegalAccessException, InvocationTargetException {
-        return createAllocationService(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings
-            .BUILT_IN_CLUSTER_SETTINGS));
+    public static AllocationService createAllocationService(Settings settings) {
+        return createAllocationService(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
     }
 
-    public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
-        InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
+    public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) {
         return new AllocationService(
             defaultAllocationDeciders(settings, clusterSettings),
-            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
+            NoopGatewayAllocator.INSTANCE,
+            new BalancedShardsAllocator(settings),
+            EmptyClusterInfoService.INSTANCE
+        );
     }
 
-    public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws
-        IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
-        Collection<AllocationDecider> deciders =
-            ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
+    public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
+        Collection<AllocationDecider> deciders = ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
         return new AllocationDeciders(deciders);
-
     }
 
     private static final AtomicInteger portGenerator = new AtomicInteger();
 
     public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
-        return new DiscoveryNode("", nodeId, new TransportAddress(TransportAddress.META_ADDRESS,
-            portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE,
-            DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
+        return new DiscoveryNode(
+            "",
+            nodeId,
+            new TransportAddress(TransportAddress.META_ADDRESS, portGenerator.incrementAndGet()),
+            attributes,
+            Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE),
+            Version.CURRENT
+        );
     }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

@@ -39,7 +39,7 @@
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
 @State(Scope.Benchmark)
-@SuppressWarnings("unused") //invoked by benchmarking framework
+@SuppressWarnings("unused") // invoked by benchmarking framework
 public class DateFormatterBenchmark {
 
     private final DateFormatter javaFormatter = DateFormatter.forPattern("8year_month_day||ordinal_date||epoch_millis");

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java

@@ -39,7 +39,7 @@
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
 @State(Scope.Benchmark)
-@SuppressWarnings("unused") //invoked by benchmarking framework
+@SuppressWarnings("unused") // invoked by benchmarking framework
 public class DateFormatterFromBenchmark {
 
     private final TemporalAccessor accessor = DateFormatter.forPattern("epoch_millis").parse("1234567890");

build.gradle

@@ -106,13 +106,15 @@ subprojects {
     // Projects that should be formatted and checked with Spotless are
     // listed here, by project path. Once the number of formatted projects
     // is greater than the number of unformatted projects, this can be
-    // switched to an exclude list, and eventualy removed completely.
+    // switched to an exclude list, and eventually removed completely.
     def projectPathsToFormat = [
+      ':benchmarks',
       ':build-tools',
       ':distribution:tools:java-version-checker',
       ':distribution:tools:keystore-cli',
       ':distribution:tools:launchers',
       ':distribution:tools:plugin-cli',
+      ':qa:os',
       ':x-pack:plugin:autoscaling',
       ':x-pack:plugin:enrich'
     ]
@@ -191,7 +193,7 @@ task verifyVersions {
       throw new GradleException("Must run in online mode to verify versions")
     }
     // Read the list from maven central.
-    // Fetch the metadata an parse the xml into Version instances because it's more straight forward here
+    // Fetch the metadata and parse the xml into Version instances because it's more straight forward here
     // rather than bwcVersion ( VersionCollection ).
     new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
       bwcVersions.compareToAuthoritative(

buildSrc/build.gradle

@@ -80,6 +80,9 @@ sourceSets {
 compileMinimumRuntimeJava {
   targetCompatibility = 8
   sourceCompatibility = 8
+  tasks.withType(JavaCompile).configureEach {
+    options.encoding = 'UTF-8'
+  }
 }
 
 jar {
@@ -196,6 +199,18 @@ if (project != rootProject) {
     }
   }
 
+  // Track reaper jar as a test input using runtime classpath normalization strategy
+  tasks.withType(Test).configureEach {
+    inputs.files(configurations.reaper).withNormalizer(ClasspathNormalizer)
+  }
+
+  normalization {
+    runtimeClasspath {
+      // We already include the reaper jar as part of our runtime classpath. Ignore the copy in META-INF.
+      ignore('META-INF/reaper.jar')
+    }
+  }
+
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(RestResourcesPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)