Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (140 commits)
  Remove AcknowledgedRestListener in favour of RestToXContentListener (#28724)
  Migrate some *ResponseTests to AbstractStreamableXContentTestCase (#28749)
  Revert "Disable BWC tests for build issues"
  Skip some plugins service tests on Windows
  Build: Consolidate archives and packages configuration (#28760)
  Ensure that azure stream has socket privileges (#28773)
  Disable BWC tests for build issues
  Pass InputStream when creating XContent parser (#28754)
  [DOCS] Fixed broken link.
  [DOCS] Changed to use transient setting to reenabled allocation. Closes #27677
  Delay path expansion on Windows
  [TEST] replace randomAsciiAlphanumOfLengthBetween with randomAsciiLettersOfLengthBetween
  REST high-level client: add support for Rollover Index API (#28698)
  [Tests] Extract the testing logic for Google Cloud Storage (#28576)
  Moved Grok helper code to a separate Gradle module and let ingest-common module depend on it.
  version set in ingest pipeline (#27573)
  [Docs] Update links to java9 docs (#28750)
  Revert "Add startup logging for standalone tests"
  Tests: don't wait for completion while trying to get completed task
  Add 5.6.9 snapshot version
  ...

Author: martijnvg

CONTRIBUTING.md

@@ -162,7 +162,12 @@ cd elasticsearch/
 ./gradlew assemble
 ```
 
-You will find the newly built packages under: `./distribution/(deb|rpm|tar|zip)/build/distributions/`.
+The package distributions (Debian and RPM) can be found under:
+`./distribution/packages/(deb|rpm)/build/distributions/`
+
+The archive distributions (tar and zip) can be found under:
+`./distribution/archives/(tar|zip)/build/distributions/`
+
 
 Before submitting your changes, run the test suite to make sure that nothing is broken, with:
 

README.textile

@@ -192,7 +192,7 @@ h3. Distributed, Highly Available
 
 Let's face it, things will fail....
 
-Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replica. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
+Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
 
 In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed.
 

TESTING.asciidoc

@@ -264,6 +264,8 @@ The REST layer is tested through specific tests that are shared between all
 the elasticsearch official clients and consist of YAML files that describe the
 operations to be executed and the obtained results that need to be tested.
 
+The YAML files support various operators defined in the link:/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc[rest-api-spec] and adhere to the link:/rest-api-spec/README.markdown[Elasticsearch REST API JSON specification]
+
 The REST tests are run automatically when executing the "./gradlew check" command. To run only the
 REST tests use the following command:
 

build.gradle

@@ -103,7 +103,7 @@ allprojects {
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
 
     // for BWC testing
-    versionCollection = versions
+    bwcVersions = versions
 
     buildMetadata = buildMetadataMap
   }
@@ -122,13 +122,13 @@ task verifyVersions {
     Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
-    knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
+    knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
 
     /* Limit the listed versions to those that have been marked as released.
      * Versions not marked as released don't get the same testing and we want
      * to make sure that we flip all unreleased versions to released as soon
      * as possible after release. */
-    Set<Version> actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot })
+    Set<Version> actualVersions = new TreeSet<>(bwcVersions.indexCompatible.findAll { false == it.snapshot })
 
     // Finally, compare!
     if (knownVersions.equals(actualVersions) == false) {
@@ -144,8 +144,19 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-allprojects {
-  ext.bwc_tests_enabled = true
+final boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when commiting bwc changes */
+if (bwc_tests_enabled == false) {
+  if (bwc_tests_disabled_issue.isEmpty()) {
+    throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
+  }
+  println "========================= WARNING ========================="
+  println "         Backwards compatibility tests are disabled!"
+  println "See ${bwc_tests_disabled_issue}"
+  println "==========================================================="
+}
+subprojects {
+  ext.bwc_tests_enabled = bwc_tests_enabled
   /*
    * Versions of Elasticsearch 5.1.1 through 5.3.0 inclusive did not start on versions of Linux with cgroups v2 enabled (kernel >= 4.5).
    * This property is provided to all projects that need to check conditionally if they should skip a BWC test task.
@@ -155,7 +166,7 @@ allprojects {
 
 task verifyBwcTestsEnabled {
   doLast {
-    if (project.bwc_tests_enabled == false) {
+    if (bwc_tests_enabled == false) {
       throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
     }
   }
@@ -196,11 +207,11 @@ subprojects {
     "org.elasticsearch.client:test:${version}": ':client:test',
     "org.elasticsearch.client:transport:${version}": ':client:transport',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
-    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
-    "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
-    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
-    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb',
+    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
+    "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
+    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar',
+    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
+    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     // for transport client
     "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
@@ -212,13 +223,14 @@ subprojects {
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
 
-  for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
-    if (version.branch != null) {
-      final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}"
-      project(snapshotProject).ext.bwcVersion = version
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject
+  bwcVersions.snapshotProjectNames.each { snapshotName ->
+    Version snapshot = bwcVersions.getSnapshotForProject(snapshotName)
+    if (snapshot != null ) {
+      String snapshotProject = ":distribution:bwc:${snapshotName}"
+      project(snapshotProject).ext.bwcVersion = snapshot
+      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${snapshot}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${snapshot}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${snapshot}"] = snapshotProject
     }
   }
 
@@ -249,7 +261,7 @@ subprojects {
         }
       }
       project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
-      project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+      project.configurations.compileOnly.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
     }
   }
 }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy

@@ -21,9 +21,14 @@ package com.carrotsearch.gradle.junit4
 
 import com.carrotsearch.ant.tasks.junit4.JUnit4
 import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.eventbus.Subscribe
+import com.carrotsearch.ant.tasks.junit4.events.TestStartedEvent
+import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedQuitEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent
+import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteStartedEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent
+import com.carrotsearch.ant.tasks.junit4.events.aggregated.ChildBootstrap
+import com.carrotsearch.ant.tasks.junit4.events.aggregated.HeartBeatEvent
 import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
 import org.gradle.internal.logging.progress.ProgressLogger
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
@@ -34,7 +39,6 @@ import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.FAI
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.IGNORED
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.IGNORED_ASSUMPTION
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.OK
-import static java.lang.Math.max
 
 /**
  * Adapts junit4's event listeners into gradle's ProgressLogger. Note that
@@ -54,137 +58,118 @@ import static java.lang.Math.max
 class TestProgressLogger implements AggregatedEventListener {
     /** Factory to build a progress logger when testing starts */
     ProgressLoggerFactory factory
-    ProgressLogger progressLogger
+    ProgressLogger parentProgressLogger
+    ProgressLogger suiteLogger
+    ProgressLogger testLogger
+    ProgressLogger[] slaveLoggers
     int totalSuites
     int totalSlaves
 
-    // sprintf formats used to align the integers we print
-    String suitesFormat
-    String slavesFormat
-    String testsFormat
-
     // Counters incremented test completion.
     volatile int suitesCompleted = 0
     volatile int testsCompleted = 0
     volatile int testsFailed = 0
     volatile int testsIgnored = 0
 
-    // Information about the last, most interesting event.
-    volatile String eventDescription
-    volatile int eventSlave
-    volatile long eventExecutionTime
-
-    /** Have we finished a whole suite yet? */
-    volatile boolean suiteFinished = false
-    /* Note that we probably overuse volatile here but it isn't hurting us and
-       lets us move things around without worrying about breaking things. */
-
     @Subscribe
     void onStart(AggregatedStartEvent e) throws IOException {
         totalSuites = e.suiteCount
         totalSlaves = e.slaveCount
-        progressLogger = factory.newOperation(TestProgressLogger)
-        progressLogger.setDescription('Randomized test runner')
-        progressLogger.started()
-        progressLogger.progress(
-            "Starting JUnit4 for ${totalSuites} suites on ${totalSlaves} jvms")
-
-        suitesFormat = "%0${widthForTotal(totalSuites)}d"
-        slavesFormat = "%-${widthForTotal(totalSlaves)}s"
-        /* Just guess the number of tests because we can't figure it out from
-          here and it isn't worth doing anything fancy to prevent the console
-          from jumping around a little. 200 is a pretty wild guess for the
-          minimum but it makes REST tests output sanely. */
-        int totalNumberOfTestsGuess = max(200, totalSuites * 10)
-        testsFormat = "%0${widthForTotal(totalNumberOfTestsGuess)}d"
+        parentProgressLogger = factory.newOperation(TestProgressLogger)
+        parentProgressLogger.setDescription('Randomized test runner')
+        parentProgressLogger.started()
+
+        suiteLogger = factory.newOperation(TestProgressLogger, parentProgressLogger)
+        suiteLogger.setDescription('Suite logger')
+        suiteLogger.started("Suites: 0/" + totalSuites)
+        testLogger = factory.newOperation(TestProgressLogger, parentProgressLogger)
+        testLogger.setDescription('Test logger')
+        testLogger.started('Tests: completed: 0, failed: 0, ignored: 0')
+        slaveLoggers = new ProgressLogger[e.slaveCount]
+        for (int i = 0; i < e.slaveCount; ++i) {
+            slaveLoggers[i] = factory.newOperation(TestProgressLogger, parentProgressLogger)
+            slaveLoggers[i].setDescription("J${i} test logger")
+            slaveLoggers[i].started("J${i}: initializing...")
+        }
+    }
+
+    @Subscribe
+    void onChildBootstrap(ChildBootstrap e) throws IOException {
+        slaveLoggers[e.getSlave().id].progress("J${e.slave.id}: starting (pid ${e.slave.pidString})")
+    }
+
+    @Subscribe
+    void onQuit(AggregatedQuitEvent e) throws IOException {
+        suiteLogger.completed()
+        testLogger.completed()
+        for (ProgressLogger slaveLogger : slaveLoggers) {
+            slaveLogger.completed()
+        }
+        parentProgressLogger.completed()
+    }
+
+    @Subscribe
+    void onSuiteStart(AggregatedSuiteStartedEvent e) throws IOException {
+        String suiteName = simpleName(e.suiteStartedEvent.description.className)
+        slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${suiteName} - initializing")
+    }
+
+    @Subscribe
+    void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException {
+        suitesCompleted++
+        suiteLogger.progress("Suites: " + suitesCompleted + "/" + totalSuites)
     }
 
     @Subscribe
     void onTestResult(AggregatedTestResultEvent e) throws IOException {
+        final String statusMessage
         testsCompleted++
         switch (e.status) {
         case ERROR:
         case FAILURE:
             testsFailed++
+            statusMessage = "failed"
             break
         case IGNORED:
         case IGNORED_ASSUMPTION:
             testsIgnored++
+            statusMessage = "ignored"
             break
         case OK:
+            String time = formatDurationInSeconds(e.executionTime)
+            statusMessage = "completed [${time}]"
             break
         default:
-            throw new IllegalArgumentException(
-                "Unknown test status: [${e.status}]")
+            throw new IllegalArgumentException("Unknown test status: [${e.status}]")
         }
-        if (!suiteFinished) {
-            updateEventInfo(e)
-        }
-
-        log()
+        testLogger.progress("Tests: completed: ${testsCompleted}, failed: ${testsFailed}, ignored: ${testsIgnored}")
+        String testName = simpleName(e.description.className) + '.' + e.description.methodName
+        slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} ${statusMessage}")
     }
 
     @Subscribe
-    void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException {
-        suitesCompleted++
-        suiteFinished = true
-        updateEventInfo(e)
-        log()
+    void onTestStarted(TestStartedEvent e) throws IOException {
+        String testName = simpleName(e.description.className) + '.' + e.description.methodName
+        slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} ...")
     }
 
-    /**
-     * Update the suite information with a junit4 event.
-     */
-    private void updateEventInfo(Object e) {
-        eventDescription = simpleName(e.description.className)
-        if (e.description.methodName != null) {
-            eventDescription += "#${e.description.methodName}"
-        }
-        eventSlave = e.slave.id
-        eventExecutionTime = e.executionTime
+    @Subscribe
+    void onHeartbeat(HeartBeatEvent e) throws IOException {
+        String testName = simpleName(e.description.className) + '.' + e.description.methodName
+        String time = formatDurationInSeconds(e.getNoEventDuration())
+        slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} stalled for ${time}")
     }
 
     /**
      * Extract a Class#getSimpleName style name from Class#getName style
      * string. We can't just use Class#getSimpleName because junit descriptions
-     * don't alway s set the class field but they always set the className
+     * don't always set the class field but they always set the className
      * field.
      */
     private static String simpleName(String className) {
         return className.substring(className.lastIndexOf('.') + 1)
     }
 
-    private void log() {
-        /* Remember that instances of this class are only ever active on one
-          thread at a time so there really aren't race conditions here. It'd be
-          OK if there were because they'd only display an overcount
-          temporarily. */
-        String log = ''
-        if (totalSuites > 1) {
-            /* Skip printing the suites to save space when there is only a
-              single suite. This is nice because when there is only a single
-              suite we log the method name and those can be long. */
-            log += sprintf("Suites [${suitesFormat}/${suitesFormat}], ",
-                [suitesCompleted, totalSuites])
-        }
-        log += sprintf("Tests [${testsFormat}|%d|%d], ",
-            [testsCompleted, testsFailed, testsIgnored])
-        log += "in ${formatDurationInSeconds(eventExecutionTime)} "
-        if (totalSlaves > 1) {
-            /* Skip printing the slaves if there is only one of them. This is
-              nice because when there is only a single slave there is often
-              only a single suite and we could use the extra space to log the
-              test method names. */
-            log += "J${sprintf(slavesFormat, eventSlave)} "
-        }
-        log += "completed ${eventDescription}"
-        progressLogger.progress(log)
-    }
-
-    private static int widthForTotal(int total) {
-        return ((total - 1) as String).length()
-    }
-
     @Override
     void setOuter(JUnit4 junit) {}
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -76,7 +76,6 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('nebula.info-java')
         project.pluginManager.apply('nebula.info-scm')
         project.pluginManager.apply('nebula.info-jar')
-        project.pluginManager.apply(ProvidedBasePlugin)
 
         globalBuildInfo(project)
         configureRepositories(project)
@@ -261,6 +260,9 @@ class BuildPlugin implements Plugin<Project> {
      * to iterate the transitive dependencies and add excludes.
      */
     static void configureConfigurations(Project project) {
+        // we want to test compileOnly deps!
+        project.configurations.testCompile.extendsFrom(project.configurations.compileOnly)
+
         // we are not shipping these jars, we act like dumb consumers of these things
         if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
             return
@@ -297,7 +299,7 @@ class BuildPlugin implements Plugin<Project> {
 
         project.configurations.compile.dependencies.all(disableTransitiveDeps)
         project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
-        project.configurations.provided.dependencies.all(disableTransitiveDeps)
+        project.configurations.compileOnly.dependencies.all(disableTransitiveDeps)
     }
 
     /** Adds repositories used by ES dependencies */
@@ -665,7 +667,7 @@ class BuildPlugin implements Plugin<Project> {
         // only require dependency licenses for non-elasticsearch deps
         project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection {
             it.group.startsWith('org.elasticsearch') == false
-        } - project.configurations.provided
+        } - project.configurations.compileOnly
     }
 
     private static configureDependenciesInfo(Project project) {