Merge branch 'master' into pause-auto-followers

Author: elasticmachine

.ci/os.ps1

@@ -33,4 +33,4 @@ $ErrorActionPreference="Continue"
    -x :distribution:packages:buildOssRpm `
    -x :distribution:packages:buildRpm `
 
-exit $?
+exit $LastExitCode

Vagrantfile

@@ -1,5 +1,5 @@
 # -*- mode: ruby -*-
-# vi: set ft=ruby :
+# vim: ft=ruby ts=2 sw=2 sts=2 et:
 
 # This Vagrantfile exists to test packaging. Read more about its use in the
 # vagrant section in TESTING.asciidoc.
@@ -63,6 +63,7 @@ Vagrant.configure(2) do |config|
         # Install Jayatana so we can work around it being present.
         [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
       SHELL
+      ubuntu_docker config
     end
   end
   'ubuntu-1804'.tap do |box|
@@ -72,6 +73,7 @@ Vagrant.configure(2) do |config|
        # Install Jayatana so we can work around it being present.
        [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
       SHELL
+      ubuntu_docker config
     end
   end
   'debian-8'.tap do |box|
@@ -87,6 +89,7 @@ Vagrant.configure(2) do |config|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/debian-9-x86_64'
       deb_common config, box
+      deb_docker config
     end
   end
   'centos-6'.tap do |box|
@@ -99,6 +102,7 @@ Vagrant.configure(2) do |config|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/centos-7-x86_64'
       rpm_common config, box
+      rpm_docker config
     end
   end
   'oel-6'.tap do |box|
@@ -117,12 +121,14 @@ Vagrant.configure(2) do |config|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/fedora-28-x86_64'
       dnf_common config, box
+      dnf_docker config
     end
   end
   'fedora-29'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/fedora-28-x86_64'
       dnf_common config, box
+      dnf_docker config
     end
   end
   'opensuse-42'.tap do |box|
@@ -185,6 +191,63 @@ def deb_common(config, name, extra: '')
   )
 end
 
+def ubuntu_docker(config)
+  config.vm.provision 'install Docker using apt', type: 'shell', inline: <<-SHELL
+    # Install packages to allow apt to use a repository over HTTPS
+    apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg2 \
+      software-properties-common
+
+    # Add Docker’s official GPG key
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+    # Set up the stable Docker repository
+    add-apt-repository \
+      "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+      $(lsb_release -cs) \
+      stable"
+
+    # Install Docker. Unlike Fedora and CentOS, this also start the daemon.
+    apt-get update
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
+
+def deb_docker(config)
+  config.vm.provision 'install Docker using apt', type: 'shell', inline: <<-SHELL
+    # Install packages to allow apt to use a repository over HTTPS
+    apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg2 \
+      software-properties-common
+
+    # Add Docker’s official GPG key
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+
+    # Set up the stable Docker repository
+    add-apt-repository \
+      "deb [arch=amd64] https://download.docker.com/linux/debian \
+      $(lsb_release -cs) \
+      stable"
+
+    # Install Docker. Unlike Fedora and CentOS, this also start the daemon.
+    apt-get update
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
 def rpm_common(config, name)
   linux_common(
     config,
@@ -195,6 +258,25 @@ def rpm_common(config, name)
   )
 end
 
+def rpm_docker(config)
+  config.vm.provision 'install Docker using yum', type: 'shell', inline: <<-SHELL
+    # Install prerequisites
+    yum install -y yum-utils device-mapper-persistent-data lvm2
+
+    # Add repository
+    yum-config-manager -y --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+    # Install Docker
+    yum install -y docker-ce docker-ce-cli containerd.io
+
+    # Start Docker
+    systemctl enable --now docker
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
 def dnf_common(config, name)
   # Autodetect doesn't work....
   if Vagrant.has_plugin?('vagrant-cachier')
@@ -211,6 +293,25 @@ def dnf_common(config, name)
   )
 end
 
+def dnf_docker(config)
+  config.vm.provision 'install Docker using dnf', type: 'shell', inline: <<-SHELL
+    # Install prerequisites
+    dnf -y install dnf-plugins-core
+
+    # Add repository
+    dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
+
+    # Install Docker
+    dnf install -y docker-ce docker-ce-cli containerd.io
+
+    # Start Docker
+    systemctl enable --now docker
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
+end
+
 def suse_common(config, name, extra: '')
   linux_common(
     config,
@@ -268,7 +369,7 @@ def linux_common(config,
 
   # This prevents leftovers from previous tests using the
   # same VM from messing up the current test
-  config.vm.provision 'clean es installs in tmp', run: 'always', type: 'shell', inline: <<-SHELL
+  config.vm.provision 'clean es installs in tmp', type: 'shell', inline: <<-SHELL
     rm -rf /tmp/elasticsearch*
   SHELL
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy

@@ -20,20 +20,16 @@
 package org.elasticsearch.gradle
 
 import org.elasticsearch.gradle.precommit.DependencyLicensesTask
-import org.gradle.api.DefaultTask
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.Dependency
-import org.gradle.api.artifacts.DependencyResolutionListener
 import org.gradle.api.artifacts.DependencySet
 import org.gradle.api.internal.ConventionTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
+import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 
-import java.util.regex.Matcher
-import java.util.regex.Pattern
-
 /**
  * A task to gather information about the dependencies and export them into a csv file.
  *
@@ -46,31 +42,31 @@ import java.util.regex.Pattern
  * </ul>
  *
  */
-public class DependenciesInfoTask extends ConventionTask {
+class DependenciesInfoTask extends ConventionTask {
 
     /** Dependencies to gather information from. */
-    @Input
-    public Configuration runtimeConfiguration
+    @InputFiles
+    Configuration runtimeConfiguration
 
     /** We subtract compile-only dependencies. */
-    @Input
-    public Configuration compileOnlyConfiguration
-
-    private LinkedHashMap<String, String> mappings
+    @InputFiles
+    Configuration compileOnlyConfiguration
 
     /** Directory to read license files */
     @InputDirectory
-    public File licensesDir = new File(project.projectDir, 'licenses')
+    File licensesDir = new File(project.projectDir, 'licenses')
 
     @OutputFile
     File outputFile = new File(project.buildDir, "reports/dependencies/dependencies.csv")
 
-    public DependenciesInfoTask() {
+    private LinkedHashMap<String, String> mappings
+
+    DependenciesInfoTask() {
         description = 'Create a CSV file with dependencies information.'
     }
 
     @TaskAction
-    public void generateDependenciesInfo() {
+    void generateDependenciesInfo() {
 
         final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies()
         // we have to resolve the transitive dependencies and create a group:artifactId:version map

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -31,7 +31,7 @@ import java.nio.file.Path
 /**
  * Generates REST tests for each snippet marked // TEST.
  */
-public class RestTestsFromSnippetsTask extends SnippetsTask {
+class RestTestsFromSnippetsTask extends SnippetsTask {
     /**
      * These languages aren't supported by the syntax highlighter so we
      * shouldn't use them.
@@ -58,7 +58,9 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
     @OutputDirectory
     File testRoot = project.file('build/rest')
 
-    public RestTestsFromSnippetsTask() {
+    Set<String> names = new HashSet<>()
+
+    RestTestsFromSnippetsTask() {
         project.afterEvaluate {
             // Wait to set this so testRoot can be customized
             project.sourceSets.test.output.dir(testRoot, builtBy: this)
@@ -238,7 +240,14 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 }
             } else {
                 current.println('---')
-                current.println("\"line_$test.start\":")
+                if (test.name != null && test.name.isBlank() == false) {
+                    if(names.add(test.name) == false) {
+                        throw new InvalidUserDataException("Duplicated snippet name '$test.name': $test")
+                    }
+                    current.println("\"$test.name\":")
+                } else {
+                    current.println("\"line_$test.start\":")
+                }
                 /* The Elasticsearch test runner doesn't support quite a few
                  * constructs unless we output this skip. We don't know if
                  * we're going to use these constructs, but we might so we
@@ -406,6 +415,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             if (lastDocsPath == test.path) {
                 return
             }
+            names.clear()
             finishLastTest()
             lastDocsPath = test.path
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -28,6 +28,7 @@ import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.ConfigurableFileTree
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskAction
 
 import java.nio.file.Path
@@ -36,7 +37,7 @@ import java.util.regex.Matcher
 /**
  * A task which will run a closure on each snippet in the documentation.
  */
-public class SnippetsTask extends DefaultTask {
+class SnippetsTask extends DefaultTask {
     private static final String SCHAR = /(?:\\\/|[^\/])/
     private static final String SUBSTITUTION = /s\/($SCHAR+)\/($SCHAR*)\//
     private static final String CATCH = /catch:\s*((?:\/[^\/]+\/)|[^ \]]+)/
@@ -51,6 +52,7 @@ public class SnippetsTask extends DefaultTask {
      * Action to take on each snippet. Called with a single parameter, an
      * instance of Snippet.
      */
+    @Internal
     Closure perSnippet
 
     /**
@@ -73,13 +75,14 @@ public class SnippetsTask extends DefaultTask {
     Map<String, String> defaultSubstitutions = [:]
 
     @TaskAction
-    public void executeTask() {
+    void executeTask() {
         /*
          * Walks each line of each file, building snippets as it encounters
          * the lines that make up the snippet.
          */
         for (File file: docs) {
             String lastLanguage
+            String name
             int lastLanguageLine
             Snippet snippet = null
             StringBuilder contents = null
@@ -153,19 +156,21 @@ public class SnippetsTask extends DefaultTask {
                 if (line ==~ /-{4,}\s*/) { // Four dashes looks like a snippet
                     if (snippet == null) {
                         Path path = docs.dir.toPath().relativize(file.toPath())
-                        snippet = new Snippet(path: path, start: lineNumber, testEnv: testEnv)
+                        snippet = new Snippet(path: path, start: lineNumber, testEnv: testEnv, name: name)
                         if (lastLanguageLine == lineNumber - 1) {
                             snippet.language = lastLanguage
                         }
+                        name = null
                     } else {
                         snippet.end = lineNumber
                     }
                     return
                 }
-                matcher = line =~ /\["?source"?,\s*"?([-\w]+)"?(,.*)?].*/
-                if (matcher.matches()) {
-                    lastLanguage = matcher.group(1)
+                def source = matchSource(line)
+                if (source.matches) {
+                    lastLanguage = source.language
                     lastLanguageLine = lineNumber
+                    name = source.name
                     return
                 }
                 if (line ==~ /\/\/\s*AUTOSENSE\s*/) {
@@ -308,6 +313,20 @@ public class SnippetsTask extends DefaultTask {
         }
     }
 
+    static Source matchSource(String line) {
+        def matcher = line =~ /\["?source"?,\s*"?([-\w]+)"?(,((?!id=).)*(id="?([-\w]+)"?)?(.*))?].*/
+        if(matcher.matches()){
+            return new Source(matches: true, language: matcher.group(1),  name: matcher.group(5))
+        }
+        return new Source(matches: false)
+    }
+
+    static class Source {
+        boolean matches
+        String language
+        String name
+    }
+
     static class Snippet {
         static final int NOT_FINISHED = -1
 
@@ -334,6 +353,7 @@ public class SnippetsTask extends DefaultTask {
         boolean curl
         List warnings = new ArrayList()
         boolean skipShardsFailures = false
+        String name
 
         @Override
         public String toString() {