Merge remote-tracking branch 'elastic/master' into pr/36141

* elastic/master: (36 commits)
  Add check for minimum required Docker version (elastic#36497)
  Minor search controller changes (elastic#36479)
  Add default methods to DocValueFormat (elastic#36480)
  Fix the mixed cluster REST test explain/11_basic_with_types.
  Modify `BigArrays` to take name of circuit breaker (elastic#36461)
  Move LoggedExec to minimumRuntime source set (elastic#36453)
  added 6.5.4 version
  Add test logging for elastic#35644
  Tests- added helper methods to ESRestTestCase for checking warnings (elastic#36443)
  SQL: move requests' parameters to requests JSON body (elastic#36149)
  [Zen2] Respect the no_master_block setting (elastic#36478)
  Require soft-deletes when access changes snapshot (elastic#36446)
  Switch more tests to zen2 (elastic#36367)
  [Painless] Add extensive tests for def to primitive casts (elastic#36455)
  Add setting to bypass Rollover action (elastic#36235)
  Try running CI against Zulu (elastic#36391)
  [DOCS] Reworked the shard allocation filtering info.  (elastic#36456)
  Log [initial_master_nodes] on formation failure (elastic#36466)
  converting ForbiddenPatternsTask to .java (elastic#36194)
  fixed typo
  ...

Author: jasontedor

.ci/matrix-runtime-javas.yml

@@ -9,3 +9,5 @@ ES_RUNTIME_JAVA:
   - java8
   - java8fips
   - java11
+  - zulu8
+  - zulu11

buildSrc/build.gradle

@@ -95,6 +95,7 @@ dependencies {
   }
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
   minimumRuntimeCompile localGroovy()
+  minimumRuntimeCompile gradleApi()
 }
 jar {
   from sourceSets.minimumRuntime.output
@@ -217,7 +218,7 @@ if (project != rootProject) {
   forbiddenPatterns {
     exclude '**/*.wav'
     // the file that actually defines nocommit
-    exclude '**/ForbiddenPatternsTask.groovy'
+    exclude '**/ForbiddenPatternsTask.java'
   }
 
   namingConventions {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -57,6 +57,7 @@ import org.gradle.util.GradleVersion
 import java.nio.charset.StandardCharsets
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
+import java.util.regex.Matcher
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -245,9 +246,16 @@ class BuildPlugin implements Plugin<Project> {
              *
              *  If either of these fail, we fail the build.
              */
+
+            // check if the Docker binary exists and record its path
+            final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
+            final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+
             final boolean buildDocker
             final String buildDockerProperty = System.getProperty("build.docker")
-            if (buildDockerProperty == null || buildDockerProperty == "true") {
+            if (buildDockerProperty == null) {
+                buildDocker = dockerBinary != null
+            } else if (buildDockerProperty == "true") {
                 buildDocker = true
             } else if (buildDockerProperty == "false") {
                 buildDocker = false
@@ -258,60 +266,68 @@ class BuildPlugin implements Plugin<Project> {
             rootProject.rootProject.ext.buildDocker = buildDocker
             rootProject.rootProject.ext.requiresDocker = []
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
-                // check if the Docker binary exists and record its path
-                final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
-                final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
-
-                int exitCode
-                String dockerErrorOutput
-                if (dockerBinary == null) {
-                    exitCode = -1
-                    dockerErrorOutput = null
-                } else {
-                    // the Docker binary executes, check that we can execute a privileged command
-                    final ByteArrayOutputStream output = new ByteArrayOutputStream()
-                    final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
-                        it.commandLine dockerBinary, "images"
-                        it.errorOutput = output
-                        it.ignoreExitValue = true
-                    })
-                    if (result.exitValue == 0) {
-                        return
-                    }
-                    exitCode = result.exitValue
-                    dockerErrorOutput = output.toString()
-                }
                 final List<String> tasks =
                         ((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString()}
                 if (tasks.isEmpty() == false) {
                     /*
                      * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
                      * exist or because execution of a privileged Docker command failed.
                      */
-                    String message
                     if (dockerBinary == null) {
-                        message = String.format(
+                        final String message = String.format(
                                 Locale.ROOT,
                                 "Docker (checked [%s]) is required to run the following task%s: \n%s",
                                 maybeDockerBinaries.join(","),
                                 tasks.size() > 1 ? "s" : "",
                                 tasks.join('\n'))
-                    } else {
-                        assert exitCode > 0 && dockerErrorOutput != null
-                        message = String.format(
+                        throwDockerRequiredException(message)
+                    }
+
+                    // we use a multi-stage Docker build, check the Docker version since 17.05
+                    final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream()
+                    LoggedExec.exec(
+                            rootProject,
+                            { ExecSpec it ->
+                                it.commandLine = [dockerBinary, '--version']
+                                it.standardOutput = dockerVersionOutput
+                            })
+                    final String dockerVersion = dockerVersionOutput.toString().trim()
+                    final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-ce)?, build [0-9a-f]{7}/
+                    assert matcher.matches() : dockerVersion
+                    final dockerMajorMinorVersion = matcher.group(1)
+                    final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
+                    if (Integer.parseInt(majorMinor[0]) < 17
+                            || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
+                        final String message = String.format(
+                                Locale.ROOT,
+                                "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
+                                dockerVersion)
+                        throwDockerRequiredException(message)
+                    }
+
+                    final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream()
+                    // the Docker binary executes, check that we can execute a privileged command
+                    final ExecResult dockerImagesResult = LoggedExec.exec(
+                            rootProject,
+                            { ExecSpec it ->
+                                it.commandLine = [dockerBinary, "images"]
+                                it.errorOutput = dockerImagesErrorOutput
+                                it.ignoreExitValue = true
+                            })
+
+                    if (dockerImagesResult.exitValue != 0) {
+                        final String message = String.format(
                                 Locale.ROOT,
                                 "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
                                         "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
                                 dockerBinary,
                                 tasks.size() > 1 ? "s" : "",
                                 tasks.join('\n'),
-                                exitCode,
-                                dockerErrorOutput.trim())
+                                dockerImagesResult.exitValue,
+                                dockerImagesErrorOutput.toString().trim())
+                        throwDockerRequiredException(message)
                     }
-                    throw new GradleException(
-                            message + "\nyou can address this by attending to the reported issue, "
-                                    + "removing the offending tasks from being executed, "
-                                    + "or by passing -Dbuild.docker=false")
+
                 }
             }
         }
@@ -322,6 +338,13 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    private static void throwDockerRequiredException(final String message) {
+        throw new GradleException(
+                message + "\nyou can address this by attending to the reported issue, "
+                        + "removing the offending tasks from being executed, "
+                        + "or by passing -Dbuild.docker=false")
+    }
+
     private static String findCompilerJavaHome() {
         String compilerJavaHome = System.getenv('JAVA_HOME')
         final String compilerJavaProperty = System.getProperty('compiler.java')