Merge branch 'master' into feature/searchable-snapshots

Author: tlrx

.ci/dockerOnLinuxExclusions

@@ -7,9 +7,11 @@
 debian-8
 opensuse-15-1
 ol-7.7
-sles-12
+sles-12.3 # older version used in Vagrant image
+sles-12.4
+sles-15.1
 
-# These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded 
+# These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded
 # for PR checks
 centos-6
 ol-6.10

build.gradle

@@ -34,6 +34,7 @@ import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 plugins {
   id 'lifecycle-base'
+  id 'elasticsearch.docker-support'
   id 'elasticsearch.global-build-info'
   id "com.diffplug.gradle.spotless" version "3.24.2" apply false
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -32,7 +32,6 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
-import org.elasticsearch.gradle.testclusters.TestDistribution
 import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.Action
 import org.gradle.api.GradleException
@@ -83,8 +82,6 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
-import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable
-import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -186,51 +183,6 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    static void requireDocker(final Task task) {
-        final Project rootProject = task.project.rootProject
-        ExtraPropertiesExtension ext = rootProject.extensions.getByType(ExtraPropertiesExtension)
-
-        if (rootProject.hasProperty('requiresDocker') == false) {
-            /*
-             * This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
-             * that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
-             * in the task graph, then we check two things:
-             *  - the Docker binary is available
-             *  - we can execute a Docker command that requires privileges
-             *
-             *  If either of these fail, we fail the build.
-             */
-
-            // check if the Docker binary exists and record its path
-            final String dockerBinary = getDockerPath().orElse(null)
-
-            final boolean buildDocker
-            final String buildDockerProperty = System.getProperty("build.docker")
-            if (buildDockerProperty == null) {
-                buildDocker = dockerBinary != null
-            } else if (buildDockerProperty == "true") {
-                buildDocker = true
-            } else if (buildDockerProperty == "false") {
-                buildDocker = false
-            } else {
-                throw new IllegalArgumentException(
-                        "expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
-            }
-
-            ext.set('buildDocker', buildDocker)
-            ext.set('requiresDocker', [])
-            rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
-                final List<String> tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List<Task>).collect { "  ${it.path}".toString()}
-
-                if (tasks.isEmpty() == false) {
-                    assertDockerIsAvailable(task.project, tasks)
-                }
-            }
-        }
-
-        (ext.get('requiresDocker') as List<Task>).add(task)
-    }
-
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
     static void requireJavaHome(Task task, int version) {
         // use root project for global accounting

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(RestResourcesPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,29 +18,19 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
-import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.gradle.api.file.FileCopyDetails
-import org.gradle.api.tasks.Copy
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.testing.Test
-import org.gradle.plugins.ide.idea.IdeaPlugin
+
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
 class RestIntegTestTask extends DefaultTask {
 
     protected Test runner
 
-    /** Flag indicating whether the rest tests in the rest spec should be run. */
-    @Input
-    Boolean includePackaged = false
-
     RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
@@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.systemProperty('test.clustername', System.getProperty("tests.clustername"))
         }
 
-        // copy the rest spec/tests onto the test classpath
-        Copy copyRestSpec = createCopyRestSpecTask()
-        project.sourceSets.test.output.builtBy(copyRestSpec)
-
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask {
         }
     }
 
-    /** Sets the includePackaged property */
-    public void includePackaged(boolean include) {
-        includePackaged = include
-    }
-
-
     @Override
     public Task dependsOn(Object... dependencies) {
         runner.dependsOn(dependencies)
@@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask {
         project.tasks.getByName("${name}Runner").configure(configure)
     }
 
-    Copy createCopyRestSpecTask() {
-        Boilerplate.maybeCreate(project.configurations, 'restSpec') {
-            project.dependencies.add(
-                    'restSpec',
-                    BuildParams.internal ? project.project(':rest-api-spec') :
-                            "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
-            )
-        }
-
-        return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy ->
-            copy.dependsOn project.configurations.restSpec
-            copy.into(project.sourceSets.test.output.resourcesDir)
-            copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
-                includeEmptyDirs = false
-                include 'rest-api-spec/**'
-                filesMatching('rest-api-spec/test/**') { FileCopyDetails details ->
-                    if (includePackaged == false) {
-                        details.exclude()
-                    }
-                }
-            }
-
-            if (project.plugins.hasPlugin(IdeaPlugin)) {
-                project.idea {
-                    module {
-                        if (scopes.TEST != null) {
-                            scopes.TEST.plus.add(project.configurations.restSpec)
-                        }
-                    }
-                }
-            }
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -26,9 +26,9 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.InvalidUserDataException
-import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
@@ -42,6 +42,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.testing.Test
 import org.gradle.plugins.ide.eclipse.model.EclipseModel
 import org.gradle.plugins.ide.idea.model.IdeaModel
+
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -74,6 +75,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         // only setup tests to build
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')
+        // need to apply plugin after test source sets are created
+        project.pluginManager.apply(RestResourcesPlugin)
 
         project.tasks.withType(Test) { Test test ->
             test.testClassesDirs = testSourceSet.output.classesDirs

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -22,8 +22,11 @@
 import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.elasticsearch.gradle.docker.DockerSupportPlugin;
+import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.tool.Boilerplate;
 import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
@@ -37,6 +40,7 @@
 import org.gradle.api.credentials.HttpHeaderCredentials;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.authentication.http.HttpHeaderAuthentication;
@@ -71,11 +75,17 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     public void apply(Project project) {
         // this is needed for isInternal
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
+
+        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+            project.getGradle().getSharedServices(),
+            DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
+        );
 
         distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
             Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
             Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
-            return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration);
+            return new ElasticsearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration);
         });
         project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
 

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -19,13 +19,16 @@
 
 package org.elasticsearch.gradle;
 
+import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.gradle.api.Buildable;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskDependency;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Locale;
 
@@ -110,6 +113,7 @@ public String toString() {
     }
 
     private final String name;
+    private final Provider<DockerSupportService> dockerSupport;
     // pkg private so plugin can configure
     final Configuration configuration;
     private final Extracted extracted;
@@ -119,21 +123,25 @@ public String toString() {
     private final Property<Platform> platform;
     private final Property<Flavor> flavor;
     private final Property<Boolean> bundledJdk;
+    private final Property<Boolean> failIfUnavailable;
 
     ElasticsearchDistribution(
         String name,
         ObjectFactory objectFactory,
+        Provider<DockerSupportService> dockerSupport,
         Configuration fileConfiguration,
         Configuration extractedConfiguration
     ) {
         this.name = name;
+        this.dockerSupport = dockerSupport;
         this.configuration = fileConfiguration;
         this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
         this.type = objectFactory.property(Type.class);
         this.type.convention(Type.ARCHIVE);
         this.platform = objectFactory.property(Platform.class);
         this.flavor = objectFactory.property(Flavor.class);
         this.bundledJdk = objectFactory.property(Boolean.class);
+        this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
         this.extracted = new Extracted(extractedConfiguration);
     }
 
@@ -182,6 +190,14 @@ public void setBundledJdk(Boolean bundledJdk) {
         this.bundledJdk.set(bundledJdk);
     }
 
+    public boolean getFailIfUnavailable() {
+        return this.failIfUnavailable.get();
+    }
+
+    public void setFailIfUnavailable(boolean failIfUnavailable) {
+        this.failIfUnavailable.set(failIfUnavailable);
+    }
+
     @Override
     public String toString() {
         return configuration.getSingleFile().toString();
@@ -203,6 +219,13 @@ public Extracted getExtracted() {
 
     @Override
     public TaskDependency getBuildDependencies() {
+        // For non-required Docker distributions, skip building the distribution is Docker is unavailable
+        if (getType() == Type.DOCKER
+            && getFailIfUnavailable() == false
+            && dockerSupport.get().getDockerAvailability().isAvailable == false) {
+            return task -> Collections.emptySet();
+        }
+
         return configuration.getBuildDependencies();
     }
 
@@ -222,7 +245,7 @@ void finalizeValues() {
         if (getType() == Type.INTEG_TEST_ZIP) {
             if (platform.getOrNull() != null) {
                 throw new IllegalArgumentException(
-                    "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"
+                    "platform cannot be set on elasticsearch distribution [" + name + "] of type [integ_test_zip]"
                 );
             }
             if (flavor.getOrNull() != null) {
@@ -232,12 +255,18 @@ void finalizeValues() {
             }
             if (bundledJdk.getOrNull() != null) {
                 throw new IllegalArgumentException(
-                    "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"
+                    "bundledJdk cannot be set on elasticsearch distribution [" + name + "] of type [integ_test_zip]"
                 );
             }
             return;
         }
 
+        if (getType() != Type.DOCKER && failIfUnavailable.get() == false) {
+            throw new IllegalArgumentException(
+                "failIfUnavailable cannot be 'false' on elasticsearch distribution [" + name + "] of type [" + getType() + "]"
+            );
+        }
+
         if (getType() == Type.ARCHIVE) {
             // defaults for archive, set here instead of via convention so integ-test-zip can verify they are not set
             if (platform.isPresent() == false) {
@@ -246,7 +275,12 @@ void finalizeValues() {
         } else { // rpm, deb or docker
             if (platform.isPresent()) {
                 throw new IllegalArgumentException(
-                    "platform not allowed for elasticsearch distribution [" + name + "] of type [" + getType() + "]"
+                    "platform cannot be set on elasticsearch distribution [" + name + "] of type [" + getType() + "]"
+                );
+            }
+            if (getType() == Type.DOCKER && bundledJdk.isPresent()) {
+                throw new IllegalArgumentException(
+                    "bundledJdk cannot be set on elasticsearch distribution [" + name + "] of type [docker]"
                 );
             }
         }