From 2fa83d51af25880f304a8394f1a6ed93d721be91 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 15 Jul 2019 10:39:05 -0700 Subject: [PATCH] Convert testclusters to use distro download plugin (#44253) Test clusters currently has its own set of logic for dealing with finding different versions of Elasticsearch, downloading them, and extracting them. This commit converts testclusters to use the DistributionDownloadPlugin. --- .../gradle/doc/DocsTestPlugin.groovy | 4 +- .../gradle/test/RestIntegTestTask.groovy | 6 +- .../elasticsearch/gradle/Distribution.java | 80 -------- .../gradle/DistributionDownloadPlugin.java | 8 +- .../gradle/ElasticsearchDistribution.java | 7 + .../gradle/test/RestTestRunnerTask.java | 4 +- .../testclusters/ElasticsearchCluster.java | 31 ++- .../testclusters/ElasticsearchNode.java | 68 ++++--- .../TestClusterConfiguration.java | 3 +- .../testclusters/TestClustersPlugin.java | 177 +----------------- .../gradle/testclusters/TestDistribution.java | 28 +++ .../DistributionDownloadPluginTests.java | 4 +- .../src/testKit/testclusters/build.gradle | 12 +- .../src/testKit/thirdPartyAudit/build.gradle | 4 +- client/rest-high-level/build.gradle | 2 +- docs/build.gradle | 4 +- .../examples/painless-whitelist/build.gradle | 2 +- .../build.gradle | 2 +- qa/build.gradle | 7 +- x-pack/plugin/build.gradle | 2 +- .../downgrade-to-basic-license/build.gradle | 4 +- .../plugin/ccr/qa/multi-cluster/build.gradle | 6 +- .../ccr/qa/non-compliant-license/build.gradle | 4 +- x-pack/plugin/ccr/qa/rest/build.gradle | 2 +- x-pack/plugin/ccr/qa/restart/build.gradle | 4 +- x-pack/plugin/ccr/qa/security/build.gradle | 4 +- .../qa/multi-node-tests/build.gradle | 2 +- .../qa/single-node-tests/build.gradle | 2 +- .../graph/qa/with-security/build.gradle | 2 +- .../plugin/ilm/qa/multi-cluster/build.gradle | 4 +- x-pack/plugin/ilm/qa/multi-node/build.gradle | 2 +- x-pack/plugin/ilm/qa/rest/build.gradle | 2 +- .../plugin/ilm/qa/with-security/build.gradle | 2 +- .../ml/qa/basic-multi-node/build.gradle | 2 +- x-pack/plugin/ml/qa/disabled/build.gradle | 2 +- .../ml/qa/ml-with-security/build.gradle | 2 +- .../qa/native-multi-node-tests/build.gradle | 2 +- .../ml/qa/single-node-tests/build.gradle | 2 +- .../qa/basic-enable-security/build.gradle | 2 +- .../security/qa/security-basic/build.gradle | 2 +- .../plugin/security/qa/tls-basic/build.gradle | 2 +- x-pack/plugin/sql/qa/build.gradle | 2 +- x-pack/plugin/sql/qa/security/build.gradle | 2 +- .../build.gradle | 2 +- x-pack/qa/kerberos-tests/build.gradle | 2 +- .../build.gradle | 4 +- x-pack/qa/multi-node/build.gradle | 2 +- x-pack/qa/oidc-op-tests/build.gradle | 2 +- .../reindex-tests-with-security/build.gradle | 2 +- x-pack/qa/saml-idp-tests/build.gradle | 2 +- .../build.gradle | 2 +- .../build.gradle | 2 +- .../build.gradle | 2 +- x-pack/qa/smoke-test-plugins-ssl/build.gradle | 2 +- x-pack/qa/smoke-test-plugins/build.gradle | 2 +- .../build.gradle | 2 +- .../build.gradle | 2 +- x-pack/qa/smoke-test-watcher/build.gradle | 2 +- 58 files changed, 177 insertions(+), 370 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy index 805a1b213e859..53a022a59052a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy @@ -21,11 +21,9 @@ package org.elasticsearch.gradle.doc import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.test.ClusterFormationTasks import org.elasticsearch.gradle.test.RestTestPlugin import org.gradle.api.Project import org.gradle.api.Task - /** * Sets up tests for documentation. */ @@ -38,7 +36,7 @@ public class DocsTestPlugin extends RestTestPlugin { super.apply(project) String distribution = System.getProperty('tests.distribution', 'default') // The distribution can be configured with -Dtests.distribution on the command line - project.testClusters.integTest.distribution = distribution.toUpperCase() + project.testClusters.integTest.testDistribution = distribution.toUpperCase() project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") } // Docs are published separately so no need to assemble project.tasks.assemble.enabled = false diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index 9d6113c60fbd2..431cac6ca764f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -18,7 +18,7 @@ */ package org.elasticsearch.gradle.test -import org.elasticsearch.gradle.VersionProperties + import org.elasticsearch.gradle.testclusters.ElasticsearchCluster import org.elasticsearch.gradle.testclusters.TestClustersPlugin import org.gradle.api.DefaultTask @@ -26,7 +26,6 @@ import org.gradle.api.Task import org.gradle.api.execution.TaskExecutionAdapter import org.gradle.api.logging.Logger import org.gradle.api.logging.Logging -import org.gradle.api.specs.Specs import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Input import org.gradle.api.tasks.TaskState @@ -38,7 +37,6 @@ import org.gradle.process.CommandLineArgumentProvider import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.stream.Stream - /** * A wrapper task around setting up a cluster and running rest tests. */ @@ -70,8 +68,6 @@ class RestIntegTestTask extends DefaultTask { } else { project.testClusters { "$name" { - distribution = 'INTEG_TEST' - version = VersionProperties.elasticsearch javaHome = project.file(project.ext.runtimeJavaHome) } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java deleted file mode 100644 index 9cb3cc52dd09e..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle; - -public enum Distribution { - - INTEG_TEST("elasticsearch"), - DEFAULT("elasticsearch"), - OSS("elasticsearch-oss"); - - private final String artifactName; - - Distribution(String name) { - this.artifactName = name; - } - - public String getArtifactName() { - return artifactName; - } - - public String getGroup() { - if (this.equals(INTEG_TEST)) { - return "org.elasticsearch.distribution.integ-test-zip"; - } else { - return "org.elasticsearch.distribution." + name().toLowerCase(); - } - } - - public String getFileExtension() { - if (this.equals(INTEG_TEST)) { - return "zip"; - } else { - return OS.conditionalString() - .onUnix(() -> "tar.gz") - .onWindows(() -> "zip") - .supply(); - } - } - - public String getClassifier() { - if (this.equals(INTEG_TEST)) { - return ""; - } else { - return OS.conditional() - .onLinux(() -> "linux-x86_64") - .onWindows(() -> "windows-x86_64") - .onMac(() -> "darwin-x86_64") - .supply(); - } - } - - public String getLiveConfiguration() { - if (this.equals(INTEG_TEST)) { - return "integ-test-zip"; - } else { - return (this.equals(OSS) ? "oss-" : "") + OS.conditional() - .onLinux(() -> "linux-tar") - .onWindows(() -> "windows-zip") - .onMac(() -> "darwin-tar") - .supply(); - } - } - -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 28748c00f46e1..5a3a4a277dda7 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -54,6 +54,7 @@ */ public class DistributionDownloadPlugin implements Plugin { + private static final String CONTAINER_NAME = "elasticsearch_distributions"; private static final String FAKE_IVY_GROUP = "elasticsearch-distribution"; private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads"; @@ -67,7 +68,7 @@ public void apply(Project project) { Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name); return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration); }); - project.getExtensions().add("elasticsearch_distributions", distributionsContainer); + project.getExtensions().add(CONTAINER_NAME, distributionsContainer); setupDownloadServiceRepo(project); @@ -78,6 +79,11 @@ public void apply(Project project) { project.afterEvaluate(this::setupDistributions); } + @SuppressWarnings("unchecked") + public static NamedDomainObjectContainer getContainer(Project project) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(CONTAINER_NAME); + } + // pkg private for tests void setupDistributions(Project project) { for (ElasticsearchDistribution distribution : distributionsContainer) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index ac18209a43373..3e2d639981501 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -20,7 +20,9 @@ package org.elasticsearch.gradle; import org.gradle.api.Buildable; +import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.FileTree; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; import org.gradle.api.tasks.TaskDependency; @@ -28,6 +30,7 @@ import java.io.File; import java.util.Iterator; import java.util.Locale; +import java.util.concurrent.Callable; public class ElasticsearchDistribution implements Buildable { @@ -90,6 +93,10 @@ public TaskDependency getBuildDependencies() { return configuration.getBuildDependencies(); } + public FileTree getFileTree(Project project) { + return project.fileTree((Callable) configuration::getSingleFile); + } + @Override public String toString() { return configuration.getSingleFile().toString(); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java index eff05f64f9c33..95040af9809e4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java @@ -8,7 +8,7 @@ import java.util.ArrayList; import java.util.Collection; -import static org.elasticsearch.gradle.Distribution.INTEG_TEST; +import static org.elasticsearch.gradle.testclusters.TestDistribution.INTEG_TEST; /** * Customized version of Gradle {@link Test} task which tracks a collection of {@link ElasticsearchCluster} as a task input. We must do this @@ -23,7 +23,7 @@ public class RestTestRunnerTask extends Test { public RestTestRunnerTask() { super(); this.getOutputs().doNotCacheIf("Build cache is only enabled for tests against clusters using the 'integ-test' distribution", - task -> clusters.stream().flatMap(c -> c.getNodes().stream()).anyMatch(n -> n.getDistribution() != INTEG_TEST)); + task -> clusters.stream().flatMap(c -> c.getNodes().stream()).anyMatch(n -> n.getTestDistribution() != INTEG_TEST)); } @Nested diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index c343f56525aea..62621e51396e6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -18,10 +18,9 @@ */ package org.elasticsearch.gradle.testclusters; -import org.elasticsearch.gradle.Distribution; +import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.PropertyNormalization; -import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.http.WaitForHttpResource; import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; @@ -43,7 +42,6 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -60,22 +58,23 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named { private final String clusterName; private final NamedDomainObjectContainer nodes; private final File workingDirBase; - private final File artifactsExtractDir; + private final Function distributionFactory; private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); private final Project project; - public ElasticsearchCluster(String path, String clusterName, Project project, File artifactsExtractDir, File workingDirBase) { + public ElasticsearchCluster(String path, String clusterName, Project project, + Function distributionFactory, File workingDirBase) { this.path = path; this.clusterName = clusterName; this.project = project; + this.distributionFactory = distributionFactory; this.workingDirBase = workingDirBase; - this.artifactsExtractDir = artifactsExtractDir; this.nodes = project.container(ElasticsearchNode.class); this.nodes.add( new ElasticsearchNode( path, clusterName + "-0", - project, artifactsExtractDir, workingDirBase - ) + project, workingDirBase, distributionFactory.apply(0) + ) ); // configure the cluster name eagerly so nodes know about it this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName))); @@ -98,8 +97,8 @@ public void setNumberOfNodes(int numberOfNodes) { for (int i = nodes.size() ; i < numberOfNodes; i++) { this.nodes.add(new ElasticsearchNode( - path, clusterName + "-" + i, project, artifactsExtractDir, workingDirBase - )); + path, clusterName + "-" + i, project, workingDirBase, distributionFactory.apply(i) + )); } } @@ -121,8 +120,8 @@ public void setVersion(String version) { } @Override - public void setDistribution(Distribution distribution) { - nodes.all(each -> each.setDistribution(distribution)); + public void setTestDistribution(TestDistribution distribution) { + nodes.all(each -> each.setTestDistribution(distribution)); } @Override @@ -248,7 +247,7 @@ public void start() { for (ElasticsearchNode node : nodes) { if (nodeNames != null) { // Can only configure master nodes if we have node names defined - if (Version.fromString(node.getVersion()).getMajor() >= 7) { + if (node.getVersion().getMajor() >= 7) { node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]"); node.defaultConfig.put("discovery.seed_providers", "file"); node.defaultConfig.put("discovery.seed_hosts", "[]"); @@ -338,12 +337,6 @@ public boolean isProcessAlive() { return nodes.stream().noneMatch(node -> node.isProcessAlive() == false); } - void eachVersionedDistribution(BiConsumer consumer) { - nodes.forEach(each -> { - consumer.accept(each.getVersion(), each.getDistribution()); - }); - } - public ElasticsearchNode singleNode() { if (nodes.size() != 1) { throw new IllegalStateException( diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 1a887b45af84b..6ae9942ddfadf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.gradle.testclusters; -import org.elasticsearch.gradle.Distribution; +import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.LazyPropertyList; import org.elasticsearch.gradle.LazyPropertyMap; @@ -26,6 +26,7 @@ import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.PropertyNormalization; import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.http.WaitForHttpResource; import org.gradle.api.Action; import org.gradle.api.Named; @@ -37,6 +38,7 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Nested; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; @@ -47,12 +49,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.LineNumberReader; - import java.io.UncheckedIOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.time.Instant; @@ -107,7 +109,6 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final String name; private final Project project; private final AtomicBoolean configurationFrozen = new AtomicBoolean(false); - private final Path artifactsExtractDir; private final Path workingDir; private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); @@ -133,19 +134,21 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final Path esStderrFile; private final Path tmpDir; - private Distribution distribution; private String version; + private TestDistribution testDistribution; + private ElasticsearchDistribution distribution; private File javaHome; private volatile Process esProcess; private Function nameCustomization = Function.identity(); private boolean isWorkingDirConfigured = false; - ElasticsearchNode(String path, String name, Project project, File artifactsExtractDir, File workingDirBase) { + ElasticsearchNode(String path, String name, Project project, File workingDirBase, + ElasticsearchDistribution distribution) { this.path = path; this.name = name; this.project = project; - this.artifactsExtractDir = artifactsExtractDir.toPath(); this.workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath(); + this.distribution = distribution; confPathRepo = workingDir.resolve("repo"); configFile = workingDir.resolve("config/elasticsearch.yml"); confPathData = workingDir.resolve("data"); @@ -156,15 +159,18 @@ public class ElasticsearchNode implements TestClusterConfiguration { esStderrFile = confPathLogs.resolve("es.stderr.log"); tmpDir = workingDir.resolve("tmp"); waitConditions.put("ports files", this::checkPortsFilesExistWithDelay); + + setTestDistribution(TestDistribution.INTEG_TEST); + setVersion(VersionProperties.getElasticsearch()); } public String getName() { return nameCustomization.apply(name); } - @Input - public String getVersion() { - return version; + @Internal + public Version getVersion() { + return distribution.getVersion(); } @Override @@ -172,18 +178,35 @@ public void setVersion(String version) { requireNonNull(version, "null version passed when configuring test cluster `" + this + "`"); checkFrozen(); this.version = version; + this.distribution.setVersion(version); } - @Input - public Distribution getDistribution() { + @Internal + public TestDistribution getTestDistribution() { + return testDistribution; + } + + // package private just so test clusters plugin can access to wire up task dependencies + @Internal + ElasticsearchDistribution getDistribution() { return distribution; } @Override - public void setDistribution(Distribution distribution) { - requireNonNull(distribution, "null distribution passed when configuring test cluster `" + this + "`"); + public void setTestDistribution(TestDistribution testDistribution) { + requireNonNull(testDistribution, "null distribution passed when configuring test cluster `" + this + "`"); checkFrozen(); - this.distribution = distribution; + this.testDistribution = testDistribution; + if (testDistribution == TestDistribution.INTEG_TEST) { + this.distribution.setType(ElasticsearchDistribution.Type.INTEG_TEST_ZIP); + } else { + this.distribution.setType(ElasticsearchDistribution.Type.ARCHIVE); + if (testDistribution == TestDistribution.DEFAULT) { + this.distribution.setFlavor(ElasticsearchDistribution.Flavor.DEFAULT); + } else { + this.distribution.setFlavor(ElasticsearchDistribution.Flavor.OSS); + } + } } @Override @@ -289,7 +312,7 @@ public Path getConfigDir() { @Override public void freeze() { requireNonNull(distribution, "null distribution passed when configuring test cluster `" + this + "`"); - requireNonNull(version, "null version passed when configuring test cluster `" + this + "`"); + requireNonNull(getVersion(), "null version passed when configuring test cluster `" + this + "`"); requireNonNull(javaHome, "null javaHome passed when configuring test cluster `" + this + "`"); LOGGER.info("Locking configuration of `{}`", this); configurationFrozen.set(true); @@ -332,7 +355,7 @@ public synchronized void start() { try { if (isWorkingDirConfigured == false) { logToProcessStdout("Configuring working directory: " + workingDir); - // Only configure working dir once so we don't loose data on restarts + // Only configure working dir once so we don't lose data on restarts isWorkingDirConfigured = true; createWorkingDir(getExtractedDistributionDir()); } @@ -442,10 +465,11 @@ private void copyExtraConfigFiles() { } private void installModules() { - if (distribution == Distribution.INTEG_TEST) { + if (testDistribution == TestDistribution.INTEG_TEST) { logToProcessStdout("Installing " + modules.size() + "modules"); for (File module : modules) { - Path destination = workingDir.resolve("modules").resolve(module.getName().replace(".zip", "").replace("-" + version, "")); + Path destination = workingDir.resolve("modules").resolve(module.getName().replace(".zip", "") + .replace("-" + version, "")); // only install modules that are not already bundled with the integ-test distribution if (Files.exists(destination) == false) { @@ -851,7 +875,7 @@ private void createConfiguration() { defaultConfig.put("node.attr.testattr", "test"); defaultConfig.put("node.portsfile", "true"); defaultConfig.put("http.port", "0"); - if (Version.fromString(version).onOrAfter(Version.fromString("6.7.0"))) { + if (getVersion().onOrAfter(Version.fromString("6.7.0"))) { defaultConfig.put("transport.port", "0"); } else { defaultConfig.put("transport.tcp.port", "0"); @@ -861,13 +885,13 @@ private void createConfiguration() { defaultConfig.put("cluster.routing.allocation.disk.watermark.high", "1b"); // increase script compilation limit since tests can rapid-fire script compilations defaultConfig.put("script.max_compilations_rate", "2048/1m"); - if (Version.fromString(version).getMajor() >= 6) { + if (getVersion().getMajor() >= 6) { defaultConfig.put("cluster.routing.allocation.disk.watermark.flood_stage", "1b"); } // Temporarily disable the real memory usage circuit breaker. It depends on real memory usage which we have no full control // over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client // can retry on circuit breaking exceptions, we can revert again to the default configuration. - if (Version.fromString(version).getMajor() >= 7) { + if (getVersion().getMajor() >= 7) { defaultConfig.put("indices.breaker.total.use_real_memory", "false"); } // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master @@ -939,7 +963,7 @@ private List readPortsFile(Path file) throws IOException { } private Path getExtractedDistributionDir() { - return artifactsExtractDir.resolve(distribution.getGroup()).resolve("elasticsearch-" + getVersion()); + return Paths.get(distribution.getExtracted().toString()).resolve("elasticsearch-" + version); } private List getInstalledFileSet(Action filter) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index a1733c56708ad..f0c53e4d13cc8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.gradle.testclusters; -import org.elasticsearch.gradle.Distribution; import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.PropertyNormalization; import org.gradle.api.logging.Logging; @@ -39,7 +38,7 @@ public interface TestClusterConfiguration { void setVersion(String version); - void setDistribution(Distribution distribution); + void setTestDistribution(TestDistribution distribution); void plugin(URI plugin); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index 3c50108d9a179..fd1ab7d02c96c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -19,21 +19,15 @@ package org.elasticsearch.gradle.testclusters; import groovy.lang.Closure; -import org.elasticsearch.gradle.BwcVersions; -import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.DistributionDownloadPlugin; +import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.test.RestTestRunnerTask; -import org.elasticsearch.gradle.tool.Boilerplate; -import org.gradle.api.Action; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.repositories.MavenArtifactRepository; -import org.gradle.api.credentials.HttpHeaderCredentials; import org.gradle.api.execution.TaskActionListener; import org.gradle.api.execution.TaskExecutionListener; -import org.gradle.api.file.FileTree; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.plugins.ExtraPropertiesExtension; @@ -53,8 +47,6 @@ public class TestClustersPlugin implements Plugin { private static final String LIST_TASK_NAME = "listTestClusters"; public static final String EXTENSION_NAME = "testClusters"; - private static final String HELPER_CONFIGURATION_PREFIX = "testclusters"; - private static final String SYNC_ARTIFACTS_TASK_NAME = "syncTestClustersArtifacts"; private static final Logger logger = Logging.getLogger(TestClustersPlugin.class); private static final String TESTCLUSTERS_INSPECT_FAILURE = "testclusters.inspect.failure"; @@ -64,13 +56,9 @@ public class TestClustersPlugin implements Plugin { private final Set runningClusters = new HashSet<>(); private final Boolean allowClusterToSurvive = Boolean.valueOf(System.getProperty(TESTCLUSTERS_INSPECT_FAILURE, "false")); - public static String getHelperConfigurationName(String version) { - return HELPER_CONFIGURATION_PREFIX + "-" + version; - } - @Override public void apply(Project project) { - Project rootProject = project.getRootProject(); + project.getPlugins().apply(DistributionDownloadPlugin.class); // enable the DSL to describe clusters NamedDomainObjectContainer container = createTestClustersContainerExtension(project); @@ -94,17 +82,11 @@ public void apply(Project project) { // After each task we determine if there are clusters that are no longer needed. configureStopClustersHook(project); - - // Since we have everything modeled in the DSL, add all the required dependencies e.x. the distribution to the - // configuration so the user doesn't have to repeat this. - autoConfigureClusterDependencies(project, rootProject, container); - } - - private static File getExtractDir(Project project) { - return new File(project.getRootProject().getBuildDir(), "testclusters/extract/"); } private NamedDomainObjectContainer createTestClustersContainerExtension(Project project) { + NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); + // Create an extensions that allows describing clusters NamedDomainObjectContainer container = project.container( ElasticsearchCluster.class, @@ -112,7 +94,7 @@ private NamedDomainObjectContainer createTestClustersConta project.getPath(), name, project, - new File(project.getRootProject().getBuildDir(), "testclusters/extract"), + i -> distros.create(name + "-" + i), new File(project.getBuildDir(), "testclusters") ) ); @@ -153,9 +135,9 @@ public void doCall(ElasticsearchCluster cluster) { "Task, but got: " + thisObject.getClass()); } usedClusters.computeIfAbsent(task, k -> new ArrayList<>()).add(cluster); - ((Task) thisObject).dependsOn( - project.getRootProject().getTasks().getByName(SYNC_ARTIFACTS_TASK_NAME) - ); + for (ElasticsearchNode node : cluster.getNodes()) { + ((Task) thisObject).dependsOn(node.getDistribution().getExtracted()); + } if (thisObject instanceof RestTestRunnerTask) { ((RestTestRunnerTask) thisObject).testCluster(cluster); } @@ -285,145 +267,4 @@ private void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) { } cluster.stop(taskFailed); } - - /** - * Boilerplate to get testClusters container extension - * - * Equivalent to project.testClusters in the DSL - */ - @SuppressWarnings("unchecked") - public static NamedDomainObjectContainer getNodeExtension(Project project) { - return (NamedDomainObjectContainer) - project.getExtensions().getByName(EXTENSION_NAME); - } - - private static void autoConfigureClusterDependencies( - Project project, - Project rootProject, - NamedDomainObjectContainer container - ) { - // Download integ test distribution from maven central - MavenArtifactRepository mavenCentral = project.getRepositories().mavenCentral(); - mavenCentral.content(spec -> { - spec.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*"); - }); - - // Other distributions from the download service - project.getRepositories().add( - project.getRepositories().ivy(spec -> { - spec.setUrl("https://artifacts.elastic.co/downloads"); - spec.patternLayout(p -> p.artifact("elasticsearch/[module]-[revision](-[classifier]).[ext]")); - HttpHeaderCredentials headerConfig = spec.getCredentials(HttpHeaderCredentials.class); - headerConfig.setName("X-Elastic-No-KPI"); - headerConfig.setValue("1"); - spec.content(c-> c.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*")); - }) - ); - - // We have a single task to sync the helper configuration to "artifacts dir" - // the clusters will look for artifacts there based on the naming conventions. - // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in - // the build. - Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> { - onCreate.getOutputs().dir(getExtractDir(rootProject)); - onCreate.getInputs().files( - project.getRootProject().getConfigurations().matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX)) - ); - onCreate.dependsOn(project.getRootProject().getConfigurations() - .matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX)) - ); - // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) - onCreate.doFirst(new Action() { - @Override - public void execute(Task task) { - // Clean up the extract dir first to make sure we have no stale files from older - // previous builds of the same distribution - project.delete(getExtractDir(rootProject)); - } - }); - onCreate.doLast(new Action() { - @Override - public void execute(Task task) { - project.getRootProject().getConfigurations() - .matching(config -> config.getName().startsWith(HELPER_CONFIGURATION_PREFIX)) - .forEach(config -> project.copy(spec -> - config.getResolvedConfiguration() - .getResolvedArtifacts() - .forEach(resolvedArtifact -> { - final FileTree files; - File file = resolvedArtifact.getFile(); - if (file.getName().endsWith(".zip")) { - files = project.zipTree(file); - } else if (file.getName().endsWith("tar.gz")) { - files = project.tarTree(file); - } else { - throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); - } - logger.info("Extracting {}@{}", resolvedArtifact, config); - spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup())); - spec.into(getExtractDir(project)); - })) - ); - } - }); - }); - - // When the project evaluated we know of all tasks that use clusters. - // Each of these have to depend on the artifacts being synced. - // We need afterEvaluate here despite the fact that container is a domain object, we can't implement this with - // all because fields can change after the fact. - project.afterEvaluate(ip -> container.forEach(esCluster -> - esCluster.eachVersionedDistribution((version, distribution) -> { - Configuration helperConfiguration = Boilerplate.maybeCreate( - rootProject.getConfigurations(), - getHelperConfigurationName(version), - onCreate -> - // We use a single configuration on the root project to resolve all testcluster dependencies ( like distros ) - // at once, only once without the need to repeat it for each project. This pays off assuming that most - // projects use the same dependencies. - onCreate.setDescription( - "Internal helper configuration used by cluster configuration to download " + - "ES distributions and plugins for " + version - ) - ); - BwcVersions.UnreleasedVersionInfo unreleasedInfo; - final List unreleased; - { - ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties(); - if (extraProperties.has("bwcVersions")) { - Object bwcVersionsObj = extraProperties.get("bwcVersions"); - if (bwcVersionsObj instanceof BwcVersions == false) { - throw new IllegalStateException("Expected project.bwcVersions to be of type VersionCollection " + - "but instead it was " + bwcVersionsObj.getClass()); - } - final BwcVersions bwcVersions = (BwcVersions) bwcVersionsObj; - unreleased = ((BwcVersions) bwcVersionsObj).getUnreleased(); - unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(version)); - } else { - logger.info("No version information available, assuming all versions used are released"); - unreleased = Collections.emptyList(); - unreleasedInfo = null; - } - } - if (unreleased.contains(Version.fromString(version))) { - Map projectNotation = new HashMap<>(); - projectNotation.put("path", unreleasedInfo.gradleProjectPath); - projectNotation.put("configuration", distribution.getLiveConfiguration()); - rootProject.getDependencies().add( - helperConfiguration.getName(), - project.getDependencies().project(projectNotation) - ); - } else { - rootProject.getDependencies().add( - helperConfiguration.getName(), - distribution.getGroup() + ":" + - distribution.getArtifactName() + ":" + - version + - (distribution.getClassifier().isEmpty() ? "" : ":" + distribution.getClassifier()) + "@" + - distribution.getFileExtension()); - - } - }))); - } - } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java new file mode 100644 index 0000000000000..61419835b189f --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.testclusters; + +/** + * An enumeration of the distributions that may be used in test clusters. + */ +public enum TestDistribution { + INTEG_TEST, + DEFAULT, + OSS +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java index 69169d13d1977..c32d8ad81a97c 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java @@ -188,9 +188,7 @@ private void assertDistroError(Project project, String name, String version, Typ private ElasticsearchDistribution createDistro(Project project, String name, String version, Type type, Platform platform, Flavor flavor, Boolean bundledJdk) { - @SuppressWarnings("unchecked") - NamedDomainObjectContainer distros = - (NamedDomainObjectContainer) project.getExtensions().getByName("elasticsearch_distributions"); + NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); return distros.create(name, distro -> { if (version != null) { distro.setVersion(version); diff --git a/buildSrc/src/testKit/testclusters/build.gradle b/buildSrc/src/testKit/testclusters/build.gradle index 56bf5c861f36d..4296965a73fb9 100644 --- a/buildSrc/src/testKit/testclusters/build.gradle +++ b/buildSrc/src/testKit/testclusters/build.gradle @@ -26,7 +26,7 @@ allprojects { all -> all.testClusters { myTestCluster { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' version = System.getProperty("test.version_under_test") javaHome = file(System.getProperty('java.home')) plugin file("${project(":dummyPlugin").buildDir}/distributions/dummy-${System.getProperty("test.version_under_test")}.zip") @@ -54,23 +54,23 @@ allprojects { all -> testClusters { multiNode { version = System.getProperty("test.version_under_test") - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' javaHome = file(System.getProperty('java.home')) numberOfNodes = 3 } releasedVersionDefault { version = "7.0.0" - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' javaHome = file(System.getProperty('java.home')) } releasedVersionOSS { version = "7.0.0" - distribution = 'OSS' + testDistribution = 'OSS' javaHome = file(System.getProperty('java.home')) } releasedVersionIntegTest { version = "7.0.0" - distribution = 'INTEG_TEST' + testDistribution = 'INTEG_TEST' javaHome = file(System.getProperty('java.home')) } } @@ -137,6 +137,6 @@ task illegalConfigAlter { useCluster testClusters.myTestCluster doFirst { println "Going to alter configuration after use" - testClusters.myTestCluster.distribution = 'OSS' + testClusters.myTestCluster.testDistribution = 'OSS' } } diff --git a/buildSrc/src/testKit/thirdPartyAudit/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/build.gradle index 725be970fd952..e3258ce310be4 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/build.gradle @@ -2,8 +2,8 @@ import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask plugins { id 'java' - //just to get build-tools - id 'elasticsearch.testclusters' + // bring in build-tools onto the classpath + id 'elasticsearch.global-build-info' apply false } repositories { diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index af60d7b8b60fc..3ad86b645e198 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -102,7 +102,7 @@ integTest.runner { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' systemProperty 'es.scripting.update.ctx_in_params', 'false' setting 'reindex.remote.whitelist', '[ "[::1]:*", "127.0.0.1:*" ]' setting 'xpack.license.self_generated.type', 'trial' diff --git a/docs/build.gradle b/docs/build.gradle index d13f4ca3b2edb..00d6a7a7616e6 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1,4 +1,4 @@ -import static org.elasticsearch.gradle.Distribution.DEFAULT +import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT /* * Licensed to Elasticsearch under one or more contributor @@ -38,7 +38,7 @@ buildRestTests.expectedUnconvertedCandidates = [ ] testClusters.integTest { - if (singleNode().distribution == DEFAULT) { + if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' } diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index 738a3be86afab..6b1a78e352f58 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -33,7 +33,7 @@ dependencies { } testClusters.integTest { - distribution = 'oss' + testDistribution = 'oss' } test.enabled = false diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle index fba9580525bcc..749cd49d6757d 100644 --- a/plugins/examples/security-authorization-engine/build.gradle +++ b/plugins/examples/security-authorization-engine/build.gradle @@ -32,7 +32,7 @@ testClusters.integTest { // This is important, so that all the modules are available too. // There are index templates that use token filters that are in analysis-module and // processors are being used that are in ingest-common module. - distribution = 'default' + testDistribution = 'DEFAULT' user role: 'custom_superuser' } diff --git a/qa/build.gradle b/qa/build.gradle index 9266a09b25735..2d3ca8282d4ca 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -11,11 +11,8 @@ subprojects { Project subproj -> } } plugins.withType(TestClustersPlugin).whenPluginAdded { - afterEvaluate { - // We need to delay this so it's not overwritten in RestIntegTestTask - testClusters.all { - distribution = System.getProperty('tests.distribution', 'oss').toUpperCase() - } + testClusters.all { + testDistribution = System.getProperty('tests.distribution', 'oss').toUpperCase() } } } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 0d83bcf2be54c..8cda8b672d6c5 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -124,7 +124,7 @@ integTest.runner { } testClusters.integTest { - distribution = 'DEFAULT' // this is important since we use the reindex module in ML + testDistribution = 'DEFAULT' // this is important since we use the reindex module in ML setting 'xpack.ml.enabled', 'true' setting 'xpack.security.enabled', 'true' // Integration tests are supposed to enable/disable exporters before/after each test diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle index b69458143d8fe..cc4b48ed6a694 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle +++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle @@ -16,7 +16,7 @@ task "leader-cluster"(type: RestIntegTestTask) { } } testClusters."leader-cluster" { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' } @@ -48,7 +48,7 @@ task "follow-cluster"(type: RestIntegTestTask) { } testClusters."follow-cluster" { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.leader_cluster.seeds', { "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\"" } diff --git a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle index 9cab8573e6b1c..2ac624195cab3 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle @@ -17,7 +17,7 @@ task "leader-cluster"(type: RestIntegTestTask) { } testClusters."leader-cluster" { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' } @@ -32,7 +32,7 @@ task "middle-cluster"(type: RestIntegTestTask) { } } testClusters."middle-cluster" { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.leader_cluster.seeds', { "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\"" } @@ -52,7 +52,7 @@ task 'follow-cluster'(type: RestIntegTestTask) { } testClusters."follow-cluster" { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.leader_cluster.seeds', diff --git a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle index fc55e4eb56bb9..dc3d5e8a9359a 100644 --- a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle +++ b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle @@ -16,7 +16,7 @@ task 'leader-cluster'(type: RestIntegTestTask) { } } testClusters.'leader-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' } task 'follow-cluster'(type: RestIntegTestTask) { @@ -29,7 +29,7 @@ task 'follow-cluster'(type: RestIntegTestTask) { } } testClusters.'follow-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.leader_cluster.seeds', { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().join(",")}\"" } diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle index bc2ca169d0ecf..8db29935a8d46 100644 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ b/x-pack/plugin/ccr/qa/rest/build.gradle @@ -13,7 +13,7 @@ task restTest(type: RestIntegTestTask) { } testClusters.restTest { - distribution = 'default' + testDistribution = 'DEFAULT' // Disable assertions in FollowingEngineAssertions, otherwise an AssertionError is thrown before // indexing a document directly in a follower index. In a rest test we like to test the exception // that is thrown in production when indexing a document directly in a follower index. diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle index d29cb136e57b2..564b5f87e0b4b 100644 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ b/x-pack/plugin/ccr/qa/restart/build.gradle @@ -14,7 +14,7 @@ task 'leader-cluster'(type: RestIntegTestTask) { } } testClusters.'leader-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' } @@ -28,7 +28,7 @@ task 'follow-cluster'(type: RestIntegTestTask) { } } testClusters.'follow-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.leader_cluster.seeds', diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle index 8b2f25a9444dd..03859867553f1 100644 --- a/x-pack/plugin/ccr/qa/security/build.gradle +++ b/x-pack/plugin/ccr/qa/security/build.gradle @@ -17,7 +17,7 @@ task 'leader-cluster'(type: RestIntegTestTask) { } testClusters.'leader-cluster' { - distribution = 'Default' + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.monitoring.enabled', 'false' @@ -36,7 +36,7 @@ task 'follow-cluster'(type: RestIntegTestTask) { } testClusters.'follow-cluster' { - distribution = 'Default' + testDistribution = 'DEFAULT' setting 'cluster.remote.leader_cluster.seeds', { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().join(",")}\"" } diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle b/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle index ab031532477c9..fe394adfc9791 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle @@ -27,7 +27,7 @@ processTestResources.dependsOn(copyKeyCerts) integTest.dependsOn copyKeyCerts testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.monitoring.enabled', 'false' diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle b/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle index 3947a18693018..3100c2002c5df 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle +++ b/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle @@ -10,7 +10,7 @@ dependencies { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' user username: "x_pack_rest_user", password: "x-pack-test-password" diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle index 1a979d83355c9..578f910146748 100644 --- a/x-pack/plugin/graph/qa/with-security/build.gradle +++ b/x-pack/plugin/graph/qa/with-security/build.gradle @@ -16,7 +16,7 @@ task copyGraphRestTests(type: Copy) { integTest.dependsOn copyGraphRestTests testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index 86b82208ca4c5..8ba0a758f7371 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -21,7 +21,7 @@ task 'leader-cluster'(type: RestIntegTestTask) { } testClusters.'leader-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'path.repo', repoDir.absolutePath setting 'xpack.ilm.enabled', 'true' setting 'xpack.ccr.enabled', 'true' @@ -48,7 +48,7 @@ task 'follow-cluster'(type: RestIntegTestTask) { } testClusters.'follow-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'path.repo', repoDir.absolutePath setting 'xpack.ilm.enabled', 'true' setting 'xpack.ccr.enabled', 'true' diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index 1a72ee5cf6f17..77464b031aa15 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -16,7 +16,7 @@ integTest.runner { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' numberOfNodes = 4 setting 'path.repo', repoDir.absolutePath diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index 11cc090b40da9..865f51c52ca85 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -20,7 +20,7 @@ task restTest(type: RestIntegTestTask) { } testClusters.restTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'true' setting 'xpack.ml.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle index 84685c3da7c88..125b35fb5dcb5 100644 --- a/x-pack/plugin/ilm/qa/with-security/build.gradle +++ b/x-pack/plugin/ilm/qa/with-security/build.gradle @@ -17,7 +17,7 @@ integTest { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index 9bbf34e544152..2d90950791cb1 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -8,7 +8,7 @@ dependencies { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' numberOfNodes = 3 setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle index 3e7f8b5ca7f50..f066e7e5bcfaf 100644 --- a/x-pack/plugin/ml/qa/disabled/build.gradle +++ b/x-pack/plugin/ml/qa/disabled/build.gradle @@ -8,7 +8,7 @@ dependencies { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index b5efa6736f3ad..1f1f48c2336fa 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -163,7 +163,7 @@ integTest.runner { testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' extraConfigFile 'roles.yml', file('roles.yml') user username: "x_pack_rest_user", password: "x-pack-test-password" user username: "ml_admin", password: "x-pack-test-password", role: "minimal,machine_learning_admin" diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index 455a5a350723c..c86cde5df0e80 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -37,7 +37,7 @@ integTest { testClusters.integTest { numberOfNodes = 3 - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'true' diff --git a/x-pack/plugin/ml/qa/single-node-tests/build.gradle b/x-pack/plugin/ml/qa/single-node-tests/build.gradle index 036b46cb0ca99..c5f14a71f39df 100644 --- a/x-pack/plugin/ml/qa/single-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/single-node-tests/build.gradle @@ -8,7 +8,7 @@ dependencies { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/security/qa/basic-enable-security/build.gradle b/x-pack/plugin/security/qa/basic-enable-security/build.gradle index 94bfb73c07679..461bc11a9b47a 100644 --- a/x-pack/plugin/security/qa/basic-enable-security/build.gradle +++ b/x-pack/plugin/security/qa/basic-enable-security/build.gradle @@ -16,7 +16,7 @@ integTest { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' numberOfNodes = 2 setting 'xpack.ilm.enabled', 'false' setting 'xpack.ml.enabled', 'false' diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle index 4dc948e477ed2..4fa19511797ef 100644 --- a/x-pack/plugin/security/qa/security-basic/build.gradle +++ b/x-pack/plugin/security/qa/security-basic/build.gradle @@ -9,7 +9,7 @@ dependencies { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' numberOfNodes = 2 setting 'xpack.ilm.enabled', 'false' diff --git a/x-pack/plugin/security/qa/tls-basic/build.gradle b/x-pack/plugin/security/qa/tls-basic/build.gradle index 2c93526168547..6326a69d4abd5 100644 --- a/x-pack/plugin/security/qa/tls-basic/build.gradle +++ b/x-pack/plugin/security/qa/tls-basic/build.gradle @@ -15,7 +15,7 @@ forbiddenPatterns { testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' numberOfNodes = 2 extraConfigFile 'http.key', file('src/test/resources/ssl/http.key') diff --git a/x-pack/plugin/sql/qa/build.gradle b/x-pack/plugin/sql/qa/build.gradle index 89f04562fffb2..f33fd4a430312 100644 --- a/x-pack/plugin/sql/qa/build.gradle +++ b/x-pack/plugin/sql/qa/build.gradle @@ -98,7 +98,7 @@ subprojects { apply plugin: 'elasticsearch.rest-test' testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' diff --git a/x-pack/plugin/sql/qa/security/build.gradle b/x-pack/plugin/sql/qa/security/build.gradle index 2cf410ed3d908..2774c4b85f4a5 100644 --- a/x-pack/plugin/sql/qa/security/build.gradle +++ b/x-pack/plugin/sql/qa/security/build.gradle @@ -30,7 +30,7 @@ subprojects { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' // Setup auditing so we can use it in some tests setting 'xpack.security.audit.enabled', 'true' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index c9c444ee9f115..19bb678abd497 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -21,7 +21,7 @@ integTest { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 4a4bd5485e152..81e5d746cc767 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -16,7 +16,7 @@ dependencies { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' // force localhost IPv4 otherwise it is a chicken and egg problem where we need the keytab for the hostname when starting the cluster // but do not know the exact address that is first in the http ports file setting 'http.host', '127.0.0.1' diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle index 4efcbdae8836b..3f3d03a7e685e 100644 --- a/x-pack/qa/multi-cluster-search-security/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/build.gradle @@ -15,7 +15,7 @@ task 'remote-cluster'(type: RestIntegTestTask) { } testClusters.'remote-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' numberOfNodes = 2 setting 'cluster.remote.connect', "false" setting 'xpack.ilm.enabled', 'false' @@ -37,7 +37,7 @@ task 'mixed-cluster'(type: RestIntegTestTask) { } testClusters.'mixed-cluster' { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' diff --git a/x-pack/qa/multi-node/build.gradle b/x-pack/qa/multi-node/build.gradle index 5bf248482aa7f..331ae0625271a 100644 --- a/x-pack/qa/multi-node/build.gradle +++ b/x-pack/qa/multi-node/build.gradle @@ -7,7 +7,7 @@ dependencies { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' numberOfNodes = 2 setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle index f80cb2035206d..9328447597e49 100644 --- a/x-pack/qa/oidc-op-tests/build.gradle +++ b/x-pack/qa/oidc-op-tests/build.gradle @@ -27,7 +27,7 @@ integTest.runner { } testClusters.integTest { - distribution = 'DEFAULT' + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'false' diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle index 292695b86e2d6..0a81e29088fd7 100644 --- a/x-pack/qa/reindex-tests-with-security/build.gradle +++ b/x-pack/qa/reindex-tests-with-security/build.gradle @@ -19,7 +19,7 @@ forbiddenPatterns { File caFile = project.file('src/test/resources/ssl/ca.p12') testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' // Whitelist reindexing from the local node so we can test it. extraConfigFile 'http.key', file('src/test/resources/ssl/http.key') extraConfigFile 'http.crt', file('src/test/resources/ssl/http.crt') diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 983e35e2d984b..b34e96dc0b401 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -42,7 +42,7 @@ task setupPorts { integTest.runner.dependsOn setupPorts testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'false' diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle index cca60e70ac132..6176ea2b3d7af 100644 --- a/x-pack/qa/security-example-spi-extension/build.gradle +++ b/x-pack/qa/security-example-spi-extension/build.gradle @@ -23,7 +23,7 @@ testClusters.integTest { // This is important, so that all the modules are available too. // There are index templates that use token filters that are in analysis-module and // processors are being used that are in ingest-common module. - distribution = 'default' + testDistribution = 'DEFAULT' setting 'xpack.security.authc.realms.custom.custom.order', '0' setting 'xpack.security.authc.realms.custom.custom.filtered_setting', 'should be filtered' diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle index c2880a2516d92..afa3c95ae90ed 100644 --- a/x-pack/qa/security-setup-password-tests/build.gradle +++ b/x-pack/qa/security-setup-password-tests/build.gradle @@ -14,7 +14,7 @@ integTest.runner { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle index 9bb5d5899db6a..d380eb308f62f 100644 --- a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle +++ b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle @@ -7,7 +7,7 @@ dependencies { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.monitoring.enabled', 'true' setting 'xpack.watcher.enabled', 'true' diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index ee31574c2dd64..ec4ee499a9c2b 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -42,7 +42,7 @@ integTest.runner.dependsOn(copyKeyCerts) def pluginsCount = 0 testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.monitoring.collection.interval', '1s' setting 'xpack.monitoring.exporters._http.type', 'http' setting 'xpack.monitoring.exporters._http.enabled', 'false' diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle index 86fb4ee0f07d6..ded92bea09e72 100644 --- a/x-pack/qa/smoke-test-plugins/build.gradle +++ b/x-pack/qa/smoke-test-plugins/build.gradle @@ -10,7 +10,7 @@ dependencies { int pluginsCount = 0 testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle index 3edf8d22cbe1d..748252044c36c 100644 --- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle +++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle @@ -7,7 +7,7 @@ dependencies { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle index 710173b110977..1e56abba3ccf2 100644 --- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle +++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle @@ -15,7 +15,7 @@ task copyWatcherRestTests(type: Copy) { integTest.runner.dependsOn copyWatcherRestTests testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false' diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle index d4ccba66f4966..7bc68fee9f499 100644 --- a/x-pack/qa/smoke-test-watcher/build.gradle +++ b/x-pack/qa/smoke-test-watcher/build.gradle @@ -7,7 +7,7 @@ dependencies { } testClusters.integTest { - distribution = "DEFAULT" + testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.enabled', 'false'