From d313bed3211c9697382de213e1fd83bf8b1e98b8 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 19 Apr 2019 11:46:40 +0300 Subject: [PATCH 1/6] artifact config tests WIP --- .../testclusters/ElasticsearchCluster.java | 5 +- .../testclusters/TestClustersPlugin.java | 48 ++++++++++++------- .../testclusters/TestClustersPluginIT.java | 8 ++++ .../src/testKit/testclusters/build.gradle | 28 ++++++++++- 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 3bc750248ae37..f20ba835e39eb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -268,7 +268,10 @@ public boolean isProcessAlive() { } void eachVersionedDistribution(BiConsumer consumer) { - nodes.forEach(each -> consumer.accept(each.getVersion(), each.getDistribution())); + nodes.forEach(each -> { + LOGGER.lifecycle("{} -> {}", each.getVersion(), each.getDistribution()); + consumer.accept(each.getVersion(), each.getDistribution()); + }); } public ElasticsearchNode singleNode() { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index e9586f4c4bad5..38a73052daacf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -28,6 +28,9 @@ import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.repositories.MavenArtifactRepository; +import org.gradle.api.credentials.HttpHeaderCredentials; import org.gradle.api.execution.TaskActionListener; import org.gradle.api.execution.TaskExecutionListener; import org.gradle.api.file.FileCollection; @@ -290,6 +293,25 @@ private static void autoConfigureClusterDependencies( Project rootProject, NamedDomainObjectContainer container ) { + // Download integ test distribution from maven central + MavenArtifactRepository mavenCentral = project.getRepositories().mavenCentral(); + mavenCentral.content(spec -> { + spec.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*"); + }); + project.getRepositories().add(mavenCentral); + + // Other distributions from the download service + project.getRepositories().add( + project.getRepositories().ivy(spec -> { + spec.setUrl("https://artifacts.elastic.co/downloads"); + spec.patternLayout(p -> p.artifact("elasticsearch/[module]-[revision](-[classifier]).[ext]")); + HttpHeaderCredentials headerConfig = spec.getCredentials(HttpHeaderCredentials.class); + headerConfig.setName("X-Elastic-No-KPI"); + headerConfig.setValue("1"); + spec.content(c-> c.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*")); + }) + ); + // When the project evaluated we know of all tasks that use clusters. // Each of these have to depend on the artifacts being synced. // We need afterEvaluate here despite the fact that container is a domain object, we can't implement this with @@ -324,24 +346,14 @@ private static void autoConfigureClusterDependencies( project.getDependencies().project(projectNotation) ); } else { - if (distribution.equals(Distribution.INTEG_TEST)) { - rootProject.getDependencies().add( - HELPER_CONFIGURATION_NAME, "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + version - ); - } else { - // declare dependencies to be downloaded from the download service. - // The BuildPlugin sets up the right repo for this to work - // TODO: move the repo definition in this plugin when ClusterFormationTasks is removed - String dependency = String.format( - "%s:%s:%s:%s@%s", - distribution.getGroup(), - distribution.getArtifactName(), - version, - distribution.getClassifier(), - distribution.getFileExtension() - ); - rootProject.getDependencies().add(HELPER_CONFIGURATION_NAME, dependency); - } + rootProject.getDependencies().add( + HELPER_CONFIGURATION_NAME, + distribution.getGroup() + ":" + + distribution.getArtifactName() + ":" + + version + + (distribution.getClassifier().isEmpty() ? "" : ":" + distribution.getClassifier()) + "@" + + distribution.getFileExtension()); + } }))); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java index b06bd067edde8..a0734a5f45261 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java @@ -104,6 +104,14 @@ public void testMultiProject() { ); } + public void testReleased() { + BuildResult result = getTestClustersRunner(":testArtifacts").build(); + assertTaskSuccessful(result, ":testArtifacts"); + assertStartedAndStoppedOnce(result, "releasedVersionDefault-1"); + assertStartedAndStoppedOnce(result, "releasedVersionOSS-2"); + assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-3"); + } + public void testIncremental() { BuildResult result = getTestClustersRunner("clean", ":user1").build(); assertTaskSuccessful(result, ":user1"); diff --git a/buildSrc/src/testKit/testclusters/build.gradle b/buildSrc/src/testKit/testclusters/build.gradle index f82013bbc7356..a2d18322b1fcc 100644 --- a/buildSrc/src/testKit/testclusters/build.gradle +++ b/buildSrc/src/testKit/testclusters/build.gradle @@ -17,8 +17,6 @@ allprojects { all -> url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision } } - - jcenter() } if (project == rootProject || project.name == "alpha" || project.name == "bravo") { @@ -58,6 +56,21 @@ testClusters { javaHome = file(System.getProperty('java.home')) numberOfNodes = 3 } + releasedVersionDefault { + version = "7.0.0" + distribution = 'DEFAULT' + javaHome = file(System.getProperty('java.home')) + } + releasedVersionOSS { + version = "7.0.0" + distribution = 'OSS' + javaHome = file(System.getProperty('java.home')) + } + releasedVersionIntegTest { + version = "7.0.0" + distribution = 'INTEG_TEST' + javaHome = file(System.getProperty('java.home')) + } } task multiNode { @@ -67,6 +80,17 @@ task multiNode { } } +task testArtifacts { + useCluster testClusters.releasedVersionDefault + useCluster testClusters.releasedVersionOSS + useCluster testClusters.releasedVersionIntegTest + doFirst { + println "$path: Cluster running @ ${testClusters.releasedVersionDefault.httpSocketURI}" + println "$path: Cluster running @ ${testClusters.releasedVersionOSS.httpSocketURI}" + println "$path: Cluster running @ ${testClusters.releasedVersionIntegTest.httpSocketURI}" + } +} + task printLog { useCluster testClusters.myTestCluster doFirst { From 5c7f9d37707aa370621ffe17cce24cdf0c30a641 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 23 Apr 2019 14:53:06 +0300 Subject: [PATCH 2/6] wip version configuration --- .../org/elasticsearch/gradle/Distribution.java | 16 +++++++++------- .../testclusters/ElasticsearchCluster.java | 1 - .../gradle/testclusters/TestClustersPlugin.java | 12 +++++++++--- .../testclusters/TestClustersPluginIT.java | 6 +++--- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java index d8897053f05bc..9cb3cc52dd09e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java @@ -20,16 +20,14 @@ public enum Distribution { - INTEG_TEST("elasticsearch", "integ-test-zip"), - DEFAULT("elasticsearch", "elasticsearch"), - OSS("elasticsearch-oss", "elasticsearch-oss"); + INTEG_TEST("elasticsearch"), + DEFAULT("elasticsearch"), + OSS("elasticsearch-oss"); private final String artifactName; - private final String group; - Distribution(String name, String group) { + Distribution(String name) { this.artifactName = name; - this.group = group; } public String getArtifactName() { @@ -37,7 +35,11 @@ public String getArtifactName() { } public String getGroup() { - return "org.elasticsearch.distribution." + group; + if (this.equals(INTEG_TEST)) { + return "org.elasticsearch.distribution.integ-test-zip"; + } else { + return "org.elasticsearch.distribution." + name().toLowerCase(); + } } public String getFileExtension() { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index f20ba835e39eb..dc0500ee37ccf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -269,7 +269,6 @@ public boolean isProcessAlive() { void eachVersionedDistribution(BiConsumer consumer) { nodes.forEach(each -> { - LOGGER.lifecycle("{} -> {}", each.getVersion(), each.getDistribution()); consumer.accept(each.getVersion(), each.getDistribution()); }); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index 38a73052daacf..5bc4291bae09d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.gradle.testclusters; +import com.perforce.p4java.Log; import groovy.lang.Closure; import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.Distribution; @@ -117,9 +118,13 @@ public void execute(Task task) { } else { throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); } - spec.from(files).into(new File(project.getRootProject().getBuildDir(), "testclusters/extract") + "/" + - resolvedArtifact.getModuleVersion().getId().getGroup() - ); + logger.lifecycle("extracting: {} to {}", file, resolvedArtifact.getModuleVersion().getId().getGroup()); + spec.from(files) + .into( + new File( + project.getRootProject().getBuildDir(), "testclusters/extract") + "/" + + resolvedArtifact.getModuleVersion().getId().getGroup() + ); })); } }); @@ -346,6 +351,7 @@ private static void autoConfigureClusterDependencies( project.getDependencies().project(projectNotation) ); } else { + logger.lifecycle("dependency {} @ {}", distribution, version); rootProject.getDependencies().add( HELPER_CONFIGURATION_NAME, distribution.getGroup() + ":" + diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java index a0734a5f45261..0eb66b5346a35 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java @@ -105,11 +105,11 @@ public void testMultiProject() { } public void testReleased() { - BuildResult result = getTestClustersRunner(":testArtifacts").build(); + BuildResult result = getTestClustersRunner("dependencies").build(); assertTaskSuccessful(result, ":testArtifacts"); assertStartedAndStoppedOnce(result, "releasedVersionDefault-1"); - assertStartedAndStoppedOnce(result, "releasedVersionOSS-2"); - assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-3"); + assertStartedAndStoppedOnce(result, "releasedVersionOSS-1"); + assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-1"); } public void testIncremental() { From ed5708ebb9e532e257708a8d616083af676ca0a0 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 23 Apr 2019 19:32:52 +0300 Subject: [PATCH 3/6] Tests for older versions --- .../elasticsearch/gradle/testclusters/TestClustersPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index 5bc4291bae09d..ba53428c44669 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -118,7 +118,7 @@ public void execute(Task task) { } else { throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); } - logger.lifecycle("extracting: {} to {}", file, resolvedArtifact.getModuleVersion().getId().getGroup()); + spec.from(files) .into( new File( @@ -351,7 +351,6 @@ private static void autoConfigureClusterDependencies( project.getDependencies().project(projectNotation) ); } else { - logger.lifecycle("dependency {} @ {}", distribution, version); rootProject.getDependencies().add( HELPER_CONFIGURATION_NAME, distribution.getGroup() + ":" + From e1b26fff8c3b16dabb9e83b46f12c576ea5a24d0 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 24 Apr 2019 21:52:19 +0300 Subject: [PATCH 4/6] use separate configurations per version --- .../testclusters/ElasticsearchNode.java | 2 +- .../testclusters/TestClustersPlugin.java | 113 ++++++++++-------- .../gradle/tool/Boilerplate.java | 19 ++- .../testclusters/TestClustersPluginIT.java | 4 +- .../src/testKit/testclusters/build.gradle | 2 +- 5 files changed, 85 insertions(+), 55 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 5a3e0d599f173..7e44cd70e5120 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -260,7 +260,7 @@ public synchronized void start() { Path distroArtifact = artifactsExtractDir .resolve(distribution.getGroup()) - .resolve(distribution.getArtifactName() + "-" + getVersion()); + .resolve("elasticsearch-" + getVersion()); if (Files.exists(distroArtifact) == false) { throw new TestClustersException("Can not start " + this + ", missing: " + distroArtifact); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index ba53428c44669..91f72bb2e3f42 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -18,18 +18,16 @@ */ package org.elasticsearch.gradle.testclusters; -import com.perforce.p4java.Log; import groovy.lang.Closure; import org.elasticsearch.gradle.BwcVersions; -import org.elasticsearch.gradle.Distribution; import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.tool.Boilerplate; import org.gradle.api.Action; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.repositories.IvyArtifactRepository; import org.gradle.api.artifacts.repositories.MavenArtifactRepository; import org.gradle.api.credentials.HttpHeaderCredentials; import org.gradle.api.execution.TaskActionListener; @@ -73,6 +71,10 @@ public class TestClustersPlugin implements Plugin { private final Thread shutdownHook = new Thread(this::shutDownAllClusters); private ExecutorService executorService = Executors.newSingleThreadExecutor(); + public static String getHelperConfigurationName(String version) { + return HELPER_CONFIGURATION_NAME + "-" + version; + } + @Override public void apply(Project project) { Project rootProject = project.getRootProject(); @@ -86,51 +88,6 @@ public void apply(Project project) { // create DSL for tasks to mark clusters these use createUseClusterTaskExtension(project, container); - if (rootProject.getConfigurations().findByName(HELPER_CONFIGURATION_NAME) == null) { - // We use a single configuration on the root project to resolve all testcluster dependencies ( like distros ) - // at once, only once without the need to repeat it for each project. This pays off assuming that most - // projects use the same dependencies. - Configuration helperConfiguration = project.getRootProject().getConfigurations().create(HELPER_CONFIGURATION_NAME); - helperConfiguration.setDescription( - "Internal helper configuration used by cluster configuration to download " + - "ES distributions and plugins." - ); - - // We have a single task to sync the helper configuration to "artifacts dir" - // the clusters will look for artifacts there based on the naming conventions. - // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in - // the build. - rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, sync -> { - sync.getInputs().files((Callable) helperConfiguration::getAsFileTree); - sync.getOutputs().dir(new File(project.getRootProject().getBuildDir(), "testclusters/extract")); - // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) - sync.doLast(new Action() { - @Override - public void execute(Task task) { - project.sync(spec -> - helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> { - final FileTree files; - File file = resolvedArtifact.getFile(); - if (file.getName().endsWith(".zip")) { - files = project.zipTree(file); - } else if (file.getName().endsWith("tar.gz")) { - files = project.tarTree(file); - } else { - throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); - } - - spec.from(files) - .into( - new File( - project.getRootProject().getBuildDir(), "testclusters/extract") + "/" + - resolvedArtifact.getModuleVersion().getId().getGroup() - ); - })); - } - }); - }); - } - // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters // that are defined in the build script and the ones that will actually be used in this invocation of gradle // we use this information to determine when the last task that required the cluster executed so that we can @@ -151,6 +108,10 @@ public void execute(Task task) { autoConfigureClusterDependencies(project, rootProject, container); } + private static File getExtractDir(Project project) { + return new File(project.getRootProject().getBuildDir(), "testclusters/extract/"); + } + private NamedDomainObjectContainer createTestClustersContainerExtension(Project project) { // Create an extensions that allows describing clusters NamedDomainObjectContainer container = project.container( @@ -317,12 +278,41 @@ private static void autoConfigureClusterDependencies( }) ); + // We have a single task to sync the helper configuration to "artifacts dir" + // the clusters will look for artifacts there based on the naming conventions. + // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in + // the build. + Task sync = Boilerplate.findOrCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> { + onCreate.getOutputs().dir(getExtractDir(rootProject)); + // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) + onCreate.doFirst(new Action() { + @Override + public void execute(Task task) { + // Clean up the extract dir first to make sure we have no stale files from older + // previous builds of the same distribution + project.delete(getExtractDir(rootProject)); + } + }); + }); + // When the project evaluated we know of all tasks that use clusters. // Each of these have to depend on the artifacts being synced. // We need afterEvaluate here despite the fact that container is a domain object, we can't implement this with // all because fields can change after the fact. project.afterEvaluate(ip -> container.forEach(esCluster -> esCluster.eachVersionedDistribution((version, distribution) -> { + Configuration helperConfiguration = Boilerplate.findOrCreate( + rootProject.getConfigurations(), + getHelperConfigurationName(version), + onCreate -> + // We use a single configuration on the root project to resolve all testcluster dependencies ( like distros ) + // at once, only once without the need to repeat it for each project. This pays off assuming that most + // projects use the same dependencies. + onCreate.setDescription( + "Internal helper configuration used by cluster configuration to download " + + "ES distributions and plugins for " + version + ) + ); BwcVersions.UnreleasedVersionInfo unreleasedInfo; final List unreleased; { @@ -347,12 +337,12 @@ private static void autoConfigureClusterDependencies( projectNotation.put("path", unreleasedInfo.gradleProjectPath); projectNotation.put("configuration", distribution.getLiveConfiguration()); rootProject.getDependencies().add( - HELPER_CONFIGURATION_NAME, + helperConfiguration.getName(), project.getDependencies().project(projectNotation) ); } else { rootProject.getDependencies().add( - HELPER_CONFIGURATION_NAME, + helperConfiguration.getName(), distribution.getGroup() + ":" + distribution.getArtifactName() + ":" + version + @@ -360,6 +350,29 @@ private static void autoConfigureClusterDependencies( distribution.getFileExtension()); } + + sync.getInputs().files((Callable) helperConfiguration::getAsFileTree); + // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) + sync.doLast(new Action() { + @Override + public void execute(Task task) { + project.copy(spec -> + helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> { + final FileTree files; + File file = resolvedArtifact.getFile(); + if (file.getName().endsWith(".zip")) { + files = project.zipTree(file); + } else if (file.getName().endsWith("tar.gz")) { + files = project.tarTree(file); + } else { + throw new IllegalArgumentException("Can't extract " + file + " unknown file extension"); + } + + spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup())); + spec.into(getExtractDir(project)); + })); + } + }); }))); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java index 67fc7473a4cfd..ed6bf8dbdddbe 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java @@ -18,14 +18,31 @@ */ package org.elasticsearch.gradle.tool; -import org.gradle.api.Project; +import org.gradle.api.*; import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.tasks.SourceSetContainer; +import java.util.Optional; + public abstract class Boilerplate { public static SourceSetContainer getJavaSourceSets(Project project) { return project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets(); } + public static T findOrCreate(NamedDomainObjectContainer collection, String name) { + return Optional.ofNullable(collection.findByName(name)) + .orElse(collection.create(name)); + + } + public static T findOrCreate(NamedDomainObjectContainer collection, String name, Action action) { + return Optional.ofNullable(collection.findByName(name)) + .orElseGet(() -> { + T result = collection.create(name); + action.execute(result); + return result; + }); + + } + } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java index 0eb66b5346a35..22adb2d9587e2 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java @@ -105,8 +105,8 @@ public void testMultiProject() { } public void testReleased() { - BuildResult result = getTestClustersRunner("dependencies").build(); - assertTaskSuccessful(result, ":testArtifacts"); + BuildResult result = getTestClustersRunner("testReleased").build(); + assertTaskSuccessful(result, ":testReleased"); assertStartedAndStoppedOnce(result, "releasedVersionDefault-1"); assertStartedAndStoppedOnce(result, "releasedVersionOSS-1"); assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-1"); diff --git a/buildSrc/src/testKit/testclusters/build.gradle b/buildSrc/src/testKit/testclusters/build.gradle index a2d18322b1fcc..ffdc38491ccf9 100644 --- a/buildSrc/src/testKit/testclusters/build.gradle +++ b/buildSrc/src/testKit/testclusters/build.gradle @@ -80,7 +80,7 @@ task multiNode { } } -task testArtifacts { +task testReleased { useCluster testClusters.releasedVersionDefault useCluster testClusters.releasedVersionOSS useCluster testClusters.releasedVersionIntegTest From 808307aea39522ccb6702881e3beca7437360499 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 24 Apr 2019 21:56:26 +0300 Subject: [PATCH 5/6] checkstyle --- .../main/java/org/elasticsearch/gradle/tool/Boilerplate.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java index ed6bf8dbdddbe..098864c17ca44 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java @@ -18,7 +18,9 @@ */ package org.elasticsearch.gradle.tool; -import org.gradle.api.*; +import org.gradle.api.Action; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Project; import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.tasks.SourceSetContainer; From 2639d0e1117f9af53e01114d86815b4fd5a67e2a Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 2 May 2019 16:28:11 +0300 Subject: [PATCH 6/6] PR review --- .../gradle/testclusters/TestClustersPlugin.java | 13 +++++-------- .../org/elasticsearch/gradle/tool/Boilerplate.java | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index 91f72bb2e3f42..c1ed6b770f04c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -32,7 +32,6 @@ import org.gradle.api.credentials.HttpHeaderCredentials; import org.gradle.api.execution.TaskActionListener; import org.gradle.api.execution.TaskExecutionListener; -import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileTree; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; @@ -48,7 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -58,7 +56,7 @@ public class TestClustersPlugin implements Plugin { private static final String LIST_TASK_NAME = "listTestClusters"; private static final String NODE_EXTENSION_NAME = "testClusters"; - private static final String HELPER_CONFIGURATION_NAME = "testclusters"; + private static final String HELPER_CONFIGURATION_PREFIX = "testclusters"; private static final String SYNC_ARTIFACTS_TASK_NAME = "syncTestClustersArtifacts"; private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1; private static final TimeUnit EXECUTOR_SHUTDOWN_TIMEOUT_UNIT = TimeUnit.MINUTES; @@ -72,7 +70,7 @@ public class TestClustersPlugin implements Plugin { private ExecutorService executorService = Executors.newSingleThreadExecutor(); public static String getHelperConfigurationName(String version) { - return HELPER_CONFIGURATION_NAME + "-" + version; + return HELPER_CONFIGURATION_PREFIX + "-" + version; } @Override @@ -264,7 +262,6 @@ private static void autoConfigureClusterDependencies( mavenCentral.content(spec -> { spec.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*"); }); - project.getRepositories().add(mavenCentral); // Other distributions from the download service project.getRepositories().add( @@ -282,7 +279,7 @@ private static void autoConfigureClusterDependencies( // the clusters will look for artifacts there based on the naming conventions. // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in // the build. - Task sync = Boilerplate.findOrCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> { + Task sync = Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> { onCreate.getOutputs().dir(getExtractDir(rootProject)); // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) onCreate.doFirst(new Action() { @@ -301,7 +298,7 @@ public void execute(Task task) { // all because fields can change after the fact. project.afterEvaluate(ip -> container.forEach(esCluster -> esCluster.eachVersionedDistribution((version, distribution) -> { - Configuration helperConfiguration = Boilerplate.findOrCreate( + Configuration helperConfiguration = Boilerplate.maybeCreate( rootProject.getConfigurations(), getHelperConfigurationName(version), onCreate -> @@ -351,7 +348,7 @@ public void execute(Task task) { } - sync.getInputs().files((Callable) helperConfiguration::getAsFileTree); + sync.getInputs().files(helperConfiguration); // NOTE: Gradle doesn't allow a lambda here ( fails at runtime ) sync.doLast(new Action() { @Override diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java index 098864c17ca44..29b0b5def20e5 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java @@ -32,12 +32,12 @@ public static SourceSetContainer getJavaSourceSets(Project project) { return project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets(); } - public static T findOrCreate(NamedDomainObjectContainer collection, String name) { + public static T maybeCreate(NamedDomainObjectContainer collection, String name) { return Optional.ofNullable(collection.findByName(name)) .orElse(collection.create(name)); } - public static T findOrCreate(NamedDomainObjectContainer collection, String name, Action action) { + public static T maybeCreate(NamedDomainObjectContainer collection, String name, Action action) { return Optional.ofNullable(collection.findByName(name)) .orElseGet(() -> { T result = collection.create(name);