Move common repository configuration to java plugin (#57057)

This commit moves the common maven repository configuration to the ES
java plugin. All java projects need this common set of repos. Note that
the Elastic download and maven repos are removed, as they are not
necessary anymore since distribution download was split into the
DistributionDownloadPlugin.

Author: rjernst

build.gradle

@@ -54,8 +54,6 @@ allprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
-BuildPlugin.configureRepositories(project)
-
 String licenseCommit
 if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
   licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.gradle
 
-
 import groovy.transform.CompileStatic
 import org.apache.commons.io.IOUtils
 import org.elasticsearch.gradle.info.BuildParams
@@ -51,7 +50,6 @@ import org.gradle.api.execution.TaskActionListener
 import org.gradle.api.plugins.ExtraPropertiesExtension
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.tasks.testing.Test
-import org.gradle.authentication.http.HttpHeaderAuthentication
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
@@ -93,7 +91,6 @@ class BuildPlugin implements Plugin<Project> {
 
         project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask)
 
-        configureRepositories(project)
         project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
         PrecommitTasks.create(project, true)
         configureFips140(project)
@@ -159,122 +156,6 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    /**
-     * Makes dependencies non-transitive.
-     *
-     * Gradle allows setting all dependencies as non-transitive very easily.
-     * Sadly this mechanism does not translate into maven pom generation. In order
-     * to effectively make the pom act as if it has no transitive dependencies,
-     * we must exclude each transitive dependency of each direct dependency.
-     *
-     * Determining the transitive deps of a dependency which has been resolved as
-     * non-transitive is difficult because the process of resolving removes the
-     * transitive deps. To sidestep this issue, we create a configuration per
-     * direct dependency version. This specially named and unique configuration
-     * will contain all of the transitive dependencies of this particular
-     * dependency. We can then use this configuration during pom generation
-     * to iterate the transitive dependencies and add excludes.
-     */
-    static void configureConfigurations(Project project) {
-        // we want to test compileOnly deps!
-        project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME))
-
-        // we are not shipping these jars, we act like dumb consumers of these things
-        if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
-            return
-        }
-        // fail on any conflicting dependency versions
-        project.configurations.all({ Configuration configuration ->
-            if (configuration.name.endsWith('Fixture')) {
-                // just a self contained test-fixture configuration, likely transitive and hellacious
-                return
-            }
-            configuration.resolutionStrategy {
-                failOnVersionConflict()
-            }
-        })
-
-        // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
-        Closure disableTransitiveDeps = { Dependency dep ->
-            if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)
-                    && dep.group.startsWith('org.elasticsearch') == false) {
-                dep.transitive = false
-            }
-        }
-
-        project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
-        project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
-        project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
-        project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
-    }
-
-    /** Adds repositories used by ES dependencies */
-    static void configureRepositories(Project project) {
-        project.getRepositories().all { repository ->
-            if (repository instanceof MavenArtifactRepository) {
-                final MavenArtifactRepository maven = (MavenArtifactRepository) repository
-                assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
-                repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
-            } else if (repository instanceof IvyArtifactRepository) {
-                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
-                assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
-            }
-        }
-        RepositoryHandler repos = project.repositories
-        if (System.getProperty('repos.mavenLocal') != null) {
-            // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
-            // useful for development ie. bwc tests where we install stuff in the local repository
-            // such that we don't have to pass hardcoded files to gradle
-            repos.mavenLocal()
-        }
-        repos.jcenter()
-        repos.ivy { IvyArtifactRepository repo ->
-            repo.name = 'elasticsearch'
-            repo.url = 'https://artifacts.elastic.co/downloads'
-            repo.patternLayout { IvyPatternRepositoryLayout layout ->
-                layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]'
-            }
-            // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
-            repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds ->
-                creds.name = 'X-Elastic-No-KPI'
-                creds.value = '1'
-            } as Action<HttpHeaderCredentials>)
-            repo.authentication.create('header', HttpHeaderAuthentication)
-        }
-        repos.maven { MavenArtifactRepository repo ->
-            repo.name = 'elastic'
-            repo.url = 'https://artifacts.elastic.co/maven'
-        }
-        String luceneVersion = VersionProperties.lucene
-        if (luceneVersion.contains('-snapshot')) {
-            // extract the revision number from the version with a regex matcher
-            List<String> matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List<String>
-            String revision = matches.get(1)
-            MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo ->
-                repo.name = 'lucene-snapshots'
-                repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}"
-            }
-            repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo ->
-                exclusiveRepo.filter {
-                    it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}")
-                }
-                exclusiveRepo.forRepositories(luceneRepo)
-            }
-        }
-    }
-
-    static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
-        if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
-            final String message = String.format(
-                    Locale.ROOT,
-                    "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
-                    repositoryName,
-                    projectPath,
-                    uri.toURL())
-            throw new GradleException(message)
-        }
-    }
-
     private static class TestFailureReportingPlugin implements Plugin<Project> {
         @Override
         void apply(Project project) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -62,7 +62,7 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         project.pluginManager.apply(TestClustersPlugin)
 
         project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
-        BuildPlugin.configureRepositories(project)
+        ElasticsearchJavaPlugin.configureRepositories(project)
         ElasticsearchJavaPlugin.configureTestTasks(project)
         ElasticsearchJavaPlugin.configureInputNormalization(project)
         BuildPlugin.configureFips140(project)

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java

@@ -36,6 +36,9 @@
 import org.gradle.api.artifacts.ModuleDependency;
 import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.artifacts.ResolutionStrategy;
+import org.gradle.api.artifacts.dsl.RepositoryHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.BasePlugin;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
@@ -56,12 +59,18 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
 import static org.elasticsearch.gradle.util.Util.toStringable;
@@ -77,6 +86,7 @@ public void apply(Project project) {
 
         project.getPluginManager().apply(JavaPlugin.class);
         configureConfigurations(project);
+        configureRepositories(project);
         configureCompile(project);
         configureInputNormalization(project);
         configureTestTasks(project);
@@ -137,6 +147,75 @@ public static void configureConfigurations(Project project) {
         disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME);
     }
 
+    private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
+
+    /** Adds repositories used by ES dependencies */
+    public static void configureRepositories(Project project) {
+        // ensure all repositories use secure urls
+        // TODO: remove this with gradle 7.0, which no longer allows insecure urls
+        project.getRepositories().all(repository -> {
+            if (repository instanceof MavenArtifactRepository) {
+                final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
+                assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
+                for (URI uri : maven.getArtifactUrls()) {
+                    assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
+                }
+            } else if (repository instanceof IvyArtifactRepository) {
+                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
+                assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
+            }
+        });
+        RepositoryHandler repos = project.getRepositories();
+        if (System.getProperty("repos.mavenLocal") != null) {
+            // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
+            // useful for development ie. bwc tests where we install stuff in the local repository
+            // such that we don't have to pass hardcoded files to gradle
+            repos.mavenLocal();
+        }
+        repos.jcenter();
+
+        String luceneVersion = VersionProperties.getLucene();
+        if (luceneVersion.contains("-snapshot")) {
+            // extract the revision number from the version with a regex matcher
+            Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
+            if (matcher.find() == false) {
+                throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
+            }
+            String revision = matcher.group(1);
+            MavenArtifactRepository luceneRepo = repos.maven(repo -> {
+                repo.setName("lucene-snapshots");
+                repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
+            });
+            repos.exclusiveContent(exclusiveRepo -> {
+                exclusiveRepo.filter(
+                    descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
+                );
+                exclusiveRepo.forRepositories(luceneRepo);
+            });
+        }
+    }
+
+    private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
+
+    private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
+        if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
+            String url;
+            try {
+                url = uri.toURL().toString();
+            } catch (MalformedURLException e) {
+                throw new IllegalStateException(e);
+            }
+            final String message = String.format(
+                Locale.ROOT,
+                "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
+                repositoryName,
+                projectPath,
+                url
+            );
+            throw new GradleException(message);
+        }
+    }
+
     /** Adds compiler settings to the project */
     public static void configureCompile(Project project) {
         project.getExtensions().getExtraProperties().set("compactProfile", "full");

buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java

@@ -18,6 +18,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.hamcrest.CoreMatchers.containsString;
+
 public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
 
     @Rule
@@ -71,7 +73,7 @@ protected void assertOutputContains(String output, Set<String> lines) {
     }
 
     protected void assertOutputContains(String output, String line) {
-        assertTrue("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output.contains(line));
+        assertThat("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output, containsString(line));
     }
 
     protected void assertOutputDoesNotContain(String output, String line) {