Merge remote-tracking branch 'es/master' into enrich

Author: martijnvg

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy

@@ -27,6 +27,7 @@ import org.gradle.api.internal.ConventionTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 
@@ -53,8 +54,9 @@ class DependenciesInfoTask extends ConventionTask {
     Configuration compileOnlyConfiguration
 
     /** Directory to read license files */
+    @Optional
     @InputDirectory
-    File licensesDir = new File(project.projectDir, 'licenses')
+    File licensesDir = new File(project.projectDir, 'licenses').exists() ? new File(project.projectDir, 'licenses') : null
 
     @OutputFile
     File outputFile = new File(project.buildDir, "reports/dependencies/dependencies.csv")
@@ -134,7 +136,7 @@ class DependenciesInfoTask extends ConventionTask {
     protected String getLicenseType(final String group, final String name) {
         File license
 
-        if (licensesDir.exists()) {
+        if (licensesDir != null) {
             licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file ->
                 String prefix = file.name.split('-LICENSE.*')[0]
                 if (group.contains(prefix) || name.contains(prefix)) {

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
 import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.elasticsearch.gradle.tool.ClasspathUtils;
 import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
@@ -56,7 +57,9 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
 
     private static final String CONTAINER_NAME = "elasticsearch_distributions";
     private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
+    private static final String FAKE_SNAPSHOT_IVY_GROUP = "elasticsearch-distribution-snapshot";
     private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
+    private static final String SNAPSHOT_REPO_NAME = "elasticsearch-snapshots";
 
     private BwcVersions bwcVersions;
     private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
@@ -72,9 +75,10 @@ public void apply(Project project) {
 
         setupDownloadServiceRepo(project);
 
-        ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
-        this.bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
-        // TODO: setup snapshot dependency instead of pointing to bwc distribution projects for external projects
+        if (ClasspathUtils.isElasticsearchProject(project)) {
+            ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
+            this.bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
+        }
 
         project.afterEvaluate(this::setupDistributions);
     }
@@ -148,13 +152,10 @@ private void setupRootDownload(Project rootProject, ElasticsearchDistribution di
         }
     }
 
-    private static void setupDownloadServiceRepo(Project project) {
-        if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
-            return;
-        }
+    private static void addIvyRepo(Project project, String name, String url, String group) {
         project.getRepositories().ivy(ivyRepo -> {
-            ivyRepo.setName(DOWNLOAD_REPO_NAME);
-            ivyRepo.setUrl("https://artifacts.elastic.co");
+            ivyRepo.setName(name);
+            ivyRepo.setUrl(url);
             ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
             // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
             ivyRepo.credentials(HttpHeaderCredentials.class, creds -> {
@@ -163,15 +164,25 @@ private static void setupDownloadServiceRepo(Project project) {
             });
             ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
             ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
-            ivyRepo.content(content -> content.includeGroup(FAKE_IVY_GROUP));
+            ivyRepo.content(content -> content.includeGroup(group));
         });
         project.getRepositories().all(repo -> {
-            if (repo.getName().equals(DOWNLOAD_REPO_NAME) == false) {
+            if (repo.getName().equals(name) == false) {
                 // all other repos should ignore the special group name
-                repo.content(content -> content.excludeGroup(FAKE_IVY_GROUP));
+                repo.content(content -> content.excludeGroup(group));
             }
         });
-        // TODO: need maven repo just for integ-test-zip, but only in external cases
+    }
+
+    private static void setupDownloadServiceRepo(Project project) {
+        if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
+            return;
+        }
+        addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://artifacts.elastic.co", FAKE_IVY_GROUP);
+        if (ClasspathUtils.isElasticsearchProject(project) == false) {
+            // external, so add snapshot repo as well
+            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots.elastic.co", FAKE_SNAPSHOT_IVY_GROUP);
+        }
     }
 
     /**
@@ -187,25 +198,30 @@ private static void setupDownloadServiceRepo(Project project) {
      */
     private Object dependencyNotation(Project project, ElasticsearchDistribution distribution) {
 
-        if (Version.fromString(VersionProperties.getElasticsearch()).equals(distribution.getVersion())) {
-            return projectDependency(project, distributionProjectPath(distribution), "default");
-            // TODO: snapshot dep when not in ES repo
-        }
-        BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(distribution.getVersion());
-        if (unreleasedInfo != null) {
-            assert distribution.getBundledJdk();
-            return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
+        if (ClasspathUtils.isElasticsearchProject(project)) {
+            // non-external project, so depend on local build
+
+            if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) {
+                return projectDependency(project, distributionProjectPath(distribution), "default");
+            }
+            BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion()));
+            if (unreleasedInfo != null) {
+                assert distribution.getBundledJdk();
+                return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
+            }
         }
 
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
             return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion();
         }
 
+
+        Version distroVersion = Version.fromString(distribution.getVersion());
         String extension = distribution.getType().toString();
         String classifier = ":x86_64";
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
-            if (distribution.getVersion().onOrAfter("7.0.0")) {
+            if (distroVersion.onOrAfter("7.0.0")) {
                 classifier = ":" + distribution.getPlatform() + "-x86_64";
             } else {
                 classifier = "";
@@ -214,10 +230,12 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis
             classifier = ":amd64";
         }
         String flavor = "";
-        if (distribution.getFlavor() == Flavor.OSS && distribution.getVersion().onOrAfter("6.3.0")) {
+        if (distribution.getFlavor() == Flavor.OSS && distroVersion.onOrAfter("6.3.0")) {
             flavor = "-oss";
         }
-        return FAKE_IVY_GROUP + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
+
+        String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
+        return group + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
     }
 
     private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -114,7 +114,7 @@ public String toString() {
     final Configuration configuration;
     private final Extracted extracted;
 
-    private final Property<Version> version;
+    private final Property<String> version;
     private final Property<Type> type;
     private final Property<Platform> platform;
     private final Property<Flavor> flavor;
@@ -124,8 +124,7 @@ public String toString() {
                               Configuration extractedConfiguration) {
         this.name = name;
         this.configuration = fileConfiguration;
-        this.version = objectFactory.property(Version.class);
-        this.version.convention(Version.fromString(VersionProperties.getElasticsearch()));
+        this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
         this.type = objectFactory.property(Type.class);
         this.type.convention(Type.ARCHIVE);
         this.platform = objectFactory.property(Platform.class);
@@ -138,12 +137,13 @@ public String getName() {
         return name;
     }
 
-    public Version getVersion() {
+    public String getVersion() {
         return version.get();
     }
 
     public void setVersion(String version) {
-        this.version.set(Version.fromString(version));
+        Version.fromString(version); // ensure the version parses, but don't store as Version since that removes -SNAPSHOT
+        this.version.set(version);
     }
 
     public Platform getPlatform() {

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -28,7 +28,10 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
@@ -261,7 +264,23 @@ public static String gitRevision(File rootDir) {
             }
             final String ref = readFirstLine(head);
             if (ref.startsWith("ref:")) {
-                revision = readFirstLine(gitDir.resolve(ref.substring("ref:".length()).trim()));
+                String refName = ref.substring("ref:".length()).trim();
+                Path refFile = gitDir.resolve(refName);
+                if (Files.exists(refFile)) {
+                    revision = readFirstLine(refFile);
+                } else if (Files.exists(dotGit.resolve("packed-refs"))) {
+                    // Check packed references for commit ID
+                    Pattern p = Pattern.compile("^([a-f0-9]{40}) " + refName + "$");
+                    try (Stream<String> lines = Files.lines(dotGit.resolve("packed-refs"))) {
+                        revision = lines.map(p::matcher)
+                            .filter(Matcher::matches)
+                            .map(m -> m.group(1))
+                            .findFirst()
+                            .orElseThrow(() -> new IOException("Packed reference not found for refName " + refName));
+                    }
+                } else {
+                    throw new GradleException("Can't find revision for refName " + refName);
+                }
             } else {
                 // we are in detached HEAD state
                 revision = ref;
@@ -274,8 +293,12 @@ public static String gitRevision(File rootDir) {
     }
 
     private static String readFirstLine(final Path path) throws IOException {
-        return Files.lines(path, StandardCharsets.UTF_8)
-            .findFirst()
-            .orElseThrow(() -> new IOException("file [" + path + "] is empty"));
+        String firstLine;
+        try (Stream<String> lines = Files.lines(path, StandardCharsets.UTF_8)) {
+            firstLine = lines
+                .findFirst()
+                .orElseThrow(() -> new IOException("file [" + path + "] is empty"));
+        }
+        return firstLine;
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -180,7 +180,7 @@ public String getName() {
 
     @Internal
     public Version getVersion() {
-        return distributions.get(currentDistro).getVersion();
+        return Version.fromString(distributions.get(currentDistro).getVersion());
     }
 
     @Internal
@@ -660,7 +660,8 @@ private Map<String, String> getESEnvironment() {
                 })
                 .collect(Collectors.joining(" "));
         }
-        defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa " +
+        String heapSize = System.getProperty("tests.heap.size", "512m");
+        defaultEnv.put("ES_JAVA_OPTS", "-Xms" + heapSize + " -Xmx" + heapSize + " -ea -esa " +
             systemPropertiesString + " " +
             jvmArgsString + " " +
             // Support  passing in additional JVM arguments

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -10,11 +10,14 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 public class RunTask extends DefaultTestClustersTask {
 
     private static final Logger logger = Logging.getLogger(RunTask.class);
+    public static final String CUSTOM_SETTINGS_PREFIX = "tests.es.";
 
     private Boolean debug = false;
 
@@ -36,12 +39,19 @@ public void beforeStart() {
         int debugPort = 8000;
         int httpPort = 9200;
         int transportPort = 9300;
+        Map<String, String> additionalSettings = System.getProperties().entrySet().stream()
+            .filter(entry -> entry.getKey().toString().startsWith(CUSTOM_SETTINGS_PREFIX))
+            .collect(Collectors.toMap(
+                entry -> entry.getKey().toString().substring(CUSTOM_SETTINGS_PREFIX.length()),
+                entry -> entry.getValue().toString()
+            ));
         for (ElasticsearchCluster cluster : getClusters()) {
             cluster.getFirstNode().setHttpPort(String.valueOf(httpPort));
             httpPort++;
             cluster.getFirstNode().setTransportPort(String.valueOf(transportPort));
             transportPort++;
             for (ElasticsearchNode node : cluster.getNodes()) {
+                additionalSettings.forEach(node::setting);
                 if (debug) {
                     logger.lifecycle(
                         "Running elasticsearch in debug mode, {} suspending until connected on debugPort {}",

buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java

@@ -38,6 +38,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 
 public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
+
     // TODO: check reuse of root task across projects MOVE TO UNIT TEST
     // TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project
 
@@ -48,32 +49,61 @@ public void testCurrent() throws Exception {
             "tests.local_distro.project", projectName);
     }
 
+    public void testCurrentExternal() throws Exception {
+        checkService(VersionProperties.getElasticsearch(), "archive", "linux", null, null,
+            "/downloads/elasticsearch/elasticsearch-" + VersionProperties.getElasticsearch() + "-linux-x86_64.tar.gz",
+            "tests.internal", "false");
+    }
+
     public void testBwc() throws Exception {
-        assertExtractedDistro("1.1.0", "archive", "linux", null, null,
+        assertExtractedDistro("8.1.0", "archive", "linux", null, null,
             "tests.local_distro.config", "linux-tar",
             "tests.local_distro.project", ":distribution:bwc:minor",
-            "tests.current_version", "2.0.0");
+            "tests.current_version", "8.0.0");
+    }
+
+    public void testBwcExternal() throws Exception {
+        checkService("8.1.0-SNAPSHOT", "archive", "linux", null, null,
+            "/downloads/elasticsearch/elasticsearch-8.1.0-SNAPSHOT-linux-x86_64.tar.gz",
+            "tests.internal", "false",
+            "tests.current_version", "9.0.0");
     }
 
     public void testReleased() throws Exception {
-        doTestReleased("7.0.0", "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
-        doTestReleased("6.5.0", "/downloads/elasticsearch/elasticsearch-6.5.0.zip");
+        checkService("7.0.0", "archive", "windows", null, null,
+            "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
+        checkService("6.5.0", "archive", "windows", null, null,
+            "/downloads/elasticsearch/elasticsearch-6.5.0.zip");
+    }
+
+    public void testReleasedExternal() throws Exception {
+        checkService("7.0.0", "archive", "windows", null, null,
+            "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip",
+            "tests.internal", "false");
+        checkService("6.5.0", "archive", "windows", null, null,
+            "/downloads/elasticsearch/elasticsearch-6.5.0.zip",
+            "tests.internal", "false");
     }
 
-    private void doTestReleased(String version, String urlPath) throws IOException {
+    private void checkService(String version, String type, String platform, String flavor, Boolean bundledJdk,
+                              String urlPath, String... sysProps) throws IOException {
+        String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz";
+        String sourceFile = "src/testKit/distribution-download/distribution/files/fake_elasticsearch." + suffix;
         WireMockServer wireMock = new WireMockServer(0);
         try {
             final byte[] filebytes;
-            try (InputStream stream =
-                     Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) {
+            try (InputStream stream = Files.newInputStream(Paths.get(sourceFile))) {
                 filebytes = stream.readAllBytes();
             }
             wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
             wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
             wireMock.start();
 
-            assertExtractedDistro(version, "archive", "windows", null, null,
-                "tests.download_service", wireMock.baseUrl());
+            List<String> allSysProps = new ArrayList<>();
+            allSysProps.addAll(Arrays.asList(sysProps));
+            allSysProps.add("tests.download_service");
+            allSysProps.add(wireMock.baseUrl());
+            assertExtractedDistro(version, type, platform, flavor, bundledJdk, allSysProps.toArray(new String[0]));
         } catch (Exception e) {
             // for debugging
             System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests());