Merge branch 'master' into searchable-snapshots

Author: DaveCTurner

.ci/bwcVersions

@@ -26,4 +26,5 @@ BWC_VERSION:
   - "7.9.2"
   - "7.9.3"
   - "7.10.0"
+  - "7.11.0"
   - "8.0.0"

CONTRIBUTING.md

@@ -137,7 +137,7 @@ specifically these lines tell you that Elasticsearch is ready:
     [2020-05-29T14:50:35,167][INFO ][o.e.h.AbstractHttpServerTransport] [runTask-0] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
     [2020-05-29T14:50:35,169][INFO ][o.e.n.Node               ] [runTask-0] started
 
-But to be honest its typically easier to wait until the console stopps scrolling
+But to be honest its typically easier to wait until the console stops scrolling
 and then run `curl` in another window like this:
 
     curl -u elastic:password localhost:9200
@@ -166,7 +166,7 @@ You can import the Elasticsearch project into IntelliJ IDEA via:
 Elasticsearch builds using Gradle and Java 14. When importing into Eclipse you
 will either need to use an appropriate JDK to run Eclipse itself (e.g. by
 specifying the VM in [eclipse.ini](https://wiki.eclipse.org/Eclipse.ini) or by
-defining the JDK Gradle uses by setting **Prefercences** > **Gradle** >
+defining the JDK Gradle uses by setting **Preferences** > **Gradle** >
 **Advanced Options** > **Java home** to an appropriate version.
 
 IMPORTANT: If you have previously imported the project by running `./gradlew eclipse`
@@ -587,13 +587,13 @@ allows you to use these configurations arbitrarily. Here are some of the most
 common configurations in our build and how we use them:
 
 <dl>
-<dt>`implementation`</dt><dd>Dependencies that are used by the project 
+<dt>`implementation`</dt><dd>Dependencies that are used by the project
 at compile and runtime but are not exposed as a compile dependency to other dependent projects.
-Dependencies added to the `implementation` configuration are considered an implementation detail 
+Dependencies added to the `implementation` configuration are considered an implementation detail
 that can be changed at a later date without affecting any dependent projects.</dd>
-<dt>`api`</dt><dd>Dependencies that are used as compile and runtime depdendencies of a project
+<dt>`api`</dt><dd>Dependencies that are used as compile and runtime dependencies of a project
  and are considered part of the external api of the project.
-<dt>`runtimeOnly`</dt><dd>Dependencies that not on the classpath at compile time but 
+<dt>`runtimeOnly`</dt><dd>Dependencies that not on the classpath at compile time but
 are on the classpath at runtime. We mostly use this configuration to make sure that
 we do not accidentally compile against dependencies of our dependencies also
 known as "transitive" dependencies".</dd>

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.snapshots.EmptySnapshotsInfoService;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -79,7 +80,8 @@ public static AllocationService createAllocationService(Settings settings, Clust
             defaultAllocationDeciders(settings, clusterSettings),
             NoopGatewayAllocator.INSTANCE,
             new BalancedShardsAllocator(settings),
-            EmptyClusterInfoService.INSTANCE
+            EmptyClusterInfoService.INSTANCE,
+            EmptySnapshotsInfoService.INSTANCE
         );
     }
 

buildSrc/build.gradle

@@ -96,12 +96,12 @@ dependencies {
   api 'org.apache.ant:ant:1.10.8'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
-  api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
+  api 'com.netflix.nebula:gradle-info-plugin:9.1.1'
   api 'org.apache.rat:apache-rat:0.11'
   api "org.elasticsearch:jna:5.5.0"
-  api 'com.github.jengelman.gradle.plugins:shadow:6.0.0'
-  api 'de.thetaphi:forbiddenapis:3.0'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.12.1'
+  api 'com.github.jengelman.gradle.plugins:shadow:6.1.0'
+  api 'de.thetaphi:forbiddenapis:3.1'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.13.4'
   api 'org.apache.maven:maven-model:3.6.2'
   api 'com.networknt:json-schema-validator:1.0.36'
   compileOnly "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy

@@ -89,6 +89,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         "darwin"  | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION   | "Contents/Home/bin/java" | ""
         "darwin"  | VENDOR_OPENJDK      | OPEN_JDK_VERSION    | "Contents/Home/bin/java" | ""
         "darwin"  | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION    | "Contents/Home/bin/java" | ""
+        "mac"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
     }
 
     def "transforms are reused across projects"() {
@@ -200,10 +202,10 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
     private static String urlPath(final String vendor, final String version, final String platform) {
         if (vendor.equals(VENDOR_ADOPTOPENJDK)) {
-            final String module = platform.equals("darwin") ? "mac" : platform;
+            final String module = isMac(platform) ? "mac" : platform;
             return "/jdk-12.0.2+10/" + module + "/x64/jdk/hotspot/normal/adoptopenjdk";
         } else if (vendor.equals(VENDOR_OPENJDK)) {
-            final String effectivePlatform = platform.equals("darwin") ? "osx" : platform;
+            final String effectivePlatform = isMac(platform) ? "osx" : platform;
             final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
             final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
             final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
@@ -212,14 +214,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     }
 
     private static byte[] filebytes(final String vendor, final String platform) throws IOException {
-        final String effectivePlatform = platform.equals("darwin") ? "osx" : platform;
+        final String effectivePlatform = isMac(platform) ? "osx" : platform;
         if (vendor.equals(VENDOR_ADOPTOPENJDK)) {
             return JdkDownloadPluginFuncTest.class.getResourceAsStream("fake_adoptopenjdk_" + effectivePlatform + "." + extension(platform)).getBytes()
         } else if (vendor.equals(VENDOR_OPENJDK)) {
             JdkDownloadPluginFuncTest.class.getResourceAsStream("fake_openjdk_" + effectivePlatform + "." + extension(platform)).getBytes()
         }
     }
 
+    private static boolean isMac(String platform) {
+        platform.equals("darwin") || platform.equals("mac")
+    }
+
     private static String extension(String platform) {
         platform.equals("windows") ? "zip" : "tar.gz";
     }

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.fixtures
+
+import org.apache.commons.io.FileUtils
+import org.gradle.testkit.runner.GradleRunner
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
+
+abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
+
+    @Rule
+    TemporaryFolder remoteRepoDirs = new TemporaryFolder()
+
+    File remoteGitRepo
+
+    def setup() {
+        remoteGitRepo = new File(setupGitRemote(), '.git')
+        "git clone ${remoteGitRepo.absolutePath} cloned".execute(Collections.emptyList(), testProjectDir.root).waitFor()
+        buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
+        settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
+    }
+
+    File setupGitRemote() {
+        URL fakeRemote = getClass().getResource("fake_git/remote")
+        File workingRemoteGit = new File(remoteRepoDirs.root, 'remote')
+        FileUtils.copyDirectory(new File(fakeRemote.file), workingRemoteGit)
+        fakeRemote.file + "/.git"
+        gradleRunner(workingRemoteGit, "wrapper").build()
+
+        execute("git init", workingRemoteGit)
+        execute('git config user.email "[email protected]"', workingRemoteGit)
+        execute('git config user.name "Build tool"', workingRemoteGit)
+        execute("git add .", workingRemoteGit)
+        execute('git commit -m"Initial"', workingRemoteGit)
+        execute("git checkout -b origin/8.0", workingRemoteGit)
+        return workingRemoteGit;
+    }
+
+    GradleRunner gradleRunner(String... arguments) {
+        gradleRunner(new File(testProjectDir.root, "cloned"), arguments)
+    }
+
+}

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -112,8 +112,19 @@ abstract class AbstractGradleFuncTest extends Specification {
     }
 
     void setupLocalGitRepo() {
-        "git init".execute(Collections.emptyList(), testProjectDir.root).waitFor()
-        "git add .".execute(Collections.emptyList(), testProjectDir.root).waitFor()
-        'git commit -m "Initial"'.execute(Collections.emptyList(), testProjectDir.root).waitFor()
+        execute("git init")
+        execute('git config user.email "[email protected]"')
+        execute('git config user.name "Build tool"')
+        execute("git add .")
+        execute('git commit -m "Initial"')
+    }
+
+    void execute(String command, File workingDir = testProjectDir.root) {
+        def proc = command.execute(Collections.emptyList(), workingDir)
+        proc.waitFor()
+        if(proc.exitValue()) {
+            System.err.println("Error running command ${command}:")
+            System.err.println("Syserr: " + proc.errorStream.text)
+        }
     }
 }

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy

@@ -19,33 +19,60 @@
 
 package org.elasticsearch.gradle.internal
 
-import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
-class InternalBwcGitPluginFuncTest extends AbstractGradleFuncTest {
+class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
     def setup() {
-        setupLocalGitRepo()
-    }
-
-    def "current repository can be cloned"() {
-        given:
-        internalBuild();
+        internalBuild()
         buildFile << """
             import org.elasticsearch.gradle.Version;
             apply plugin: org.elasticsearch.gradle.internal.InternalBwcGitPlugin  
             
             bwcGitConfig {
-                 bwcVersion = project.provider { Version.fromString("7.10.0") }
-                 bwcBranch = project.provider { "7.x" }
+                 bwcVersion = project.provider { Version.fromString("8.1.0") }
+                 bwcBranch = project.provider { "8.0" }
                  checkoutDir = project.provider{file("build/checkout")}
             }
         """
+    }
+
+    def "current repository can be cloned"() {
         when:
         def result = gradleRunner("createClone", '--stacktrace').build()
         then:
         result.task(":createClone").outcome == TaskOutcome.SUCCESS
-        file("build/checkout/build.gradle").exists()
-        file("build/checkout/settings.gradle").exists()
+        file("cloned/build/checkout/build.gradle").exists()
+        file("cloned/build/checkout/settings.gradle").exists()
+    }
+
+    def "can resolve checkout folder as project artifact"() {
+        given:
+        settingsFile << "include ':consumer'"
+        file("cloned/consumer/build.gradle") << """
+            configurations {
+                consumeCheckout
+            }
+            dependencies {
+                consumeCheckout project(path:":", configuration: "checkout")
+            }
+            
+            tasks.register("register") {
+                dependsOn configurations.consumeCheckout
+                doLast {
+                    configurations.consumeCheckout.files.each {
+                        println "checkoutDir artifact: " + it
+                    }
+                }
+            }
+        """
+        when:
+        def result = gradleRunner(":consumer:register", '--stacktrace', "-DtestRemoteRepo=" + remoteGitRepo,
+                "-Dbwc.remote=origin").build()
+        then:
+        result.task(":checkoutBwcBranch").outcome == TaskOutcome.SUCCESS
+        result.task(":consumer:register").outcome == TaskOutcome.SUCCESS
+        normalizedOutput(result.output).contains("/cloned/build/checkout")
     }
 }

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -99,11 +99,14 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
         import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
         import org.gradle.api.internal.artifacts.ArtifactAttributes;
 
+        def snapshotFile = file("snapshot-\${version}.txt")
+        snapshotFile << 'some snapshot content'
         distribution_archives {
             producerTar {
                 content {
                     project.copySpec {
                         from 'someFile.txt'
+                        from snapshotFile
                     }
                 }
             }
@@ -134,19 +137,27 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
         when:
         def result = gradleRunner("copyArchive").build()
 
-        then:"tar task executed and target folder contains plain tar"
+        then: "tar task executed and target folder contains plain tar"
         result.task(':buildProducerTar').outcome == TaskOutcome.SUCCESS
         result.task(':consumer:copyArchive').outcome == TaskOutcome.SUCCESS
         file("producer-tar/build/distributions/elasticsearch.tar.gz").exists()
         file("consumer/build/archives/elasticsearch.tar.gz").exists()
 
         when:
-        result = gradleRunner("copyDir").build()
-        then:"plain copy task executed and target folder contains plain content"
+        result = gradleRunner("copyDir", "-Pversion=1.0").build()
+        then: "plain copy task executed and target folder contains plain content"
         result.task(':buildProducer').outcome == TaskOutcome.SUCCESS
         result.task(':consumer:copyDir').outcome == TaskOutcome.SUCCESS
         file("producer-tar/build/install/someFile.txt").exists()
+        file("producer-tar/build/install/snapshot-1.0.txt").exists()
         file("consumer/build/dir/someFile.txt").exists()
+
+        when:
+        gradleRunner("copyDir", "-Pversion=2.0").build()
+        then: "old content is cleared out"
+        file("producer-tar/build/install/someFile.txt").exists()
+        !file("producer-tar/build/install/snapshot-1.0.txt").exists()
+        file("producer-tar/build/install/snapshot-2.0.txt").exists()
     }
 
     private static boolean assertTarPermissionDefaults(File tarArchive) {