Merge branch 'master' into peer-recovery-retention-leases

Author: DaveCTurner

.ci/bwcVersions

@@ -0,0 +1,10 @@
+BWC_VERSION:
+  - "7.0.0"
+  - "7.0.1"
+  - "7.1.0"
+  - "7.1.1"
+  - "7.2.0"
+  - "7.2.1"
+  - "7.3.0"
+  - "7.4.0"
+  - "8.0.0"

build.gradle

@@ -29,16 +29,13 @@ import org.gradle.util.DistributionLocator
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
 plugins {
-    id 'com.gradle.build-scan' version '2.2.1'
+    id 'com.gradle.build-scan' version '2.3'
     id 'base'
     id 'elasticsearch.global-build-info'
 }
-if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) {
-    buildScan {
-        termsOfServiceUrl = 'https://gradle.com/terms-of-service'
-        termsOfServiceAgree = 'yes'
-    }
-}
+
+apply plugin: 'nebula.info-scm'
+apply from: 'gradle/build-scan.gradle'
 
 // common maven publishing configuration
 allprojects {
@@ -49,7 +46,6 @@ allprojects {
 
 BuildPlugin.configureRepositories(project)
 
-apply plugin: 'nebula.info-scm'
 String licenseCommit
 if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
   licenseCommit = scminfo.change ?: "master" // leniency for non git builds
@@ -107,6 +103,17 @@ subprojects {
  * logic in VersionUtils.java. */
 BwcVersions versions = new BwcVersions(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
 
+task updateCIBwcVersions() {
+    doLast {
+        File yml = file(".ci/bwcVersions")
+        yml.text = ""
+        yml << "BWC_VERSION:\n"
+        versions.indexCompatible.each {
+            yml << "  - \"$it\"\n"
+        }
+    }
+}
+
 // build metadata from previous build, contains eg hashes for bwc builds
 String buildMetadataValue = System.getenv('BUILD_METADATA')
 if (buildMetadataValue == null) {
@@ -150,6 +157,12 @@ task verifyVersions {
                       .collect { Version.fromString(it) }
       )
     }
+    String ciYml = file(".ci/bwcVersions").text
+    bwcVersions.indexCompatible.each {
+        if (ciYml.contains("\"$it\"\n") == false) {
+            throw new Exception(".ci/bwcVersions is outdated, run `./gradlew updateCIBwcVersions` and check in the results");
+        }
+    }
   }
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -21,11 +21,9 @@ package org.elasticsearch.gradle.doc
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.test.ClusterFormationTasks
 import org.elasticsearch.gradle.test.RestTestPlugin
 import org.gradle.api.Project
 import org.gradle.api.Task
-
 /**
  * Sets up tests for documentation.
  */
@@ -38,7 +36,7 @@ public class DocsTestPlugin extends RestTestPlugin {
         super.apply(project)
         String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.testClusters.integTest.distribution = distribution.toUpperCase()
+        project.testClusters.integTest.testDistribution = distribution.toUpperCase()
         project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") }
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -353,7 +353,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
 
         private void testSetup(Snippet snippet) {
             if (lastDocsPath == snippet.path) {
-                throw new InvalidUserDataException("$snippet: wasn't first")
+                throw new InvalidUserDataException("$snippet: wasn't first. TESTSETUP can only be used in the first snippet of a document.")
             }
             setupCurrent(snippet)
             current.println('---')

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,15 +18,14 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
+
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.logging.Logger
 import org.gradle.api.logging.Logging
-import org.gradle.api.specs.Specs
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
@@ -37,7 +36,6 @@ import org.gradle.plugins.ide.idea.IdeaPlugin
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.util.stream.Stream
-
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
@@ -69,8 +67,6 @@ class RestIntegTestTask extends DefaultTask {
         } else {
             project.testClusters {
                 "$name" {
-                    distribution = 'INTEG_TEST'
-                    version = VersionProperties.elasticsearch
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }

buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java

@@ -54,6 +54,7 @@
  */
 public class DistributionDownloadPlugin implements Plugin<Project> {
 
+    private static final String CONTAINER_NAME = "elasticsearch_distributions";
     private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
     private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
 
@@ -67,7 +68,7 @@ public void apply(Project project) {
             Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
             return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration);
         });
-        project.getExtensions().add("elasticsearch_distributions", distributionsContainer);
+        project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
 
         setupDownloadServiceRepo(project);
 
@@ -78,6 +79,11 @@ public void apply(Project project) {
         project.afterEvaluate(this::setupDistributions);
     }
 
+    @SuppressWarnings("unchecked")
+    public static NamedDomainObjectContainer<ElasticsearchDistribution> getContainer(Project project) {
+        return (NamedDomainObjectContainer<ElasticsearchDistribution>) project.getExtensions().getByName(CONTAINER_NAME);
+    }
+
     // pkg private for tests
     void setupDistributions(Project project) {
         for (ElasticsearchDistribution distribution : distributionsContainer) {

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -20,14 +20,17 @@
 package org.elasticsearch.gradle;
 
 import org.gradle.api.Buildable;
+import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.TaskDependency;
 
 import java.io.File;
 import java.util.Iterator;
 import java.util.Locale;
+import java.util.concurrent.Callable;
 
 public class ElasticsearchDistribution implements Buildable {
 
@@ -65,7 +68,7 @@ public String toString() {
     }
 
     // package private to tests can use
-    static final Platform CURRENT_PLATFORM = OS.<Platform>conditional()
+    public static final Platform CURRENT_PLATFORM = OS.<Platform>conditional()
         .onLinux(() -> Platform.LINUX)
         .onWindows(() -> Platform.WINDOWS)
         .onMac(() -> Platform.DARWIN)
@@ -90,6 +93,10 @@ public TaskDependency getBuildDependencies() {
             return configuration.getBuildDependencies();
         }
 
+        public FileTree getFileTree(Project project) {
+            return project.fileTree((Callable<File>) configuration::getSingleFile);
+        }
+
         @Override
         public String toString() {
             return configuration.getSingleFile().toString();

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -18,8 +18,10 @@
  */
 package org.elasticsearch.gradle.precommit;
 
+import de.thetaphi.forbiddenapis.cli.CliMain;
 import org.apache.commons.io.output.NullOutputStream;
 import org.elasticsearch.gradle.JdkJarHellCheck;
+import org.elasticsearch.gradle.OS;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
@@ -51,6 +53,7 @@
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
@@ -69,6 +72,12 @@ public class ThirdPartyAuditTask extends DefaultTask {
     private static final Pattern VIOLATION_PATTERN = Pattern.compile(
         "\\s\\sin ([a-zA-Z0-9$.]+) \\(.*\\)"
     );
+    private static final int SIG_KILL_EXIT_VALUE = 137;
+    private static final List<Integer> EXPECTED_EXIT_CODES = Arrays.asList(
+        CliMain.EXIT_SUCCESS,
+        CliMain.EXIT_VIOLATION,
+        CliMain.EXIT_UNSUPPORTED_JDK
+    );
 
     private Set<String> missingClassExcludes = new TreeSet<>();
 
@@ -327,7 +336,7 @@ private String formatClassList(Set<String> classList) {
 
     private String runForbiddenAPIsCli() throws IOException {
         ByteArrayOutputStream errorOut = new ByteArrayOutputStream();
-        getProject().javaexec(spec -> {
+        ExecResult result = getProject().javaexec(spec -> {
             if (javaHome != null) {
                 spec.setExecutable(javaHome + "/bin/java");
             }
@@ -336,6 +345,7 @@ private String runForbiddenAPIsCli() throws IOException {
                 getRuntimeConfiguration(),
                 getProject().getConfigurations().getByName("compileOnly")
             );
+            spec.jvmArgs("-Xmx1g");
             spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain");
             spec.args(
                 "-f", getSignatureFile().getAbsolutePath(),
@@ -348,10 +358,18 @@ private String runForbiddenAPIsCli() throws IOException {
             }
             spec.setIgnoreExitValue(true);
         });
+        if (OS.current().equals(OS.LINUX) && result.getExitValue() == SIG_KILL_EXIT_VALUE) {
+            throw new IllegalStateException(
+                "Third party audit was killed buy SIGKILL, could be a victim of the Linux OOM killer"
+            );
+        }
         final String forbiddenApisOutput;
         try (ByteArrayOutputStream outputStream = errorOut) {
             forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name());
         }
+        if (EXPECTED_EXIT_CODES.contains(result.getExitValue()) == false) {
+            throw new IllegalStateException("Forbidden APIs cli failed: " + forbiddenApisOutput);
+        }
         return forbiddenApisOutput;
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -8,7 +8,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 
-import static org.elasticsearch.gradle.Distribution.INTEG_TEST;
+import static org.elasticsearch.gradle.testclusters.TestDistribution.INTEG_TEST;
 
 /**
  * Customized version of Gradle {@link Test} task which tracks a collection of {@link ElasticsearchCluster} as a task input. We must do this
@@ -23,7 +23,7 @@ public class RestTestRunnerTask extends Test {
     public RestTestRunnerTask() {
         super();
         this.getOutputs().doNotCacheIf("Build cache is only enabled for tests against clusters using the 'integ-test' distribution",
-            task -> clusters.stream().flatMap(c -> c.getNodes().stream()).anyMatch(n -> n.getDistribution() != INTEG_TEST));
+            task -> clusters.stream().flatMap(c -> c.getNodes().stream()).anyMatch(n -> n.getTestDistribution() != INTEG_TEST));
     }
 
     @Nested