Merge branch 'master' into replicated-closed-indices

Author: tlrx

.ci/packer_cache.sh

@@ -21,4 +21,4 @@ export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
 export JAVA8_HOME="${HOME}"/.java/java8
 export JAVA11_HOME="${HOME}"/.java/java11
 export JAVA12_HOME="${HOME}"/.java/java12
-./gradlew --parallel  clean pullFixture  --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies
+./gradlew --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies

build.gradle

@@ -616,9 +616,10 @@ if (System.properties.get("build.compare") != null) {
 
 allprojects {
   task resolveAllDependencies {
-    doLast {
-      configurations.findAll { it.isCanBeResolved() }.each { it.resolve() }
-    }
+      dependsOn tasks.matching { it.name == "pullFixture"}
+      doLast {
+        configurations.findAll { it.isCanBeResolved() }.each { it.resolve() }
+      }
   }
 }
 

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -3,15 +3,14 @@ package com.carrotsearch.gradle.junit4
 import com.carrotsearch.ant.tasks.junit4.JUnit4
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
 import org.gradle.api.tasks.TaskContainer
 
 class RandomizedTestingPlugin implements Plugin<Project> {
 
     void apply(Project project) {
-        setupSeed(project)
+        String seed = setupSeed(project)
         createUnitTestTask(project.tasks)
-        configureAnt(project.ant)
+        configureAnt(project.ant, seed)
     }
 
     /**
@@ -21,12 +20,12 @@ class RandomizedTestingPlugin implements Plugin<Project> {
      * outcome of subsequent runs. Pinning the seed up front like this makes
      * the reproduction line from one run be useful on another run.
      */
-    static void setupSeed(Project project) {
+    static String setupSeed(Project project) {
         if (project.rootProject.ext.has('testSeed')) {
             /* Skip this if we've already pinned the testSeed. It is important
              * that this checks the rootProject so that we know we've only ever
              * initialized one time. */
-            return
+            return project.rootProject.ext.testSeed
         }
         String testSeed = System.getProperty('tests.seed')
         if (testSeed == null) {
@@ -39,6 +38,8 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         project.rootProject.subprojects {
             project.ext.testSeed = testSeed
         }
+
+        return testSeed
     }
 
     static void createUnitTestTask(TaskContainer tasks) {
@@ -52,7 +53,8 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         }
     }
 
-    static void configureAnt(AntBuilder ant) {
+    static void configureAnt(AntBuilder ant, String seed) {
         ant.project.addTaskDefinition('junit4:junit4', JUnit4.class)
+        ant.properties.put('tests.seed', seed)
     }
 }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy

@@ -32,6 +32,7 @@ import com.carrotsearch.ant.tasks.junit4.events.aggregated.HeartBeatEvent
 import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
 import org.gradle.internal.logging.progress.ProgressLogger
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
+import org.junit.runner.Description
 
 import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDurationInSeconds
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.ERROR
@@ -113,7 +114,7 @@ class TestProgressLogger implements AggregatedEventListener {
 
     @Subscribe
     void onSuiteStart(AggregatedSuiteStartedEvent e) throws IOException {
-        String suiteName = simpleName(e.suiteStartedEvent.description.className)
+        String suiteName = simpleName(e.suiteStartedEvent.description)
         slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${suiteName} - initializing")
     }
 
@@ -146,31 +147,45 @@ class TestProgressLogger implements AggregatedEventListener {
             throw new IllegalArgumentException("Unknown test status: [${e.status}]")
         }
         testLogger.progress("Tests: completed: ${testsCompleted}, failed: ${testsFailed}, ignored: ${testsIgnored}")
-        String testName = simpleName(e.description.className) + '.' + e.description.methodName
+        String testName = testName(e.description)
         slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} ${statusMessage}")
     }
 
     @Subscribe
     void onTestStarted(TestStartedEvent e) throws IOException {
-        String testName = simpleName(e.description.className) + '.' + e.description.methodName
+        String testName = testName(e.description)
         slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} ...")
     }
 
     @Subscribe
     void onHeartbeat(HeartBeatEvent e) throws IOException {
-        String testName = simpleName(e.description.className) + '.' + e.description.methodName
+        String testName = testName(e.description)
         String time = formatDurationInSeconds(e.getNoEventDuration())
         slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} stalled for ${time}")
     }
 
+    /**
+     * Build the test name in the format of <className>.<methodName>
+     */
+    private static String testName(Description description) {
+        String className = simpleName(description)
+        if (description == null) {
+            return className + "." + "<unknownMethod>"
+        }
+        return className + "." + description.methodName
+    }
+
     /**
      * Extract a Class#getSimpleName style name from Class#getName style
      * string. We can't just use Class#getSimpleName because junit descriptions
      * don't always set the class field but they always set the className
      * field.
      */
-    private static String simpleName(String className) {
-        return className.substring(className.lastIndexOf('.') + 1)
+    private static String simpleName(Description description) {
+        if (description == null) {
+            return "<unknownClass>"
+        }
+        return description.className.substring(description.className.lastIndexOf('.') + 1)
     }
 
     @Override

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -118,11 +118,16 @@ class ClusterConfiguration {
         if (seedNode == node) {
             return null
         }
-        ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
+        ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond',
+                timeoutproperty: "failed.${seedNode.transportPortsFile.path}") {
             resourceexists {
                 file(file: seedNode.transportPortsFile.toString())
             }
         }
+        if (ant.properties.containsKey("failed.${seedNode.transportPortsFile.path}".toString())) {
+            throw new GradleException("Failed to locate seed node transport file [${seedNode.transportPortsFile}]: " +
+                    "timed out waiting for it to be created after ${waitSeconds} seconds")
+        }
         return seedNode.transportUri()
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -547,7 +547,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                     project.gradle.removeListener(batsPackagingReproListener)
                 }
                 if (project.extensions.esvagrant.boxes.contains(box)) {
-                    packagingTest.dependsOn(batsPackagingTest)
+                    // these tests are temporarily disabled for suse boxes while we debug an issue
+                    // https://github.com/elastic/elasticsearch/issues/30295
+                    if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                        packagingTest.dependsOn(batsPackagingTest)
+                    }
                 }
             }
 
@@ -586,7 +590,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                 project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(javaPackagingTest)
+                // these tests are temporarily disabled for suse boxes while we debug an issue
+                // https://github.com/elastic/elasticsearch/issues/30295
+                if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                    packagingTest.dependsOn(javaPackagingTest)
+                }
             }
 
             /*

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -243,35 +243,46 @@ synchronized void stop(boolean tailLogs) {
         }
         logger.info("Stopping `{}`, tailLogs: {}", this, tailLogs);
         requireNonNull(esProcess, "Can't stop `" + this + "` as it was not started or already stopped.");
-        stopHandle(esProcess.toHandle());
+        // Test clusters are not reused, don't spend time on a graceful shutdown
+        stopHandle(esProcess.toHandle(), true);
         if (tailLogs) {
             logFileContents("Standard output of node", esStdoutFile);
             logFileContents("Standard error of node", esStderrFile);
         }
         esProcess = null;
     }
 
-    private void stopHandle(ProcessHandle processHandle) {
+    private void stopHandle(ProcessHandle processHandle, boolean forcibly) {
         // Stop all children first, ES could actually be a child when there's some wrapper process like on Windows.
-        if (processHandle.isAlive()) {
-            processHandle.children().forEach(this::stopHandle);
-        }
-        logProcessInfo("Terminating elasticsearch process:", processHandle.info());
-        if (processHandle.isAlive()) {
-            processHandle.destroy();
-        } else {
+        if (processHandle.isAlive() == false) {
             logger.info("Process was not running when we tried to terminate it.");
+            return;
         }
-        waitForProcessToExit(processHandle);
-        if (processHandle.isAlive()) {
+
+        // Stop all children first, ES could actually be a child when there's some wrapper process like on Windows.
+        processHandle.children().forEach(each -> stopHandle(each, forcibly));
+
+        logProcessInfo(
+            "Terminating elasticsearch process" + (forcibly ? " forcibly " : "gracefully") + ":",
+            processHandle.info()
+        );
+
+        if (forcibly) {
+            processHandle.destroyForcibly();
+        } else {
+            processHandle.destroy();
+            waitForProcessToExit(processHandle);
+            if (processHandle.isAlive() == false) {
+                return;
+            }
             logger.info("process did not terminate after {} {}, stopping it forcefully",
-                ES_DESTROY_TIMEOUT, ES_DESTROY_TIMEOUT_UNIT
-            );
+                ES_DESTROY_TIMEOUT, ES_DESTROY_TIMEOUT_UNIT);
             processHandle.destroyForcibly();
         }
+
         waitForProcessToExit(processHandle);
         if (processHandle.isAlive()) {
-            throw new TestClustersException("Was not able to terminate es process");
+            throw new TestClustersException("Was not able to terminate elasticsearch process");
         }
     }