Rerun test task when test jdk crashed with System exit (#71881) (#72003)

Related to #52610 this PR introduces a rerun of all tests for a test task if the test jvm 
has crashed because of a system exit. We furthermore log potential tests that caused 
the System.exit based on which tests have been active at the time of the system exit.

We also modified the build scan logic to track unexpected test jvm exists 
with the tag `unexpected-test-jvm-exit`

Author: breskeby

build.gradle

@@ -56,6 +56,8 @@ if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
 String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE-2.0.txt"
 
 subprojects {
+  apply plugin:'elasticsearch.internal-test-rerun'
+
   // Default to the SSPL+Elastic dual license
   project.ext.projectLicenses = [
     'Server Side Public License, v 1': 'https://www.mongodb.com/licensing/server-side-public-license',
@@ -102,6 +104,7 @@ subprojects {
     project.licenseFile = project.rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
     project.noticeFile = project.rootProject.file('NOTICE.txt')
   }
+
 }
 
 /**

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -67,6 +67,7 @@ abstract class AbstractGradleFuncTest extends Specification {
         return input.readLines()
                 .collect { it.replace('\\', '/') }
                 .collect {it.replace(normalizedPathPrefix , '.') }
+                .collect {it.replaceAll(/Gradle Test Executor \d/ , 'Gradle Test Executor 1') }
                 .join("\n")
     }
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy

@@ -0,0 +1,242 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+
+class InternalTestRerunPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "does not rerun on failed tests"() {
+        when:
+        buildFile.text = """
+        plugins {
+          id 'java'
+          id 'elasticsearch.internal-test-rerun'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+        
+        dependencies {
+            testImplementation 'junit:junit:4.13.1'
+        }
+        
+        tasks.named("test").configure {
+            maxParallelForks = 4
+            testLogging {
+                events "standard_out", "failed"
+                exceptionFormat "short"
+            }
+        }
+        
+        """
+        createTest("SimpleTest")
+        createTest("SimpleTest2")
+        createTest("SimpleTest3")
+        createTest("SimpleTest4")
+        createTest("SimpleTest5")
+        createFailedTest("SimpleTest6")
+        createFailedTest("SimpleTest7")
+        createFailedTest("SimpleTest8")
+        createTest("SomeOtherTest")
+        createTest("SomeOtherTest1")
+        createTest("SomeOtherTest2")
+        createTest("SomeOtherTest3")
+        createTest("SomeOtherTest4")
+        createTest("SomeOtherTest5")
+        then:
+        def result = gradleRunner("test").buildAndFail()
+        result.output.contains("total executions: 2") == false
+        and: "no jvm system exit tracing provided"
+        normalized(result.output).contains("""Test jvm exited unexpectedly.
+Test jvm system exit trace:""") == false
+    }
+
+    def "all tests are rerun when test jvm has crashed"() {
+        when:
+        settingsFile.text = """
+        plugins {
+            id "com.gradle.enterprise" version "3.6.1"
+        }
+        gradleEnterprise {
+            server = 'https://gradle-enterprise.elastic.co/'
+        }
+        """ + settingsFile.text
+
+        buildFile.text = """
+        plugins {
+          id 'java'
+          id 'elasticsearch.internal-test-rerun'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+        
+        dependencies {
+            testImplementation 'junit:junit:4.13.1'
+        }
+        
+        tasks.named("test").configure {
+            maxParallelForks = 4
+            testLogging {
+                // set options for log level LIFECYCLE
+                events "started", "passed", "standard_out", "failed"
+                exceptionFormat "short"
+            }
+        }
+        
+        """
+        createTest("AnotherTest")
+        createFailedTest("AnotherTest1")
+        createTest("AnotherTest2")
+        createTest("AnotherTest3")
+        createTest("AnotherTest4")
+        createTest("AnotherTest5")
+        createSystemExitTest("AnotherTest6")
+        createTest("AnotherTest7")
+        createTest("AnotherTest8")
+        createFailedTest("AnotherTest9")
+        createTest("AnotherTest10")
+        createTest("SimpleTest")
+        createTest("SimpleTest2")
+        createTest("SimpleTest3")
+        createTest("SimpleTest4")
+        createTest("SimpleTest5")
+        createTest("SimpleTest6")
+        createTest("SimpleTest7")
+        createTest("SimpleTest8")
+        createTest("SomeOtherTest")
+        then:
+        def result = gradleRunner("test").build()
+        result.output.contains("AnotherTest6 total executions: 2")
+        // triggered only in the second overall run
+        and: 'Tracing is provided'
+        normalized(result.output).contains("""================
+Test jvm exited unexpectedly.
+Test jvm system exit trace (run: 1)
+Gradle Test Executor 1 > AnotherTest6 > someTest
+================""")
+    }
+
+    def "reruns tests till max rerun count is reached"() {
+        when:
+        buildFile.text = """
+        plugins {
+          id 'java'
+          id 'elasticsearch.internal-test-rerun'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+        
+        dependencies {
+            testImplementation 'junit:junit:4.13.1'
+        }
+        
+        tasks.named("test").configure {
+            rerun {
+                maxReruns = 4
+            }
+            testLogging {
+                // set options for log level LIFECYCLE
+                events "standard_out", "failed"
+                exceptionFormat "short"
+            }
+        }
+        """
+        createSystemExitTest("JdkKillingTest", 5)
+        then:
+        def result = gradleRunner("test").buildAndFail()
+        result.output.contains("JdkKillingTest total executions: 4")
+        result.output.contains("Max retries(4) hit")
+        and: 'Tracing is provided'
+        normalized(result.output).contains("Test jvm system exit trace (run: 1)")
+        normalized(result.output).contains("Test jvm system exit trace (run: 2)")
+        normalized(result.output).contains("Test jvm system exit trace (run: 3)")
+        normalized(result.output).contains("Test jvm system exit trace (run: 4)")
+    }
+
+    private String testMethodContent(boolean withSystemExit, boolean fail, int timesFailing = 1) {
+        return """
+            int count = countExecutions();
+            System.out.println(getClass().getSimpleName() + " total executions: " + count);
+
+            ${withSystemExit ? """
+                    if(count <= ${timesFailing}) {
+                        System.exit(1);
+                    }
+                    """ : ''
+            }
+
+            ${fail ? """
+                    if(count <= ${timesFailing}) {
+                        Assert.fail();
+                    }
+                    """ : ''
+            }
+        """
+    }
+
+    private File createSystemExitTest(String clazzName, timesFailing = 1) {
+        createTest(clazzName, testMethodContent(true, false, timesFailing))
+    }
+    private File createFailedTest(String clazzName) {
+        createTest(clazzName, testMethodContent(false, true, 1))
+    }
+
+    private File createTest(String clazzName, String content = testMethodContent(false, false, 1)) {
+        file("src/test/java/org/acme/${clazzName}.java") << """
+            import org.junit.Test;
+            import org.junit.Before;
+            import org.junit.After;
+            import org.junit.Assert;
+            import java.nio.*;
+            import java.nio.file.*;
+            import java.io.IOException;
+            
+            public class $clazzName {
+                Path executionLogPath = Paths.get("test-executions" + getClass().getSimpleName() +".log");
+                
+                @Before 
+                public void beforeTest() {
+                    logExecution();
+                }
+                
+                @After 
+                public void afterTest() {
+                }
+                
+                @Test 
+                public void someTest() {
+                    ${content}
+                }
+                
+                int countExecutions() {
+                    try {
+                        return Files.readAllLines(executionLogPath).size();
+                    }
+                    catch(IOException e) {
+                        return 0;
+                    }
+                }
+               
+                void logExecution() {
+                    try {
+                        Files.write(executionLogPath, "Test executed\\n".getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+                    } catch (IOException e) {
+                        // exception handling
+                    }
+                }
+            }
+        """
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+
+import static org.elasticsearch.gradle.internal.test.rerun.TestTaskConfigurer.configureTestTask;
+
+public class TestRerunPlugin implements Plugin<Project> {
+
+    private final ObjectFactory objectFactory;
+
+    @Inject
+    TestRerunPlugin(ObjectFactory objectFactory) {
+        this.objectFactory = objectFactory;
+    }
+
+    @Override
+    public void apply(Project project) {
+        project.getTasks().withType(Test.class).configureEach(task -> configureTestTask(task, objectFactory));
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun;
+
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+
+/**
+ * Allows configuring test rerun mechanics.
+ * <p>
+ * This extension is added with the name 'rerun' to all {@link Test} tasks.
+ */
+public class TestRerunTaskExtension {
+
+    /**
+     * The default number of reruns we allow for a test task.
+     */
+    public static final Integer DEFAULT_MAX_RERUNS = 1;
+
+    /**
+     * The name of the extension added to each test task.
+     */
+    public static String NAME = "rerun";
+
+    private final Property<Integer> maxReruns;
+
+    private final Property<Boolean> didRerun;
+
+    @Inject
+    public TestRerunTaskExtension(ObjectFactory objects) {
+        this.maxReruns = objects.property(Integer.class).convention(DEFAULT_MAX_RERUNS);
+        this.didRerun = objects.property(Boolean.class).convention(Boolean.FALSE);
+    }
+
+    /**
+     * The maximum number of times to rerun all tests.
+     * <p>
+     * This setting defaults to {@code 0}, which results in no retries.
+     * Any value less than 1 disables rerunning.
+     *
+     * @return the maximum number of times to rerun all tests of a task
+     */
+    public Property<Integer> getMaxReruns() {
+        return maxReruns;
+    }
+
+    /**
+     /**
+     * @return whether tests tests have been rerun or not. Defaults to false.
+     */
+    public Property<Boolean> getDidRerun() {
+        return didRerun;
+    }
+
+}