Skip to content

Commit be55668

Browse files
marcphilippmarcphilipp
authored
Use suite events for more accurate reporting (#2985)
In 4.13 testSuiteStarted/Finished were introduced. While top-level test class events were already reported at the appropriate time, events for intermediate levels such as iterations of the `Parameterized` runner or classes executed by the `Suite` runner were created synthetically when the first test was started and the last test was finished, respectively.
1 parent 5fdb138 commit be55668

File tree

8 files changed

+281
-8
lines changed

8 files changed

+281
-8
lines changed

‎documentation/src/docs/asciidoc/release-notes/release-notes-5.9.0.adoc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,4 +56,5 @@ GitHub.
5656

5757
==== New Features and Improvements
5858

59-
* ❓
59+
* More accurate reporting of intermediate start/finish events, e.g. iterations of the
60+
`Parameterized` runner and classes executed indirectly via the `Suite` runner.

‎junit-vintage-engine/src/main/java/org/junit/vintage/engine/JUnit4VersionCheck.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ static void checkSupported(Supplier<String> versionSupplier) {
5151
}
5252
}
5353

54-
private static BigDecimal parseVersion(String versionString) {
54+
static BigDecimal parseVersion(String versionString) {
5555
try {
5656
Matcher matcher = versionPattern.matcher(versionString);
5757
if (matcher.matches()) {

‎junit-vintage-engine/src/main/java/org/junit/vintage/engine/execution/RunListenerAdapter.java

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,15 @@ public void testRunStarted(Description description) {
5454
}
5555
}
5656

57+
@Override
58+
public void testSuiteStarted(Description description) {
59+
RunnerTestDescriptor runnerTestDescriptor = testRun.getRunnerTestDescriptor();
60+
// runnerTestDescriptor is reported in testRunStarted
61+
if (!runnerTestDescriptor.getDescription().equals(description)) {
62+
testStarted(lookupOrRegisterNextTestDescriptor(description), EventType.REPORTED);
63+
}
64+
}
65+
5766
@Override
5867
public void testIgnored(Description description) {
5968
testIgnored(lookupOrRegisterNextTestDescriptor(description), determineReasonForIgnoredTest(description));
@@ -80,17 +89,29 @@ public void testFinished(Description description) {
8089
}
8190

8291
@Override
83-
public void testRunFinished(Result result) {
92+
public void testSuiteFinished(Description description) {
8493
RunnerTestDescriptor runnerTestDescriptor = testRun.getRunnerTestDescriptor();
85-
if (testRun.isNotSkipped(runnerTestDescriptor)) {
86-
if (testRun.isNotStarted(runnerTestDescriptor)) {
87-
fireExecutionStarted(runnerTestDescriptor, EventType.SYNTHETIC);
94+
// runnerTestDescriptor is reported in testRunFinished
95+
if (!runnerTestDescriptor.getDescription().equals(description)) {
96+
reportContainerFinished(lookupOrRegisterNextTestDescriptor(description));
97+
}
98+
}
99+
100+
@Override
101+
public void testRunFinished(Result result) {
102+
reportContainerFinished(testRun.getRunnerTestDescriptor());
103+
}
104+
105+
private void reportContainerFinished(TestDescriptor containerTestDescriptor) {
106+
if (testRun.isNotSkipped(containerTestDescriptor)) {
107+
if (testRun.isNotStarted(containerTestDescriptor)) {
108+
fireExecutionStarted(containerTestDescriptor, EventType.SYNTHETIC);
88109
}
89110
testRun.getInProgressTestDescriptorsWithSyntheticStartEvents().stream() //
90111
.filter(this::canFinish) //
91112
.forEach(this::fireExecutionFinished);
92-
if (testRun.isNotFinished(runnerTestDescriptor)) {
93-
fireExecutionFinished(runnerTestDescriptor);
113+
if (testRun.isNotFinished(containerTestDescriptor)) {
114+
fireExecutionFinished(containerTestDescriptor);
94115
}
95116
}
96117
}
@@ -143,6 +164,7 @@ private void handleFailure(Failure failure, Function<Throwable, TestExecutionRes
143164
testStarted(testDescriptor, EventType.SYNTHETIC);
144165
}
145166
if (testRun.isNotFinished(testDescriptor) && testDescriptor.isContainer()
167+
&& testRun.hasSyntheticStartEvent(testDescriptor)
146168
&& testRun.isDescendantOfRunnerTestDescriptor(testDescriptor)) {
147169
testFinished(testDescriptor);
148170
}

‎junit-vintage-engine/src/main/java/org/junit/vintage/engine/execution/TestRun.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,10 @@ boolean isDescendantOfRunnerTestDescriptor(TestDescriptor testDescriptor) {
9191
return runnerDescendants.contains(testDescriptor);
9292
}
9393

94+
boolean hasSyntheticStartEvent(TestDescriptor testDescriptor) {
95+
return inProgressDescriptors.get(testDescriptor) == EventType.SYNTHETIC;
96+
}
97+
9498
Optional<VintageTestDescriptor> lookupNextTestDescriptor(Description description) {
9599
return lookupUnambiguouslyOrApplyFallback(description, VintageDescriptors::getNextUnstarted);
96100
}

‎junit-vintage-engine/src/test/java/org/junit/vintage/engine/VintageTestEngineExecutionTests.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
package org.junit.vintage.engine;
1212

1313
import static org.assertj.core.api.Assertions.assertThat;
14+
import static org.junit.jupiter.api.Assumptions.assumeTrue;
1415
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
1516
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
1617
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
@@ -31,6 +32,10 @@
3132
import static org.junit.runner.Description.createSuiteDescription;
3233
import static org.junit.runner.Description.createTestDescription;
3334

35+
import java.math.BigDecimal;
36+
37+
import junit.runner.Version;
38+
3439
import org.assertj.core.api.Condition;
3540
import org.junit.AssumptionViolatedException;
3641
import org.junit.jupiter.api.Test;
@@ -80,6 +85,9 @@
8085
import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithRunnerWithDuplicateChangingChildDescriptions;
8186
import org.junit.vintage.engine.samples.junit4.MalformedJUnit4TestCase;
8287
import org.junit.vintage.engine.samples.junit4.ParameterizedTestCase;
88+
import org.junit.vintage.engine.samples.junit4.ParameterizedTimingTestCase;
89+
import org.junit.vintage.engine.samples.junit4.ParameterizedWithAfterParamFailureTestCase;
90+
import org.junit.vintage.engine.samples.junit4.ParameterizedWithBeforeParamFailureTestCase;
8391
import org.junit.vintage.engine.samples.junit4.PlainJUnit4TestCaseWithFiveTestMethods;
8492
import org.junit.vintage.engine.samples.junit4.PlainJUnit4TestCaseWithLifecycleMethods;
8593
import org.junit.vintage.engine.samples.junit4.PlainJUnit4TestCaseWithSingleTestWhichFails;
@@ -455,6 +463,74 @@ void executesIgnoredParameterizedTestCase() {
455463
event(engine(), finishedSuccessfully()));
456464
}
457465

466+
@Test
467+
void executesParameterizedTimingTestCase() {
468+
assumeTrue(atLeastJUnit4_13(), "@BeforeParam and @AfterParam were introduced in JUnit 4.13");
469+
470+
Class<?> testClass = ParameterizedTimingTestCase.class;
471+
472+
var events = execute(testClass).allEvents().debug();
473+
474+
var firstParamStartedEvent = events.filter(event(container("[foo]"), started())::matches).findFirst() //
475+
.orElseThrow(() -> new AssertionError("No start event for [foo]"));
476+
var firstParamFinishedEvent = events.filter(
477+
event(container("[foo]"), finishedSuccessfully())::matches).findFirst() //
478+
.orElseThrow(() -> new AssertionError("No finish event for [foo]"));
479+
var secondParamStartedEvent = events.filter(event(container("[bar]"), started())::matches).findFirst() //
480+
.orElseThrow(() -> new AssertionError("No start event for [bar]"));
481+
var secondParamFinishedEvent = events.filter(
482+
event(container("[bar]"), finishedSuccessfully())::matches).findFirst() //
483+
.orElseThrow(() -> new AssertionError("No finish event for [bar]"));
484+
485+
assertThat(ParameterizedTimingTestCase.EVENTS.get("beforeParam(foo)")).isAfterOrEqualTo(
486+
firstParamStartedEvent.getTimestamp());
487+
assertThat(ParameterizedTimingTestCase.EVENTS.get("afterParam(foo)")).isBeforeOrEqualTo(
488+
firstParamFinishedEvent.getTimestamp());
489+
assertThat(ParameterizedTimingTestCase.EVENTS.get("beforeParam(bar)")).isAfterOrEqualTo(
490+
secondParamStartedEvent.getTimestamp());
491+
assertThat(ParameterizedTimingTestCase.EVENTS.get("afterParam(bar)")).isBeforeOrEqualTo(
492+
secondParamFinishedEvent.getTimestamp());
493+
}
494+
495+
@Test
496+
void executesParameterizedWithAfterParamFailureTestCase() {
497+
assumeTrue(atLeastJUnit4_13(), "@AfterParam was introduced in JUnit 4.13");
498+
499+
Class<?> testClass = ParameterizedWithAfterParamFailureTestCase.class;
500+
501+
execute(testClass).allEvents().assertEventsMatchExactly( //
502+
event(engine(), started()), //
503+
event(container(testClass), started()), //
504+
event(container("[foo]"), started()), //
505+
event(test("test[foo]"), started()), //
506+
event(test("test[foo]"), finishedSuccessfully()), //
507+
event(container("[foo]"), finishedWithFailure(instanceOf(AssertionError.class))), //
508+
event(container("[bar]"), started()), //
509+
event(test("test[bar]"), started()), //
510+
event(test("test[bar]"),
511+
finishedWithFailure(instanceOf(AssertionError.class), message("expected:<[foo]> but was:<[bar]>"))), //
512+
event(container("[bar]"), finishedWithFailure(instanceOf(AssertionError.class))), //
513+
event(container(testClass), finishedSuccessfully()), //
514+
event(engine(), finishedSuccessfully()));
515+
}
516+
517+
@Test
518+
void executesParameterizedWithBeforeParamFailureTestCase() {
519+
assumeTrue(atLeastJUnit4_13(), "@BeforeParam was introduced in JUnit 4.13");
520+
521+
Class<?> testClass = ParameterizedWithBeforeParamFailureTestCase.class;
522+
523+
execute(testClass).allEvents().assertEventsMatchExactly( //
524+
event(engine(), started()), //
525+
event(container(testClass), started()), //
526+
event(container("[foo]"), started()), //
527+
event(container("[foo]"), finishedWithFailure(instanceOf(AssertionError.class))), //
528+
event(container("[bar]"), started()), //
529+
event(container("[bar]"), finishedWithFailure(instanceOf(AssertionError.class))), //
530+
event(container(testClass), finishedSuccessfully()), //
531+
event(engine(), finishedSuccessfully()));
532+
}
533+
458534
@Test
459535
void executesJUnit4TestCaseWithExceptionThrowingRunner() {
460536
Class<?> testClass = JUnit4TestCaseWithExceptionThrowingRunner.class;
@@ -806,4 +882,8 @@ private static LauncherDiscoveryRequest request(Class<?> testClass) {
806882
return LauncherDiscoveryRequestBuilder.request().selectors(selectClass(testClass)).build();
807883
}
808884

885+
private static boolean atLeastJUnit4_13() {
886+
return JUnit4VersionCheck.parseVersion(Version.id()).compareTo(new BigDecimal("4.13")) >= 0;
887+
}
888+
809889
}

‎junit-vintage-engine/src/testFixtures/java/org/junit/vintage/engine/samples/junit4/ParameterizedTimingTestCase.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright 2015-2022 the original author or authors.
3+
*
4+
* All rights reserved. This program and the accompanying materials are
5+
* made available under the terms of the Eclipse Public License v2.0 which
6+
* accompanies this distribution and is available at
7+
*
8+
* https://www.eclipse.org/legal/epl-v20.html
9+
*/
10+
11+
package org.junit.vintage.engine.samples.junit4;
12+
13+
import static org.junit.Assert.assertEquals;
14+
15+
import java.time.Instant;
16+
import java.util.LinkedHashMap;
17+
import java.util.List;
18+
import java.util.Map;
19+
20+
import org.junit.BeforeClass;
21+
import org.junit.Test;
22+
import org.junit.runner.RunWith;
23+
import org.junit.runners.Parameterized;
24+
import org.junit.runners.Parameterized.AfterParam;
25+
import org.junit.runners.Parameterized.BeforeParam;
26+
import org.junit.runners.Parameterized.Parameter;
27+
import org.junit.runners.Parameterized.Parameters;
28+
29+
/**
30+
* @since 5.9
31+
*/
32+
@RunWith(Parameterized.class)
33+
public class ParameterizedTimingTestCase {
34+
35+
public static Map<String, Instant> EVENTS = new LinkedHashMap<>();
36+
37+
@BeforeClass
38+
public static void beforeClass() throws Exception {
39+
EVENTS.clear();
40+
}
41+
42+
@BeforeParam
43+
public static void beforeParam(String param) throws Exception {
44+
EVENTS.put("beforeParam(" + param + ")", Instant.now());
45+
Thread.sleep(100);
46+
}
47+
48+
@AfterParam
49+
public static void afterParam(String param) throws Exception {
50+
Thread.sleep(100);
51+
System.out.println("ParameterizedTimingTestCase.afterParam");
52+
EVENTS.put("afterParam(" + param + ")", Instant.now());
53+
}
54+
55+
@Parameters(name = "{0}")
56+
public static Iterable<String> parameters() {
57+
return List.of("foo", "bar");
58+
}
59+
60+
@Parameter
61+
public String value;
62+
63+
@Test
64+
public void test() {
65+
assertEquals("foo", value);
66+
}
67+
68+
}

‎junit-vintage-engine/src/testFixtures/java/org/junit/vintage/engine/samples/junit4/ParameterizedWithAfterParamFailureTestCase.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright 2015-2022 the original author or authors.
3+
*
4+
* All rights reserved. This program and the accompanying materials are
5+
* made available under the terms of the Eclipse Public License v2.0 which
6+
* accompanies this distribution and is available at
7+
*
8+
* https://www.eclipse.org/legal/epl-v20.html
9+
*/
10+
11+
package org.junit.vintage.engine.samples.junit4;
12+
13+
import static org.junit.Assert.assertEquals;
14+
import static org.junit.Assert.fail;
15+
16+
import java.util.List;
17+
18+
import org.junit.Test;
19+
import org.junit.runner.RunWith;
20+
import org.junit.runners.Parameterized;
21+
import org.junit.runners.Parameterized.AfterParam;
22+
import org.junit.runners.Parameterized.Parameter;
23+
import org.junit.runners.Parameterized.Parameters;
24+
25+
/**
26+
* @since 5.9
27+
*/
28+
@RunWith(Parameterized.class)
29+
public class ParameterizedWithAfterParamFailureTestCase {
30+
31+
@AfterParam
32+
public static void afterParam() {
33+
fail();
34+
}
35+
36+
@Parameters(name = "{0}")
37+
public static Iterable<String> parameters() {
38+
return List.of("foo", "bar");
39+
}
40+
41+
@Parameter
42+
public String value;
43+
44+
@Test
45+
public void test() {
46+
assertEquals("foo", value);
47+
}
48+
49+
}

‎junit-vintage-engine/src/testFixtures/java/org/junit/vintage/engine/samples/junit4/ParameterizedWithBeforeParamFailureTestCase.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright 2015-2022 the original author or authors.
3+
*
4+
* All rights reserved. This program and the accompanying materials are
5+
* made available under the terms of the Eclipse Public License v2.0 which
6+
* accompanies this distribution and is available at
7+
*
8+
* https://www.eclipse.org/legal/epl-v20.html
9+
*/
10+
11+
package org.junit.vintage.engine.samples.junit4;
12+
13+
import static org.junit.Assert.assertEquals;
14+
import static org.junit.Assert.fail;
15+
16+
import java.util.List;
17+
18+
import org.junit.Test;
19+
import org.junit.runner.RunWith;
20+
import org.junit.runners.Parameterized;
21+
import org.junit.runners.Parameterized.BeforeParam;
22+
import org.junit.runners.Parameterized.Parameter;
23+
import org.junit.runners.Parameterized.Parameters;
24+
25+
/**
26+
* @since 5.9
27+
*/
28+
@RunWith(Parameterized.class)
29+
public class ParameterizedWithBeforeParamFailureTestCase {
30+
31+
@BeforeParam
32+
public static void beforeParam() {
33+
fail();
34+
}
35+
36+
@Parameters(name = "{0}")
37+
public static Iterable<String> parameters() {
38+
return List.of("foo", "bar");
39+
}
40+
41+
@Parameter
42+
public String value;
43+
44+
@Test
45+
public void test() {
46+
assertEquals("foo", value);
47+
}
48+
49+
}

0 commit comments

Comments
 (0)