Skip to content

Commit 2c25c9a

Browse files
mpkorstanjempkorstanje
committed
[Core] Fix JUnit report xml attributes
Cucumber JUnits xml format doesn't pass XSD validation in the Jenkins XUnit plugin because it is missing the errors attribute. Additionally time should be presented using numbers with both decimal and thousands separators. From: https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd Via: https://maven.apache.org/surefire/maven-surefire-plugin/ Closes #1777
1 parent 8ee4880 commit 2c25c9a

11 files changed

+335
-177
lines changed

‎core/pom.xml

+9-3
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
<properties>
1515
<project.Automatic-Module-Name>io.cucumber.core</project.Automatic-Module-Name>
1616
<jsoup.version>1.12.1</jsoup.version>
17-
<xmlunit.version>1.6</xmlunit.version>
17+
<xmlunit.version>2.6.3</xmlunit.version>
1818
<webbit.version>0.4.15</webbit.version>
1919
<webbit-rest.version>0.3.0</webbit-rest.version>
2020
<hamcrest-json.version>0.2</hamcrest-json.version>
@@ -51,8 +51,14 @@
5151
</dependency>
5252

5353
<dependency>
54-
<groupId>xmlunit</groupId>
55-
<artifactId>xmlunit</artifactId>
54+
<groupId>org.xmlunit</groupId>
55+
<artifactId>xmlunit-core</artifactId>
56+
<version>${xmlunit.version}</version>
57+
<scope>test</scope>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.xmlunit</groupId>
61+
<artifactId>xmlunit-matchers</artifactId>
5662
<version>${xmlunit.version}</version>
5763
<scope>test</scope>
5864
</dependency>

‎core/src/main/java/io/cucumber/core/plugin/JUnitFormatter.java

+55-51
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,20 @@
11
package io.cucumber.core.plugin;
22

3-
import io.cucumber.plugin.event.PickleStepTestStep;
3+
import io.cucumber.core.exception.CucumberException;
4+
import io.cucumber.plugin.EventListener;
5+
import io.cucumber.plugin.StrictAware;
46
import io.cucumber.plugin.event.EventPublisher;
7+
import io.cucumber.plugin.event.PickleStepTestStep;
58
import io.cucumber.plugin.event.Result;
69
import io.cucumber.plugin.event.Status;
710
import io.cucumber.plugin.event.TestCaseFinished;
811
import io.cucumber.plugin.event.TestCaseStarted;
912
import io.cucumber.plugin.event.TestRunFinished;
13+
import io.cucumber.plugin.event.TestRunStarted;
1014
import io.cucumber.plugin.event.TestSourceRead;
1115
import io.cucumber.plugin.event.TestStepFinished;
12-
import io.cucumber.core.exception.CucumberException;
13-
14-
import io.cucumber.plugin.EventListener;
15-
import io.cucumber.plugin.StrictAware;
1616
import org.w3c.dom.Document;
1717
import org.w3c.dom.Element;
18-
import org.w3c.dom.NodeList;
1918

2019
import javax.xml.XMLConstants;
2120
import javax.xml.parsers.DocumentBuilderFactory;
@@ -26,10 +25,6 @@
2625
import javax.xml.transform.TransformerFactory;
2726
import javax.xml.transform.dom.DOMSource;
2827
import javax.xml.transform.stream.StreamResult;
29-
30-
import static java.util.Locale.ROOT;
31-
import static java.util.concurrent.TimeUnit.SECONDS;
32-
3328
import java.io.Closeable;
3429
import java.io.IOException;
3530
import java.io.PrintWriter;
@@ -38,13 +33,18 @@
3833
import java.net.URL;
3934
import java.text.DecimalFormat;
4035
import java.text.NumberFormat;
36+
import java.time.Duration;
37+
import java.time.Instant;
4138
import java.util.ArrayList;
4239
import java.util.List;
4340
import java.util.Locale;
4441

42+
import static java.util.Locale.ROOT;
43+
import static java.util.concurrent.TimeUnit.SECONDS;
44+
4545
public final class JUnitFormatter implements EventListener, StrictAware {
4646

47-
private static final long NANOS_PER_SECONDS = SECONDS.toNanos(1L);
47+
private static final long MILLIS_PER_SECOND = SECONDS.toMillis(1L);
4848
private final Writer writer;
4949
private final Document document;
5050
private final Element rootElement;
@@ -55,6 +55,7 @@ public final class JUnitFormatter implements EventListener, StrictAware {
5555
private String currentFeatureFile = null;
5656
private String previousTestCaseName;
5757
private int exampleNumber;
58+
private Instant started;
5859

5960
@SuppressWarnings("WeakerAccess") // Used by plugin factory
6061
public JUnitFormatter(URL writer) throws IOException {
@@ -72,13 +73,24 @@ private static String getUniqueTestNameForScenarioExample(String testCaseName, i
7273
return testCaseName + (testCaseName.contains(" ") ? " " : "_") + exampleNumber;
7374
}
7475

76+
private static String calculateTotalDurationString(Duration result) {
77+
DecimalFormat numberFormat = (DecimalFormat) NumberFormat.getNumberInstance(Locale.US);
78+
double duration = (double) result.toMillis() / MILLIS_PER_SECOND;
79+
return numberFormat.format(duration);
80+
}
81+
7582
@Override
7683
public void setEventPublisher(EventPublisher publisher) {
84+
publisher.registerHandlerFor(TestRunStarted.class, this::handleTestRunStarted);
7785
publisher.registerHandlerFor(TestSourceRead.class, this::handleTestSourceRead);
7886
publisher.registerHandlerFor(TestCaseStarted.class, this::handleTestCaseStarted);
7987
publisher.registerHandlerFor(TestCaseFinished.class, this::handleTestCaseFinished);
8088
publisher.registerHandlerFor(TestStepFinished.class, this::handleTestStepFinished);
81-
publisher.registerHandlerFor(TestRunFinished.class, event -> finishReport());
89+
publisher.registerHandlerFor(TestRunFinished.class, this::handleTestRunFinished);
90+
}
91+
92+
private void handleTestRunStarted(TestRunStarted event) {
93+
this.started = event.getInstant();
8294
}
8395

8496
@Override
@@ -101,7 +113,7 @@ private void handleTestCaseStarted(TestCaseStarted event) {
101113
testCase.writeElement(root);
102114
rootElement.appendChild(root);
103115

104-
increaseAttributeValue(rootElement, "tests");
116+
increaseTestCount(rootElement);
105117
}
106118

107119
private void handleTestStepFinished(TestStepFinished event) {
@@ -119,13 +131,15 @@ private void handleTestCaseFinished(TestCaseFinished event) {
119131
}
120132
}
121133

122-
private void finishReport() {
134+
private void handleTestRunFinished(TestRunFinished event) {
123135
try {
136+
Instant finished = event.getInstant();
124137
// set up a transformer
125138
rootElement.setAttribute("name", JUnitFormatter.class.getName());
126139
rootElement.setAttribute("failures", String.valueOf(rootElement.getElementsByTagName("failure").getLength()));
127140
rootElement.setAttribute("skipped", String.valueOf(rootElement.getElementsByTagName("skipped").getLength()));
128-
rootElement.setAttribute("time", getTotalDuration(rootElement.getElementsByTagName("testcase")));
141+
rootElement.setAttribute("errors", "0");
142+
rootElement.setAttribute("time", calculateTotalDurationString(Duration.between(started, finished)));
129143

130144
TransformerFactory factory = TransformerFactory.newInstance();
131145
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
@@ -148,28 +162,12 @@ private void closeQuietly(Closeable out) {
148162
}
149163
}
150164

151-
private String getTotalDuration(NodeList testCaseNodes) {
152-
double totalDurationSecondsForAllTimes = 0.0d;
153-
for (int i = 0; i < testCaseNodes.getLength(); i++) {
154-
try {
155-
double testCaseTime =
156-
Double.parseDouble(testCaseNodes.item(i).getAttributes().getNamedItem("time").getNodeValue());
157-
totalDurationSecondsForAllTimes += testCaseTime;
158-
} catch (NumberFormatException | NullPointerException e) {
159-
throw new CucumberException(e);
160-
}
161-
}
162-
DecimalFormat nfmt = (DecimalFormat) NumberFormat.getNumberInstance(Locale.US);
163-
nfmt.applyPattern("0.######");
164-
return nfmt.format(totalDurationSecondsForAllTimes);
165-
}
166-
167-
private void increaseAttributeValue(Element element, String attribute) {
165+
private void increaseTestCount(Element element) {
168166
int value = 0;
169-
if (element.hasAttribute(attribute)) {
170-
value = Integer.parseInt(element.getAttribute(attribute));
167+
if (element.hasAttribute("tests")) {
168+
value = Integer.parseInt(element.getAttribute("tests"));
171169
}
172-
element.setAttribute(attribute, String.valueOf(++value));
170+
element.setAttribute("tests", String.valueOf(++value));
173171
}
174172

175173
final class TestCase {
@@ -203,24 +201,25 @@ private String calculateElementName(io.cucumber.plugin.event.TestCase testCase)
203201
}
204202

205203
void addTestCaseElement(Document doc, Element tc, Result result) {
206-
tc.setAttribute("time", calculateTotalDurationString(result));
204+
tc.setAttribute("time", calculateTotalDurationString(result.getDuration()));
207205

208206
StringBuilder sb = new StringBuilder();
209207
addStepAndResultListing(sb);
210208
Element child;
211209
Status status = result.getStatus();
212210
if (status.is(Status.FAILED) || status.is(Status.AMBIGUOUS)) {
213211
addStackTrace(sb, result);
214-
child = createElementWithMessage(doc, sb, "failure", printStackTrace(result.getError()));
212+
child = createFailure(doc, sb, result.getError().getMessage(), result.getError().getClass());
215213
} else if (status.is(Status.PENDING) || status.is(Status.UNDEFINED)) {
216214
if (strict) {
217-
child = createElementWithMessage(doc, sb, "failure", "The scenario has pending or undefined step(s)");
215+
Throwable error = result.getError();
216+
child = createFailure(doc, sb, "The scenario has pending or undefined step(s)", error == null ? Exception.class : error.getClass());
218217
} else {
219218
child = createElement(doc, sb, "skipped");
220219
}
221220
} else if (status.is(Status.SKIPPED) && result.getError() != null) {
222221
addStackTrace(sb, result);
223-
child = createElementWithMessage(doc, sb, "skipped", printStackTrace(result.getError()));
222+
child = createSkipped(doc, sb, printStackTrace(result.getError()));
224223
} else {
225224
child = createElement(doc, sb, "system-out");
226225
}
@@ -229,20 +228,18 @@ void addTestCaseElement(Document doc, Element tc, Result result) {
229228
}
230229

231230
void handleEmptyTestCase(Document doc, Element tc, Result result) {
232-
tc.setAttribute("time", calculateTotalDurationString(result));
231+
tc.setAttribute("time", calculateTotalDurationString(result.getDuration()));
233232

234-
String resultType = strict ? "failure" : "skipped";
235-
Element child = createElementWithMessage(doc, new StringBuilder(), resultType, "The scenario has no steps");
233+
Element child;
234+
if (strict) {
235+
child = createFailure(doc, new StringBuilder(), "The scenario has no steps", Exception.class);
236+
} else {
237+
child = createSkipped(doc, new StringBuilder(), "The scenario has no steps");
238+
}
236239

237240
tc.appendChild(child);
238241
}
239242

240-
private String calculateTotalDurationString(Result result) {
241-
DecimalFormat numberFormat = (DecimalFormat) NumberFormat.getNumberInstance(Locale.US);
242-
numberFormat.applyPattern("0.######");
243-
return numberFormat.format(((double) result.getDuration().toNanos() / NANOS_PER_SECONDS));
244-
}
245-
246243
private void addStepAndResultListing(StringBuilder sb) {
247244
for (int i = 0; i < steps.size(); i++) {
248245
int length = sb.length();
@@ -275,9 +272,16 @@ private String printStackTrace(Throwable error) {
275272
return stringWriter.toString();
276273
}
277274

278-
private Element createElementWithMessage(Document doc, StringBuilder sb, String elementType, String message) {
279-
Element child = createElement(doc, sb, elementType);
275+
private Element createSkipped(Document doc, StringBuilder sb, String message) {
276+
Element child = createElement(doc, sb, "skipped");
277+
child.setAttribute("message", message);
278+
return child;
279+
}
280+
281+
private Element createFailure(Document doc, StringBuilder sb, String message, Class<? extends Throwable> type) {
282+
Element child = createElement(doc, sb, "failure");
280283
child.setAttribute("message", message);
284+
child.setAttribute("type", type.getName());
281285
return child;
282286
}
283287

@@ -290,6 +294,6 @@ private Element createElement(Document doc, StringBuilder sb, String elementType
290294
child.appendChild(doc.createCDATASection(normalizedLineEndings));
291295
return child;
292296
}
293-
294297
}
298+
295299
}

‎core/src/main/java/io/cucumber/core/plugin/TestNGFormatter.java

+21-26
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import io.cucumber.plugin.event.TestCaseFinished;
88
import io.cucumber.plugin.event.TestCaseStarted;
99
import io.cucumber.plugin.event.TestRunFinished;
10+
import io.cucumber.plugin.event.TestRunStarted;
1011
import io.cucumber.plugin.event.TestSourceRead;
1112
import io.cucumber.plugin.event.TestStepFinished;
1213
import io.cucumber.core.exception.CucumberException;
@@ -33,13 +34,13 @@
3334
import java.io.StringWriter;
3435
import java.io.Writer;
3536
import java.net.URL;
36-
import java.text.SimpleDateFormat;
3737
import java.time.Duration;
38+
import java.time.Instant;
3839
import java.util.ArrayList;
39-
import java.util.Date;
4040
import java.util.List;
4141

4242
import static java.time.Duration.ZERO;
43+
import static java.time.format.DateTimeFormatter.ISO_INSTANT;
4344
import static java.util.Locale.ROOT;
4445

4546
public final class TestNGFormatter implements EventListener, StrictAware {
@@ -57,6 +58,7 @@ public final class TestNGFormatter implements EventListener, StrictAware {
5758
private String currentFeatureFile = null;
5859
private String previousTestCaseName;
5960
private int exampleNumber;
61+
private Instant started;
6062

6163
@SuppressWarnings("WeakerAccess") // Used by plugin factory
6264
public TestNGFormatter(URL url) throws IOException {
@@ -77,10 +79,15 @@ public TestNGFormatter(URL url) throws IOException {
7779
@Override
7880
public void setEventPublisher(EventPublisher publisher) {
7981
publisher.registerHandlerFor(TestSourceRead.class, this::handleTestSourceRead);
82+
publisher.registerHandlerFor(TestRunStarted.class, this::handleTestRunStarted);
8083
publisher.registerHandlerFor(TestCaseStarted.class, this::handleTestCaseStarted);
8184
publisher.registerHandlerFor(TestCaseFinished.class, this::handleTestCaseFinished);
8285
publisher.registerHandlerFor(TestStepFinished.class, this::handleTestStepFinished);
83-
publisher.registerHandlerFor(TestRunFinished.class, event -> finishReport());
86+
publisher.registerHandlerFor(TestRunFinished.class, this::handleTestRunFinished);
87+
}
88+
89+
private void handleTestRunStarted(TestRunStarted event) {
90+
this.started = event.getInstant();
8491
}
8592

8693
@Override
@@ -104,7 +111,7 @@ private void handleTestCaseStarted(TestCaseStarted event) {
104111
root = document.createElement("test-method");
105112
clazz.appendChild(root);
106113
testCase = new TestCase(event.getTestCase());
107-
testCase.start(root);
114+
testCase.start(root, event.getInstant());
108115
}
109116

110117
private void handleTestStepFinished(TestStepFinished event) {
@@ -117,19 +124,21 @@ private void handleTestStepFinished(TestStepFinished event) {
117124
}
118125

119126
private void handleTestCaseFinished(TestCaseFinished event) {
120-
testCase.finish(document, root);
127+
testCase.finish(document, root, event.getInstant());
121128
}
122129

123-
private void finishReport() {
130+
private void handleTestRunFinished(TestRunFinished event) {
124131
try {
132+
Instant finished = event.getInstant();
133+
Duration duration = Duration.between(started, finished);
125134
results.setAttribute("total", String.valueOf(getElementsCountByAttribute(suite, "status", ".*")));
126135
results.setAttribute("passed", String.valueOf(getElementsCountByAttribute(suite, "status", "PASS")));
127136
results.setAttribute("failed", String.valueOf(getElementsCountByAttribute(suite, "status", "FAIL")));
128137
results.setAttribute("skipped", String.valueOf(getElementsCountByAttribute(suite, "status", "SKIP")));
129138
suite.setAttribute("name", TestNGFormatter.class.getName());
130-
suite.setAttribute("duration-ms", getTotalDuration(suite.getElementsByTagName("test-method")));
139+
suite.setAttribute("duration-ms", String.valueOf(duration.toMillis()));
131140
test.setAttribute("name", TestNGFormatter.class.getName());
132-
test.setAttribute("duration-ms", getTotalDuration(suite.getElementsByTagName("test-method")));
141+
test.setAttribute("duration-ms", String.valueOf(duration.toMillis()));
133142

134143
TransformerFactory factory = TransformerFactory.newInstance();
135144
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
@@ -170,34 +179,20 @@ private int getElementsCountByAttribute(Node node, String attributeName, String
170179
return count;
171180
}
172181

173-
private String getTotalDuration(NodeList testCaseNodes) {
174-
long totalDuration = 0;
175-
for (int i = 0; i < testCaseNodes.getLength(); i++) {
176-
try {
177-
String duration = testCaseNodes.item(i).getAttributes().getNamedItem("duration-ms").getNodeValue();
178-
totalDuration += Long.parseLong(duration);
179-
} catch (NumberFormatException | NullPointerException e) {
180-
throw new CucumberException(e);
181-
}
182-
}
183-
return String.valueOf(totalDuration);
184-
}
185-
186182
final class TestCase {
187183

188184
private final List<PickleStepTestStep> steps = new ArrayList<>();
189185
private final List<Result> results = new ArrayList<>();
190186
private final List<Result> hooks = new ArrayList<>();
191187
private final io.cucumber.plugin.event.TestCase testCase;
192-
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
193188

194189
TestCase(io.cucumber.plugin.event.TestCase testCase) {
195190
this.testCase = testCase;
196191
}
197192

198-
private void start(Element element) {
193+
private void start(Element element, Instant instant) {
199194
element.setAttribute("name", calculateElementName(testCase));
200-
element.setAttribute("started-at", dateFormat.format(new Date()));
195+
element.setAttribute("started-at", ISO_INSTANT.format(instant));
201196
}
202197

203198
private String calculateElementName(io.cucumber.plugin.event.TestCase testCase) {
@@ -211,9 +206,9 @@ private String calculateElementName(io.cucumber.plugin.event.TestCase testCase)
211206
}
212207
}
213208

214-
void finish(Document doc, Element element) {
209+
void finish(Document doc, Element element, Instant instant) {
215210
element.setAttribute("duration-ms", calculateTotalDurationString());
216-
element.setAttribute("finished-at", dateFormat.format(new Date()));
211+
element.setAttribute("finished-at", ISO_INSTANT.format(instant));
217212
StringBuilder stringBuilder = new StringBuilder();
218213
addStepAndResultListing(stringBuilder);
219214
Result skipped = null;

0 commit comments

Comments
 (0)