Skip to content

Commit b5b6f2b

Browse files
mpkorstanjempkorstanje
committed
[Core] Implement TeamCity output format plugin
This plugin inserts markers that can be picked up by IDEA and Teamcity. These in turn can then use this information to display a test tree and show the output grouped by scenario. This is plugin is nessesary because Intelij keeps using reflection to extract more information from the plugin system then what would normally be available. By substituting the CucumberJvm[1-5]SMFormatter with this plugin we avoid runtime errors. Unfortunately the teamcity format is poorly documented[1]. Reverse engingeering[2][3] the format from the plugin yields some information but the exact function is rather opaque. 1. https://confluence.jetbrains.com/display/TCD9/Build+Script+Interaction+with+TeamCity#BuildScriptInteractionwithTeamCity-MessageFlowId 2. https://github.com/JetBrains/intellij-community/tree/master/plugins/cucumber-jvm-formatter5/src/org/jetbrains/plugins/cucumber/java/run 3. https://github.com/mpkorstanje/intellij-community/blob/master/plugins/cucumber-jvm-formatter/src/org/jetbrains/plugins/cucumber/java/run
1 parent de3f234 commit b5b6f2b

File tree

20 files changed

+643
-70
lines changed

20 files changed

+643
-70
lines changed

‎core/src/main/java/io/cucumber/core/options/PluginOption.java

+10-8
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import io.cucumber.core.plugin.PrettyFormatter;
1313
import io.cucumber.core.plugin.ProgressFormatter;
1414
import io.cucumber.core.plugin.RerunFormatter;
15+
import io.cucumber.core.plugin.TeamCityPlugin;
1516
import io.cucumber.core.plugin.TestNGFormatter;
1617
import io.cucumber.core.plugin.TimelineFormatter;
1718
import io.cucumber.core.plugin.UnusedStepsSummaryPrinter;
@@ -44,15 +45,16 @@ public class PluginOption implements Options.Plugin {
4445
put("timeline", TimelineFormatter.class);
4546
put("unused", UnusedStepsSummaryPrinter.class);
4647
put("usage", UsageFormatter.class);
48+
put("teamcity", UsageFormatter.class);
4749
}};
4850

4951
// Refuse plugins known to implement the old API
50-
private static final HashMap<String, Class<? extends Plugin>> OLD_INTELLIJ_IDEA_PLUGIN_CLASSES = new HashMap<String, Class<? extends Plugin>>() {{
51-
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvmSMFormatter", PrettyFormatter.class);
52-
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm2SMFormatter", PrettyFormatter.class);
53-
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm3SMFormatter", PrettyFormatter.class);
54-
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm4SMFormatter", PrettyFormatter.class);
55-
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm5SMFormatter", PrettyFormatter.class);
52+
private static final HashMap<String, Class<? extends Plugin>> INCOMPATIBLE_INTELLIJ_IDEA_PLUGIN_CLASSES = new HashMap<String, Class<? extends Plugin>>() {{
53+
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvmSMFormatter", TeamCityPlugin.class);
54+
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm2SMFormatter", TeamCityPlugin.class);
55+
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm3SMFormatter", TeamCityPlugin.class);
56+
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm4SMFormatter", TeamCityPlugin.class);
57+
put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm5SMFormatter", TeamCityPlugin.class);
5658
}};
5759

5860
private final String pluginString;
@@ -76,9 +78,9 @@ public static PluginOption parse(String pluginArgumentPattern) {
7678
}
7779

7880
private static Class<? extends Plugin> parsePluginName(String pluginName) {
79-
Class<? extends Plugin> oldApiPlugin = OLD_INTELLIJ_IDEA_PLUGIN_CLASSES.get(pluginName);
81+
Class<? extends Plugin> oldApiPlugin = INCOMPATIBLE_INTELLIJ_IDEA_PLUGIN_CLASSES.get(pluginName);
8082
if (oldApiPlugin != null) {
81-
log.warn(() -> "Incompatible IntelliJ IDEA Plugin detected. Falling back to pretty formatter");
83+
log.debug(() -> "Incompatible IntelliJ IDEA Plugin detected. Falling back to teamcity plugin");
8284
return oldApiPlugin;
8385
}
8486

‎core/src/main/java/io/cucumber/core/plugin/TeamCityPlugin.java

+379
Large diffs are not rendered by default.

‎core/src/main/java/io/cucumber/core/runner/PickleStepDefinitionMatch.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ public String getPattern() {
157157
}
158158

159159
private StackTraceElement getStepLocation() {
160-
return new StackTraceElement("✽", step.getText(), uri.getSchemeSpecificPart(), step.getLine());
160+
return new StackTraceElement("✽", step.getText(), uri.toString(), step.getLine());
161161
}
162162

163163
StepDefinition getStepDefinition() {

‎core/src/main/java/io/cucumber/core/runner/Runner.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ private PickleStepDefinitionMatch matchStepToStepDefinition(Pickle pickle, Step
119119
}
120120
List<String> snippets = generateSnippetsForStep(step);
121121
if (!snippets.isEmpty()) {
122-
bus.send(new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), step.getLine(), snippets));
122+
bus.send(new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), pickle.getScenarioLocation().getLine(), step.getLine(), snippets));
123123
}
124124
return new UndefinedPickleStepDefinitionMatch(pickle.getUri(), step);
125125
} catch (AmbiguousStepDefinitionsException e) {

‎core/src/test/java/io/cucumber/core/plugin/CanonicalEventOrderTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ private static Event createTestCaseEvent(final URI uri, final int line) {
4343

4444
private Event runStarted = new TestRunStarted(getInstant());
4545
private Event testRead = new TestSourceRead(getInstant(), URI.create("file:path/to.feature"), "source");
46-
private Event suggested = new SnippetsSuggestedEvent(getInstant(), URI.create("file:path/to/1.feature"), 0, Collections.emptyList());
46+
private Event suggested = new SnippetsSuggestedEvent(getInstant(), URI.create("file:path/to/1.feature"), 0, 0, Collections.emptyList());
4747
private Event feature1Case1Started = createTestCaseEvent(URI.create("file:path/to/1.feature"), 1);
4848
private Event feature1Case2Started = createTestCaseEvent(URI.create("file:path/to/1.feature"), 9);
4949
private Event feature1Case3Started = createTestCaseEvent(URI.create("file:path/to/1.feature"), 11);

‎core/src/test/java/io/cucumber/core/plugin/TeamCityPluginTest.java

+140
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
package io.cucumber.core.plugin;
2+
3+
import io.cucumber.core.feature.TestFeatureParser;
4+
import io.cucumber.core.gherkin.Feature;
5+
import io.cucumber.core.runner.TestHelper;
6+
import io.cucumber.plugin.event.Result;
7+
import org.junit.jupiter.api.Test;
8+
import org.mockito.stubbing.Answer;
9+
10+
import java.io.ByteArrayOutputStream;
11+
import java.io.File;
12+
import java.io.PrintStream;
13+
import java.util.AbstractMap.SimpleEntry;
14+
import java.util.ArrayList;
15+
import java.util.HashMap;
16+
import java.util.List;
17+
import java.util.Map;
18+
19+
import static io.cucumber.core.runner.TestHelper.createWriteHookAction;
20+
import static io.cucumber.core.runner.TestHelper.result;
21+
import static java.nio.charset.StandardCharsets.UTF_8;
22+
import static org.hamcrest.CoreMatchers.containsString;
23+
import static org.hamcrest.MatcherAssert.assertThat;
24+
25+
class TeamCityPluginTest {
26+
27+
private final List<Feature> features = new ArrayList<>();
28+
private final Map<String, Result> stepsToResult = new HashMap<>();
29+
private final Map<String, String> stepsToLocation = new HashMap<>();
30+
private final List<SimpleEntry<String, Result>> hooks = new ArrayList<>();
31+
private final List<String> hookLocations = new ArrayList<>();
32+
private final List<Answer<Object>> hookActions = new ArrayList<>();
33+
private final String location = new File("").toURI().toString();
34+
35+
@Test
36+
void should_handle_scenario_outline() {
37+
Feature feature = TestFeatureParser.parse("path/test.feature", "" +
38+
"Feature: feature name\n" +
39+
" Scenario Outline: <name>\n" +
40+
" Given first step\n" +
41+
" Then <arg> step\n" +
42+
" Examples: examples name\n" +
43+
" | name | arg |\n" +
44+
" | name 1 | second |\n" +
45+
" | name 2 | third |\n");
46+
features.add(feature);
47+
stepsToLocation.put("first step", "path/step_definitions.java:3");
48+
stepsToLocation.put("second step", "path/step_definitions.java:7");
49+
stepsToLocation.put("third step", "path/step_definitions.java:11");
50+
51+
String formatterOutput = runFeaturesWithFormatter();
52+
53+
assertThat(formatterOutput, containsString("" +
54+
"##teamcity[enteredTheMatrix timestamp = '1970-01-01T12:00:00.000+0000']\n" +
55+
"##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' name = 'Cucumber']\n" +
56+
"##teamcity[customProgressStatus testsCategory = 'Scenarios' count = '0' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
57+
"##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:1' name = 'feature name']\n" +
58+
"##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:2' name = '<name>']\n" +
59+
"##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:5' name = 'examples name']\n" +
60+
"##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:7' name = 'Example #1']\n" +
61+
"##teamcity[customProgressStatus type = 'testStarted' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
62+
"##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:3' captureStandardOutput = 'true' name = 'first step']\n" +
63+
"##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'first step']\n" +
64+
"##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:4' captureStandardOutput = 'true' name = 'second step']\n" +
65+
"##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'second step']\n" +
66+
"##teamcity[customProgressStatus type = 'testFinished' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
67+
"##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'Example #1']\n" +
68+
"##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:8' name = 'Example #2']\n" +
69+
"##teamcity[customProgressStatus type = 'testStarted' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
70+
"##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:3' captureStandardOutput = 'true' name = 'first step']\n" +
71+
"##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'first step']\n" +
72+
"##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:4' captureStandardOutput = 'true' name = 'third step']\n" +
73+
"##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'third step']\n" +
74+
"##teamcity[customProgressStatus type = 'testFinished' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
75+
"##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'Example #2']\n" +
76+
"##teamcity[customProgressStatus testsCategory = '' count = '0' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
77+
"##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'examples name']\n" +
78+
"##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = '<name>']\n" +
79+
"##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'feature name']\n" +
80+
"##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'Cucumber']\n"
81+
));
82+
}
83+
84+
@Test
85+
void should_print_error_message_for_failed_steps() {
86+
Feature feature = TestFeatureParser.parse("path/test.feature", "" +
87+
"Feature: feature name\n" +
88+
" Scenario: scenario name\n" +
89+
" Given first step\n");
90+
features.add(feature);
91+
stepsToLocation.put("first step", "path/step_definitions.java:3");
92+
stepsToResult.put("first step", result("failed"));
93+
94+
String formatterOutput = runFeaturesWithFormatter();
95+
96+
assertThat(formatterOutput, containsString("" +
97+
"##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step failed' details = 'the stack trace' name = 'first step']\n"
98+
));
99+
}
100+
101+
@Test
102+
void should_print_error_message_for_before_hooks() {
103+
Feature feature = TestFeatureParser.parse("path/test.feature", "" +
104+
"Feature: feature name\n" +
105+
" Scenario: scenario name\n" +
106+
" Given first step\n");
107+
features.add(feature);
108+
stepsToLocation.put("first step", "path/step_definitions.java:3");
109+
stepsToResult.put("first step", result("passed"));
110+
hooks.add(TestHelper.hookEntry("before", result("failed")));
111+
hookLocations.add("HookDefinition.java:3");
112+
113+
String formatterOutput = runFeaturesWithFormatter();
114+
115+
assertThat(formatterOutput, containsString("" +
116+
"##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = 'java:test://HookDefinition.java:3' captureStandardOutput = 'true' name = 'Before']\n" +
117+
"##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step failed' details = 'the stack trace' name = 'Before']\n"
118+
));
119+
}
120+
121+
private String runFeaturesWithFormatter() {
122+
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
123+
PrintStream printStream = new PrintStream(byteArrayOutputStream);
124+
final TeamCityPlugin formatter = new TeamCityPlugin(printStream);
125+
126+
TestHelper.builder()
127+
.withFormatterUnderTest(formatter)
128+
.withFeatures(features)
129+
.withStepsToResult(stepsToResult)
130+
.withStepsToLocation(stepsToLocation)
131+
.withHooks(hooks)
132+
.withHookLocations(hookLocations)
133+
.withHookActions(hookActions)
134+
.build()
135+
.run();
136+
137+
return new String(byteArrayOutputStream.toByteArray(), UTF_8);
138+
}
139+
140+
}

‎gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesExamples.java

+11-5
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,28 @@
44
import io.cucumber.core.gherkin.Examples;
55
import io.cucumber.core.gherkin.Location;
66

7+
import java.util.Collection;
8+
import java.util.List;
79
import java.util.concurrent.atomic.AtomicInteger;
8-
import java.util.stream.Stream;
10+
import java.util.stream.Collectors;
911

1012
final class GherkinMessagesExamples implements Examples {
1113

1214
private final io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario.Examples examples;
15+
private final List<Example> children;
1316

1417
GherkinMessagesExamples(io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario.Examples examples) {
1518
this.examples = examples;
19+
20+
AtomicInteger row = new AtomicInteger(1);
21+
this.children = examples.getTableBodyList().stream()
22+
.map(tableRow -> new GherkinMessagesExample(tableRow, row.getAndIncrement()))
23+
.collect(Collectors.toList());
1624
}
1725

1826
@Override
19-
public Stream<Example> children() {
20-
AtomicInteger row = new AtomicInteger(1);
21-
return examples.getTableBodyList().stream()
22-
.map(tableRow -> new GherkinMessagesExample(tableRow, row.getAndIncrement()));
27+
public Collection<Example> children() {
28+
return children;
2329
}
2430

2531
@Override

‎gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesFeature.java

+12-9
Original file line numberDiff line numberDiff line change
@@ -3,49 +3,52 @@
33
import io.cucumber.core.gherkin.Feature;
44
import io.cucumber.core.gherkin.Located;
55
import io.cucumber.core.gherkin.Location;
6-
import io.cucumber.core.gherkin.Pickle;
76
import io.cucumber.core.gherkin.Node;
7+
import io.cucumber.core.gherkin.Pickle;
88
import io.cucumber.messages.Messages;
99
import io.cucumber.messages.Messages.GherkinDocument;
1010
import io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario;
1111

1212
import java.net.URI;
13+
import java.util.Collection;
1314
import java.util.List;
1415
import java.util.Objects;
1516
import java.util.Optional;
16-
import java.util.stream.Stream;
17+
import java.util.stream.Collectors;
1718

1819
final class GherkinMessagesFeature implements Feature {
1920
private final URI uri;
2021
private final List<Pickle> pickles;
2122
private final List<Messages.Envelope> envelopes;
2223
private final GherkinDocument gherkinDocument;
2324
private final String gherkinSource;
25+
private final List<Node> children;
2426

2527
GherkinMessagesFeature(GherkinDocument gherkinDocument, URI uri, String gherkinSource, List<Pickle> pickles, List<Messages.Envelope> envelopes) {
2628
this.gherkinDocument = gherkinDocument;
2729
this.uri = uri;
2830
this.gherkinSource = gherkinSource;
2931
this.pickles = pickles;
3032
this.envelopes = envelopes;
31-
}
32-
33-
@Override
34-
public Stream<Node> children() {
35-
return gherkinDocument.getFeature().getChildrenList().stream()
33+
this.children = gherkinDocument.getFeature().getChildrenList().stream()
3634
.filter(featureChild -> featureChild.hasRule() || featureChild.hasScenario())
3735
.map(featureChild -> {
3836
if (featureChild.hasRule()) {
3937
return new GherkinMessagesRule(featureChild.getRule());
4038
}
41-
4239
Scenario scenario = featureChild.getScenario();
4340
if (scenario.getExamplesCount() > 0) {
4441
return new GherkinMessagesScenarioOutline(scenario);
4542
} else {
4643
return new GherkinMessagesScenario(scenario);
4744
}
48-
});
45+
})
46+
.collect(Collectors.toList());
47+
}
48+
49+
@Override
50+
public Collection<Node> children() {
51+
return children;
4952
}
5053

5154
@Override

‎gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesRule.java

+13-8
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,23 @@
11
package io.cucumber.core.gherkin.messages;
22

33
import io.cucumber.core.gherkin.Location;
4-
import io.cucumber.core.gherkin.Rule;
54
import io.cucumber.core.gherkin.Node;
5+
import io.cucumber.core.gherkin.Rule;
66
import io.cucumber.messages.Messages;
77
import io.cucumber.messages.Messages.GherkinDocument.Feature.FeatureChild.RuleChild;
88

9-
import java.util.stream.Stream;
9+
import java.util.Collection;
10+
import java.util.List;
11+
import java.util.stream.Collectors;
1012

1113
final class GherkinMessagesRule implements Rule {
1214

1315
private final Messages.GherkinDocument.Feature.FeatureChild.Rule rule;
16+
private final List<Node> children;
1417

1518
GherkinMessagesRule(Messages.GherkinDocument.Feature.FeatureChild.Rule rule) {
1619
this.rule = rule;
17-
}
18-
19-
@Override
20-
public Stream<Node> children() {
21-
return rule.getChildrenList().stream()
20+
this.children = rule.getChildrenList().stream()
2221
.filter(RuleChild::hasScenario)
2322
.map(ruleChild -> {
2423
Messages.GherkinDocument.Feature.Scenario scenario = ruleChild.getScenario();
@@ -27,7 +26,13 @@ public Stream<Node> children() {
2726
} else {
2827
return new GherkinMessagesScenario(scenario);
2928
}
30-
});
29+
})
30+
.collect(Collectors.toList());
31+
}
32+
33+
@Override
34+
public Collection<Node> children() {
35+
return children;
3136
}
3237

3338
@Override

‎gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesScenarioOutline.java

+9-4
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,26 @@
55
import io.cucumber.core.gherkin.ScenarioOutline;
66
import io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario;
77

8-
import java.util.stream.Stream;
8+
import java.util.Collection;
9+
import java.util.List;
10+
import java.util.stream.Collectors;
911

1012
final class GherkinMessagesScenarioOutline implements ScenarioOutline {
1113

1214
private final Scenario scenario;
15+
private final List<Examples> children;
1316

1417
GherkinMessagesScenarioOutline(Scenario scenario) {
1518
this.scenario = scenario;
19+
this.children = scenario.getExamplesList().stream()
20+
.map(GherkinMessagesExamples::new)
21+
.collect(Collectors.toList());
1622
}
1723

1824

1925
@Override
20-
public Stream<Examples> children() {
21-
return scenario.getExamplesList().stream()
22-
.map(GherkinMessagesExamples::new);
26+
public Collection<Examples> children() {
27+
return children;
2328
}
2429

2530
@Override

0 commit comments

Comments
 (0)