Skip to content

Commit 9779d26

Browse files
davidjgossdavidjgoss
authored
feat: include exceptions in test step result messages (#2229)
1 parent 88eb48d commit 9779d26

12 files changed

+183
-39
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
1010
## [Unreleased]
1111
### Added
1212
- Affirm support for Node.js 19 [#2230](https://github.com/cucumber/cucumber-js/pull/2230)
13+
- Include some exception details in the result of a test step for downstream tools [#2229](https://github.com/cucumber/cucumber-js/pull/2229)
1314

1415
### Fixed
1516
- Handle invalid characters when generating XML for JUnit formatter [#2228](https://github.com/cucumber/cucumber-js/pull/2228)

‎compatibility/features/cdata/cdata.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import assert from 'assert'
2+
import { Given } from '../../../src'
3+
4+
Given(
5+
'I have {int} <![CDATA[cukes]]> in my belly',
6+
function (cukeCount: number) {
7+
assert(cukeCount)
8+
}
9+
)

‎features/fixtures/formatters/failed.message.json.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,10 @@ module.exports = [
142142
},
143143
status: 'FAILED',
144144
message: 'Error: my error',
145+
exception: {
146+
type: 'Error',
147+
message: 'my error',
148+
},
145149
},
146150
timestamp: {
147151
seconds: 0,

‎features/fixtures/formatters/retried.message.json.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,10 @@ module.exports = [
142142
},
143143
status: 'FAILED',
144144
message: 'Error: my error',
145+
exception: {
146+
type: 'Error',
147+
message: 'my error',
148+
},
145149
},
146150
timestamp: {
147151
seconds: 0,

‎package-lock.json

Lines changed: 7 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@
237237
"yup": "^0.32.11"
238238
},
239239
"devDependencies": {
240-
"@cucumber/compatibility-kit": "11.0.1",
240+
"@cucumber/compatibility-kit": "11.2.0",
241241
"@cucumber/query": "12.0.1",
242242
"@microsoft/api-documenter": "7.19.27",
243243
"@microsoft/api-extractor": "7.33.7",

‎src/formatter/junit_formatter.ts

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,15 @@ interface IJUnitTestCase {
3636
steps: IJUnitTestStep[]
3737
}
3838

39+
interface IJUnitTestCaseFailure {
40+
type: string
41+
message?: string
42+
detail: string
43+
}
44+
3945
interface IJUnitTestCaseResult {
4046
status: TestStepResultStatus
41-
message?: string
47+
failure?: IJUnitTestCaseFailure
4248
}
4349

4450
interface IJUnitTestStep {
@@ -60,16 +66,6 @@ interface IBuildJUnitTestStepOptions {
6066
testStepResult: messages.TestStepResult
6167
}
6268

63-
const statusDescriptions: Record<TestStepResultStatus, string> = {
64-
UNKNOWN: `A result couldn't be established`,
65-
PASSED: 'Everything went fine',
66-
SKIPPED: 'The test case was skipped',
67-
PENDING: 'A step in the test case is not yet implemented',
68-
UNDEFINED: 'A step in the test case is not defined',
69-
AMBIGUOUS: 'Multiple definitions match one of the steps in the test case',
70-
FAILED: 'A hook or step failed',
71-
}
72-
7369
export default class JunitFormatter extends Formatter {
7470
private readonly names: Record<string, string[]> = {}
7571
public static readonly documentation: string = 'Outputs JUnit report'
@@ -134,8 +130,20 @@ export default class JunitFormatter extends Formatter {
134130
}
135131

136132
private getTestCaseResult(steps: IJUnitTestStep[]): IJUnitTestCaseResult {
137-
const worstResult = getWorstTestStepResult(steps.map((step) => step.result))
138-
return worstResult
133+
const { status, message, exception } = getWorstTestStepResult(
134+
steps.map((step) => step.result)
135+
)
136+
return {
137+
status,
138+
failure:
139+
message || exception
140+
? {
141+
type: exception?.type,
142+
message: exception?.message,
143+
detail: message,
144+
}
145+
: undefined,
146+
}
139147
}
140148

141149
private durationToSeconds(duration: Duration): number {
@@ -262,11 +270,11 @@ export default class JunitFormatter extends Formatter {
262270
xmlTestCase.ele('skipped')
263271
} else if (test.result.status !== TestStepResultStatus.PASSED) {
264272
const xmlFailure = xmlTestCase.ele('failure', {
265-
type: test.result.status,
266-
message: statusDescriptions[test.result.status],
273+
type: test.result.failure?.type,
274+
message: test.result.failure?.message,
267275
})
268-
if (test.result.message) {
269-
xmlFailure.cdata(test.result.message)
276+
if (test.result?.failure) {
277+
xmlFailure.cdata(test.result.failure.detail)
270278
}
271279
}
272280
xmlTestCase.ele('system-out', {}).cdata(test.systemOutput)

‎src/formatter/junit_formatter_spec.ts

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ function getJUnitFormatterSupportCodeLibrary(
2020
clock.tick(1)
2121
})
2222

23+
Given('I have <![CDATA[cukes]]> in my belly', function () {
24+
clock.tick(1)
25+
})
26+
2327
let willPass = false
2428
Given('a flaky step', function () {
2529
clock.tick(1)
@@ -145,7 +149,7 @@ describe('JunitFormatter', () => {
145149
'<?xml version="1.0"?>\n' +
146150
'<testsuite failures="1" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
147151
' <testcase classname="my feature" name="my scenario" time="0.001">\n' +
148-
' <failure type="FAILED" message="A hook or step failed"><![CDATA[error]]></failure>\n' +
152+
' <failure type="Error" message="error"><![CDATA[error]]></failure>\n' +
149153
' <system-out><![CDATA[Given a failing step......................................................failed]]></system-out>\n' +
150154
' </testcase>\n' +
151155
'</testsuite>'
@@ -179,7 +183,7 @@ describe('JunitFormatter', () => {
179183
'<?xml version="1.0"?>\n' +
180184
'<testsuite failures="1" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
181185
' <testcase classname="my feature" name="my scenario" time="0.001">\n' +
182-
' <failure type="FAILED" message="A hook or step failed"><![CDATA[Error: include invalid character]]></failure>\n' +
186+
' <failure type="Error" message="Error: include invalid character"><![CDATA[Error: include invalid character]]></failure>\n' +
183187
' <system-out><![CDATA[Given a failing step with invalid character...............................failed]]></system-out>\n' +
184188
' </testcase>\n' +
185189
'</testsuite>'
@@ -250,7 +254,7 @@ describe('JunitFormatter', () => {
250254
'<?xml version="1.0"?>\n' +
251255
'<testsuite failures="1" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
252256
' <testcase classname="my feature" name="my scenario" time="0.001">\n' +
253-
' <failure type="PENDING" message="A step in the test case is not yet implemented"/>\n' +
257+
' <failure/>\n' +
254258
' <system-out><![CDATA[Given a pending step.....................................................pending]]></system-out>\n' +
255259
' </testcase>\n' +
256260
'</testsuite>'
@@ -322,7 +326,7 @@ describe('JunitFormatter', () => {
322326
'<?xml version="1.0"?>\n' +
323327
'<testsuite failures="1" skipped="0" name="cucumber-js" time="0" tests="1">\n' +
324328
' <testcase classname="my feature" name="my scenario" time="0">\n' +
325-
' <failure type="UNDEFINED" message="A step in the test case is not defined"/>\n' +
329+
' <failure/>\n' +
326330
' <system-out><![CDATA[Given a passing step...................................................undefined]]></system-out>\n' +
327331
' </testcase>\n' +
328332
'</testsuite>'
@@ -409,7 +413,7 @@ describe('JunitFormatter', () => {
409413
' <system-out><![CDATA[Given a passing step......................................................passed]]></system-out>\n' +
410414
' </testcase>\n' +
411415
' <testcase classname="my feature" name="my templated scenario [1]" time="0.001">\n' +
412-
' <failure type="FAILED" message="A hook or step failed"><![CDATA[error]]></failure>\n' +
416+
' <failure type="Error" message="error"><![CDATA[error]]></failure>\n' +
413417
' <system-out><![CDATA[Given a failing step......................................................failed]]></system-out>\n' +
414418
' </testcase>\n' +
415419
'</testsuite>'
@@ -518,4 +522,43 @@ describe('JunitFormatter', () => {
518522
)
519523
})
520524
})
525+
526+
describe('content containing CDATA', () => {
527+
it('outputs the feature', async () => {
528+
// Arrange
529+
const sources = [
530+
{
531+
data: [
532+
'Feature: my feature',
533+
' my feature description',
534+
'',
535+
' Scenario: my scenario',
536+
' my scenario description',
537+
'',
538+
' Given I have <![CDATA[cukes]]> in my belly',
539+
].join('\n'),
540+
uri: 'a.feature',
541+
},
542+
]
543+
544+
const supportCodeLibrary = getJUnitFormatterSupportCodeLibrary(clock)
545+
546+
// Act
547+
const output = await testFormatter({
548+
sources,
549+
supportCodeLibrary,
550+
type: 'junit',
551+
})
552+
553+
// Assert
554+
expect(output).xml.to.deep.equal(
555+
'<?xml version="1.0"?>\n' +
556+
'<testsuite failures="0" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
557+
' <testcase classname="my feature" name="my scenario" time="0.001">\n' +
558+
' <system-out><![CDATA[Given I have <![CDATA[cukes]]]]><![CDATA[> in my belly................................passed]]></system-out>\n' +
559+
' </testcase>\n' +
560+
'</testsuite>'
561+
)
562+
})
563+
})
521564
})

‎src/runtime/format_error.ts

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
1+
import { TestStepResult } from '@cucumber/messages'
12
import { format } from 'assertion-error-formatter'
23
import errorStackParser from 'error-stack-parser'
34
import { filterStackTrace } from '../filter_stack_trace'
45

5-
export function formatError(error: Error, filterStackTraces: boolean): string {
6+
export function formatError(
7+
error: Error,
8+
filterStackTraces: boolean
9+
): Pick<TestStepResult, 'message' | 'exception'> {
610
let filteredStack: string
711
if (filterStackTraces) {
812
try {
@@ -13,10 +17,17 @@ export function formatError(error: Error, filterStackTraces: boolean): string {
1317
// if we weren't able to parse and filter, we'll settle for the original
1418
}
1519
}
16-
return format(error, {
20+
const message = format(error, {
1721
colorFns: {
1822
errorStack: (stack: string) =>
1923
filteredStack ? `\n${filteredStack}` : stack,
2024
},
2125
})
26+
return {
27+
message,
28+
exception: {
29+
type: error.name || 'Error',
30+
message: typeof error === 'string' ? error : error.message,
31+
},
32+
}
2233
}

‎src/runtime/format_error_spec.ts

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { expect } from 'chai'
2+
import assert from 'assert'
3+
import { formatError } from './format_error'
4+
5+
describe('formatError', () => {
6+
function testFormatError(fn: () => void, filterStackTraces: boolean = false) {
7+
try {
8+
fn()
9+
return undefined
10+
} catch (error) {
11+
return formatError(error, filterStackTraces)
12+
}
13+
}
14+
15+
it('should handle a custom error', () => {
16+
expect(
17+
testFormatError(() => {
18+
assert.ok(false, 'Thing that should have been truthy was falsy!')
19+
}).exception
20+
).to.eql({
21+
type: 'AssertionError',
22+
message: 'Thing that should have been truthy was falsy!',
23+
})
24+
})
25+
26+
it('should handle a generic error', () => {
27+
expect(
28+
testFormatError(() => {
29+
throw new Error('A generally bad thing happened!')
30+
}).exception
31+
).to.eql({
32+
type: 'Error',
33+
message: 'A generally bad thing happened!',
34+
})
35+
})
36+
37+
it('should handle an omitted message', () => {
38+
expect(
39+
testFormatError(() => {
40+
throw new Error()
41+
}).exception
42+
).to.eql({
43+
type: 'Error',
44+
message: '',
45+
})
46+
})
47+
48+
it('should handle a thrown string', () => {
49+
expect(
50+
testFormatError(() => {
51+
throw 'Yikes!'
52+
}).exception
53+
).to.eql({
54+
type: 'Error',
55+
message: 'Yikes!',
56+
})
57+
})
58+
})

‎src/runtime/step_runner.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,13 @@ export async function run({
6262

6363
const duration = stopwatch.stop().duration()
6464
let status: messages.TestStepResultStatus
65-
let message: string
65+
let details = {}
6666
if (result === 'skipped') {
6767
status = messages.TestStepResultStatus.SKIPPED
6868
} else if (result === 'pending') {
6969
status = messages.TestStepResultStatus.PENDING
7070
} else if (doesHaveValue(error)) {
71-
message = formatError(error, filterStackTraces)
71+
details = formatError(error, filterStackTraces)
7272
status = messages.TestStepResultStatus.FAILED
7373
} else {
7474
status = messages.TestStepResultStatus.PASSED
@@ -77,7 +77,7 @@ export async function run({
7777
return {
7878
duration,
7979
status,
80-
message,
80+
...details,
8181
}
8282
}
8383

0 commit comments

Comments
 (0)