Skip to content

Associate comment details on #expect w/ test issue #1025

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 26, 2024
Merged

Associate comment details on #expect w/ test issue #1025

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 122 additions & 18 deletions src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ export enum TestSymbol {
difference = "difference",
warning = "warning",
details = "details",
none = "none",
}

export interface EventMessage {
Expand Down Expand Up @@ -222,9 +223,29 @@ export class SwiftTestingOutputParser {
* @param chunk A chunk of stdout emitted during a test run.
*/
public parseStdout = (() => {
const values = Object.values(TestSymbol).map(symbol =>
regexEscapedString(SymbolRenderer.eventMessageSymbol(symbol))
);
const values = [
...Object.values(TestSymbol)
.filter(symbol => symbol !== TestSymbol.none)
.map(symbol =>
regexEscapedString(
// Trim the ANSI reset code from the search since some lines
// are fully colorized from the symbol to the end of line.
SymbolRenderer.eventMessageSymbol(symbol).replace(
SymbolRenderer.resetANSIEscapeCode,
""
)
)
),
// It is possible there is no symbol for a line produced by swift-testing,
// for instance if the user has a multi line comment before a failing expectation
// only the first line of the printed comment will have a symbol, but to make the
// indentation consistent the subsequent lines will have three spaces. We don't want
// to treat this as output produced by the user during the test run, so omit these.
// This isn't ideal since this will swallow lines the user prints if they start with
// three spaces, but until we have user output as part of the JSON event stream we have
// this workaround.
" ",
];

// Build a regex of all the line beginnings that come out of swift-testing events.
const isSwiftTestingLineBeginning = new RegExp(`^${values.join("|")}`);
Expand Down Expand Up @@ -313,13 +334,47 @@ export class SwiftTestingOutputParser {
testIndex: number | undefined
) {
messages.forEach(message => {
runState.recordOutput(
testIndex,
`${SymbolRenderer.eventMessageSymbol(message.symbol)} ${message.text}\r\n`
);
runState.recordOutput(testIndex, `${MessageRenderer.render(message)}\r\n`);
});
}

/**
* Partitions a collection of messages in to issues and details about the issues.
* This is used to print the issues first, followed by the details.
*/
private partitionIssueMessages(messages: EventMessage[]): {
issues: EventMessage[];
details: EventMessage[];
} {
return messages.reduce(
(buckets, message) => {
const key =
message.symbol === "details" ||
message.symbol === "default" ||
message.symbol === "none"
? "details"
: "issues";
return { ...buckets, [key]: [...buckets[key], message] };
},
{
issues: [],
details: [],
}
);
}

/*
* A multi line comment preceeding an issue will have a 'default' symbol for
* all lines except the first one. To match the swift-testing command line we
* should show no symbol on these lines.
*/
private transformIssueMessageSymbols(messages: EventMessage[]): EventMessage[] {
return messages.map(message => ({
...message,
symbol: message.symbol === "default" ? TestSymbol.none : message.symbol,
}));
}

private parse(item: SwiftTestEvent, runState: ITestRunState) {
if (
item.kind === "test" &&
Expand Down Expand Up @@ -391,14 +446,31 @@ export class SwiftTestingOutputParser {
sourceLocation.line,
sourceLocation.column
);
item.payload.messages.forEach(message => {
runState.recordIssue(testIndex, message.text, isKnown, location);

const messages = this.transformIssueMessageSymbols(item.payload.messages);
const { issues, details } = this.partitionIssueMessages(messages);

// Order the details after the issue text.
const additionalDetails = details
.map(message => MessageRenderer.render(message))
.join("\n");

issues.forEach(message => {
runState.recordIssue(
testIndex,
additionalDetails.length > 0
? `${MessageRenderer.render(message)}\n${additionalDetails}`
: MessageRenderer.render(message),
isKnown,
location
);
});
this.recordOutput(runState, item.payload.messages, testIndex);

this.recordOutput(runState, messages, testIndex);

if (item.payload._testCase && testID !== item.payload.testID) {
const testIndex = this.getTestCaseIndex(runState, item.payload.testID);
item.payload.messages.forEach(message => {
messages.forEach(message => {
runState.recordIssue(testIndex, message.text, isKnown, location);
});
}
Expand Down Expand Up @@ -439,6 +511,32 @@ export class SwiftTestingOutputParser {
}
}

export class MessageRenderer {
/**
* Converts a swift-testing `EventMessage` to a colorized symbol and message text.
*
* @param message An event message, typically found on an `EventRecordPayload`.
* @returns A string colorized with ANSI escape codes.
*/
static render(message: EventMessage): string {
return `${SymbolRenderer.eventMessageSymbol(message.symbol)} ${MessageRenderer.colorize(message.symbol, message.text)}`;
}

private static colorize(symbolType: TestSymbol, message: string): string {
const ansiEscapeCodePrefix = "\u{001B}[";
const resetANSIEscapeCode = `${ansiEscapeCodePrefix}0m`;
switch (symbolType) {
case TestSymbol.details:
case TestSymbol.skip:
case TestSymbol.difference:
case TestSymbol.passWithKnownIssue:
return `${ansiEscapeCodePrefix}90m${message}${resetANSIEscapeCode}`;
default:
return message;
}
}
}

export class SymbolRenderer {
/**
* Converts a swift-testing symbol identifier in to a colorized unicode symbol.
Expand All @@ -450,6 +548,9 @@ export class SymbolRenderer {
return this.colorize(symbol, this.symbol(symbol));
}

static ansiEscapeCodePrefix = "\u{001B}[";
static resetANSIEscapeCode = `${SymbolRenderer.ansiEscapeCodePrefix}0m`;

// This is adapted from
// https://github.com/apple/swift-testing/blob/786ade71421eb1d8a9c1d99c902cf1c93096e7df/Sources/Testing/Events/Recorder/Event.Symbol.swift#L102
public static symbol(symbol: TestSymbol): string {
Expand All @@ -469,6 +570,8 @@ export class SymbolRenderer {
return "\u{25B2}"; // Unicode: BLACK UP-POINTING TRIANGLE
case TestSymbol.details:
return "\u{2192}"; // Unicode: RIGHTWARDS ARROW
case TestSymbol.none:
return "";
}
} else {
switch (symbol) {
Expand All @@ -486,28 +589,29 @@ export class SymbolRenderer {
return "\u{26A0}\u{FE0E}"; // Unicode: WARNING SIGN + VARIATION SELECTOR-15 (disable emoji)
case TestSymbol.details:
return "\u{21B3}"; // Unicode: DOWNWARDS ARROW WITH TIP RIGHTWARDS
case TestSymbol.none:
return " ";
}
}
}

// This is adapted from
// https://github.com/apple/swift-testing/blob/786ade71421eb1d8a9c1d99c902cf1c93096e7df/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift#L164
private static colorize(symbolType: TestSymbol, symbol: string): string {
const ansiEscapeCodePrefix = "\u{001B}[";
const resetANSIEscapeCode = `${ansiEscapeCodePrefix}0m`;
switch (symbolType) {
case TestSymbol.default:
case TestSymbol.details:
case TestSymbol.skip:
case TestSymbol.difference:
case TestSymbol.passWithKnownIssue:
return `${ansiEscapeCodePrefix}90m${symbol}${resetANSIEscapeCode}`;
return `${SymbolRenderer.ansiEscapeCodePrefix}90m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.pass:
return `${ansiEscapeCodePrefix}92m${symbol}${resetANSIEscapeCode}`;
return `${SymbolRenderer.ansiEscapeCodePrefix}92m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.fail:
return `${ansiEscapeCodePrefix}91m${symbol}${resetANSIEscapeCode}`;
return `${SymbolRenderer.ansiEscapeCodePrefix}91m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.warning:
return `${ansiEscapeCodePrefix}93m${symbol}${resetANSIEscapeCode}`;
case TestSymbol.details:
return `${SymbolRenderer.ansiEscapeCodePrefix}93m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.none:
default:
return symbol;
}
Expand Down
34 changes: 23 additions & 11 deletions test/suite/testexplorer/SwiftTestingOutputParser.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
SourceLocation,
TestSymbol,
SymbolRenderer,
MessageRenderer,
} from "../../../src/TestExplorer/TestParsers/SwiftTestingOutputParser";
import { TestRunState, TestStatus } from "./MockTestRunState";
import { Readable } from "stream";
Expand Down Expand Up @@ -114,7 +115,7 @@ suite("SwiftTestingOutputParser Suite", () => {
]);
});

test("Failed test with one issue", async () => {
async function performTestFailure(messages: EventMessage[]) {
const testRunState = new TestRunState(["MyTests.MyTests/testFail()"], true);
const issueLocation = {
_filePath: "file:///some/file.swift",
Expand All @@ -124,25 +125,23 @@ suite("SwiftTestingOutputParser Suite", () => {
const events = new TestEventStream([
testEvent("runStarted"),
testEvent("testCaseStarted", "MyTests.MyTests/testFail()"),
testEvent(
"issueRecorded",
"MyTests.MyTests/testFail()",
[{ text: "Expectation failed: bar == foo", symbol: TestSymbol.fail }],
issueLocation
),
testEvent("issueRecorded", "MyTests.MyTests/testFail()", messages, issueLocation),
testEvent("testCaseEnded", "MyTests.MyTests/testFail()"),
testEvent("runEnded"),
]);

await outputParser.watch("file:///mock/named/pipe", testRunState, events);

const renderedMessages = messages.map(message => MessageRenderer.render(message));
const fullFailureMessage = renderedMessages.join("\n");

assert.deepEqual(testRunState.tests, [
{
name: "MyTests.MyTests/testFail()",
status: TestStatus.failed,
issues: [
{
message: "Expectation failed: bar == foo",
message: fullFailureMessage,
location: new vscode.Location(
vscode.Uri.file(issueLocation._filePath),
new vscode.Position(issueLocation.line - 1, issueLocation?.column ?? 0)
Expand All @@ -154,11 +153,24 @@ suite("SwiftTestingOutputParser Suite", () => {
timing: {
timestamp: 0,
},
output: [
`\u001b[91m${SymbolRenderer.symbol(TestSymbol.fail)}\u001b[0m Expectation failed: bar == foo\r\n`,
],
output: renderedMessages.map(message => `${message}\r\n`),
},
]);
}

test("Failed with an issue that has a comment", async () => {
await performTestFailure([
{ text: "Expectation failed: bar == foo", symbol: TestSymbol.fail },
{ symbol: TestSymbol.details, text: "// One" },
{ symbol: TestSymbol.details, text: "// Two" },
{ symbol: TestSymbol.details, text: "// Three" },
]);
});

test("Failed test with one issue", async () => {
await performTestFailure([
{ text: "Expectation failed: bar == foo", symbol: TestSymbol.fail },
]);
});

test("Parameterized test", async () => {
Expand Down
38 changes: 32 additions & 6 deletions test/suite/testexplorer/TestExplorerIntegration.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
import { Version } from "../../../src/utilities/version";
import { TestKind } from "../../../src/TestExplorer/TestKind";
import { mockNamespace } from "../../unit-tests/MockUtils";
import {
MessageRenderer,
TestSymbol,
} from "../../../src/TestExplorer/TestParsers/SwiftTestingOutputParser";

suite("Test Explorer Suite", function () {
const MAX_TEST_RUN_TIME_MINUTES = 5;
Expand Down Expand Up @@ -196,7 +200,12 @@ suite("Test Explorer Suite", function () {
failed: [
{
test: "PackageTests.testWithKnownIssueAndUnknownIssue()",
issues: ["Expectation failed: 2 == 3"],
issues: [
MessageRenderer.render({
symbol: TestSymbol.fail,
text: "Expectation failed: 2 == 3",
}),
],
},
],
});
Expand Down Expand Up @@ -387,7 +396,12 @@ suite("Test Explorer Suite", function () {
failed: [
{
test: "PackageTests.topLevelTestFailing()",
issues: ["Expectation failed: 1 == 2"],
issues: [
MessageRenderer.render({
symbol: TestSymbol.fail,
text: "Expectation failed: 1 == 2",
}),
],
},
],
});
Expand All @@ -399,13 +413,16 @@ suite("Test Explorer Suite", function () {
runProfile,
"PackageTests.MixedSwiftTestingSuite"
);

assertTestResults(testRun, {
passed: ["PackageTests.MixedSwiftTestingSuite/testPassing()"],
skipped: ["PackageTests.MixedSwiftTestingSuite/testDisabled()"],
failed: [
{
test: "PackageTests.MixedSwiftTestingSuite/testFailing()",
issues: ["testFailing() \u{203A} Expectation failed: 1 == 2"],
issues: [
`testFailing() \u{203A} ${MessageRenderer.render({ symbol: TestSymbol.fail, text: "Expectation failed: 1 == 2" })}`,
],
},
{
issues: [],
Expand All @@ -429,7 +446,12 @@ suite("Test Explorer Suite", function () {
],
failed: [
{
issues: ["2 \u{203A} Expectation failed: (arg → 2) != 2"],
issues: [
`2 \u{203A} ${MessageRenderer.render({
symbol: TestSymbol.fail,
text: "Expectation failed: (arg → 2) != 2",
})}`,
],
test: "PackageTests.parameterizedTest(_:)/PackageTests.swift:51:2/argumentIDs: Optional([Testing.Test.Case.Argument.ID(bytes: [50])])",
},
{
Expand All @@ -453,7 +475,9 @@ suite("Test Explorer Suite", function () {
failed: [
{
test: "PackageTests.MixedSwiftTestingSuite/testFailing()",
issues: ["testFailing() \u{203A} Expectation failed: 1 == 2"],
issues: [
`testFailing() \u{203A} ${MessageRenderer.render({ symbol: TestSymbol.fail, text: "Expectation failed: 1 == 2" })}`,
],
},
{
issues: [],
Expand All @@ -480,7 +504,9 @@ suite("Test Explorer Suite", function () {
failed: [
{
test: "PackageTests.MixedSwiftTestingSuite/testFailing()",
issues: ["testFailing() \u{203A} Expectation failed: 1 == 2"],
issues: [
`testFailing() \u{203A} ${MessageRenderer.render({ symbol: TestSymbol.fail, text: "Expectation failed: 1 == 2" })}`,
],
},
{
issues: [],
Expand Down