Associate comment details on #expect w/ test issue

plemarquand · plemarquand · commit 40f475d90e0b · 2024-08-22T15:53:46.000-04:00
If a swift-testing #expect macro has preceding comments they are sent
along with the expectation failure text as entries in the EventMessage
array. Pull them out of the array so they don't appear as their own
separate issues and append them to the actual issue text.

Also fixes up some of the issue message colorization to be more
consistent with how swift-testing behaves on the command line.
diff --git a/src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts b/src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts
@@ -148,6 +148,7 @@ export enum TestSymbol {
     difference = "difference",
     warning = "warning",
     details = "details",
+    none = "none",
 }
 
 export interface EventMessage {
@@ -222,9 +223,27 @@ export class SwiftTestingOutputParser {
      * @param chunk A chunk of stdout emitted during a test run.
      */
     public parseStdout = (() => {
-        const values = Object.values(TestSymbol).map(symbol =>
-            regexEscapedString(SymbolRenderer.eventMessageSymbol(symbol))
-        );
+        const values = [
+            ...Object.values(TestSymbol).map(symbol =>
+                regexEscapedString(
+                    // Trim the ANSI reset code from the search since some lines
+                    // are fully colorized from the symbol to the end of line.
+                    SymbolRenderer.eventMessageSymbol(symbol).replace(
+                        SymbolRenderer.resetANSIEscapeCode,
+                        ""
+                    )
+                )
+            ),
+            // It is possible there is no symbol for a line produced by swift-testing,
+            // for instance if the user has a multi line comment before a failing expectation
+            // only the first line of the printed comment will have a symbol, but to make the
+            // indentation consistent the subsequent lines will have three spaces. We don't want
+            // to treat this as output produced by the user during the test run, so omit these.
+            // This isn't ideal since this will swallow lines the user prints if they start with
+            // three spaces, but until we have user output as part of the JSON event stream we have
+            // this workaround.
+            "   ",
+        ];
 
         // Build a regex of all the line beginnings that come out of swift-testing events.
         const isSwiftTestingLineBeginning = new RegExp(`^${values.join("|")}`);
@@ -313,13 +332,47 @@ export class SwiftTestingOutputParser {
         testIndex: number | undefined
     ) {
         messages.forEach(message => {
-            runState.recordOutput(
-                testIndex,
-                `${SymbolRenderer.eventMessageSymbol(message.symbol)} ${message.text}\r\n`
-            );
+            runState.recordOutput(testIndex, `${MessageRenderer.render(message)}\r\n`);
         });
     }
 
+    /**
+     * Partitions a collection of messages in to issues and details about the issues.
+     * This is used to print the issues first, followed by the details.
+     */
+    private partitionIssueMessages(messages: EventMessage[]): {
+        issues: EventMessage[];
+        details: EventMessage[];
+    } {
+        return messages.reduce(
+            (buckets, message) => {
+                const key =
+                    message.symbol === "details" ||
+                    message.symbol === "default" ||
+                    message.symbol === "none"
+                        ? "details"
+                        : "issues";
+                return { ...buckets, [key]: [...buckets[key], message] };
+            },
+            {
+                issues: [],
+                details: [],
+            }
+        );
+    }
+
+    /*
+     * A multi line comment preceeding an issue will have a 'default' symbol for
+     * all lines except the first one. To match the swift-testing command line we
+     * should show no symbol on these lines.
+     */
+    private transformIssueMessageSymbols(messages: EventMessage[]): EventMessage[] {
+        return messages.map(message => ({
+            ...message,
+            symbol: message.symbol === "default" ? TestSymbol.none : message.symbol,
+        }));
+    }
+
     private parse(item: SwiftTestEvent, runState: ITestRunState) {
         if (
             item.kind === "test" &&
@@ -391,14 +444,31 @@ export class SwiftTestingOutputParser {
                     sourceLocation.line,
                     sourceLocation.column
                 );
-                item.payload.messages.forEach(message => {
-                    runState.recordIssue(testIndex, message.text, isKnown, location);
+
+                const messages = this.transformIssueMessageSymbols(item.payload.messages);
+                const { issues, details } = this.partitionIssueMessages(messages);
+
+                // Order the details after the issue text.
+                const additionalDetails = details
+                    .map(message => MessageRenderer.render(message))
+                    .join("\n");
+
+                issues.forEach(message => {
+                    runState.recordIssue(
+                        testIndex,
+                        additionalDetails.length > 0
+                            ? `${MessageRenderer.render(message)}\n${additionalDetails}`
+                            : MessageRenderer.render(message),
+                        isKnown,
+                        location
+                    );
                 });
-                this.recordOutput(runState, item.payload.messages, testIndex);
+
+                this.recordOutput(runState, messages, testIndex);
 
                 if (item.payload._testCase && testID !== item.payload.testID) {
                     const testIndex = this.getTestCaseIndex(runState, item.payload.testID);
-                    item.payload.messages.forEach(message => {
+                    messages.forEach(message => {
                         runState.recordIssue(testIndex, message.text, isKnown, location);
                     });
                 }
@@ -439,6 +509,32 @@ export class SwiftTestingOutputParser {
     }
 }
 
+export class MessageRenderer {
+    /**
+     * Converts a swift-testing `EventMessage` to a colorized symbol and message text.
+     *
+     * @param message An event message, typically found on an `EventRecordPayload`.
+     * @returns A string colorized with ANSI escape codes.
+     */
+    static render(message: EventMessage): string {
+        return `${SymbolRenderer.eventMessageSymbol(message.symbol)} ${MessageRenderer.colorize(message.symbol, message.text)}`;
+    }
+
+    private static colorize(symbolType: TestSymbol, message: string): string {
+        const ansiEscapeCodePrefix = "\u{001B}[";
+        const resetANSIEscapeCode = `${ansiEscapeCodePrefix}0m`;
+        switch (symbolType) {
+            case TestSymbol.details:
+            case TestSymbol.skip:
+            case TestSymbol.difference:
+            case TestSymbol.passWithKnownIssue:
+                return `${ansiEscapeCodePrefix}90m${message}${resetANSIEscapeCode}`;
+            default:
+                return message;
+        }
+    }
+}
+
 export class SymbolRenderer {
     /**
      * Converts a swift-testing symbol identifier in to a colorized unicode symbol.
@@ -450,6 +546,9 @@ export class SymbolRenderer {
         return this.colorize(symbol, this.symbol(symbol));
     }
 
+    static ansiEscapeCodePrefix = "\u{001B}[";
+    static resetANSIEscapeCode = `${SymbolRenderer.ansiEscapeCodePrefix}0m`;
+
     // This is adapted from
     // https://github.com/apple/swift-testing/blob/786ade71421eb1d8a9c1d99c902cf1c93096e7df/Sources/Testing/Events/Recorder/Event.Symbol.swift#L102
     public static symbol(symbol: TestSymbol): string {
@@ -469,6 +568,8 @@ export class SymbolRenderer {
                     return "\u{25B2}"; // Unicode: BLACK UP-POINTING TRIANGLE
                 case TestSymbol.details:
                     return "\u{2192}"; // Unicode: RIGHTWARDS ARROW
+                case TestSymbol.none:
+                    return "";
             }
         } else {
             switch (symbol) {
@@ -486,28 +587,29 @@ export class SymbolRenderer {
                     return "\u{26A0}\u{FE0E}"; // Unicode: WARNING SIGN + VARIATION SELECTOR-15 (disable emoji)
                 case TestSymbol.details:
                     return "\u{21B3}"; // Unicode: DOWNWARDS ARROW WITH TIP RIGHTWARDS
+                case TestSymbol.none:
+                    return " ";
             }
         }
     }
 
     // This is adapted from
     // https://github.com/apple/swift-testing/blob/786ade71421eb1d8a9c1d99c902cf1c93096e7df/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift#L164
     private static colorize(symbolType: TestSymbol, symbol: string): string {
-        const ansiEscapeCodePrefix = "\u{001B}[";
-        const resetANSIEscapeCode = `${ansiEscapeCodePrefix}0m`;
         switch (symbolType) {
             case TestSymbol.default:
+            case TestSymbol.details:
             case TestSymbol.skip:
             case TestSymbol.difference:
             case TestSymbol.passWithKnownIssue:
-                return `${ansiEscapeCodePrefix}90m${symbol}${resetANSIEscapeCode}`;
+                return `${SymbolRenderer.ansiEscapeCodePrefix}90m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
             case TestSymbol.pass:
-                return `${ansiEscapeCodePrefix}92m${symbol}${resetANSIEscapeCode}`;
+                return `${SymbolRenderer.ansiEscapeCodePrefix}92m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
             case TestSymbol.fail:
-                return `${ansiEscapeCodePrefix}91m${symbol}${resetANSIEscapeCode}`;
+                return `${SymbolRenderer.ansiEscapeCodePrefix}91m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
             case TestSymbol.warning:
-                return `${ansiEscapeCodePrefix}93m${symbol}${resetANSIEscapeCode}`;
-            case TestSymbol.details:
+                return `${SymbolRenderer.ansiEscapeCodePrefix}93m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+            case TestSymbol.none:
             default:
                 return symbol;
         }
diff --git a/test/suite/testexplorer/SwiftTestingOutputParser.test.ts b/test/suite/testexplorer/SwiftTestingOutputParser.test.ts
@@ -24,6 +24,7 @@ import {
     SourceLocation,
     TestSymbol,
     SymbolRenderer,
+    MessageRenderer,
 } from "../../../src/TestExplorer/TestParsers/SwiftTestingOutputParser";
 import { TestRunState, TestStatus } from "./MockTestRunState";
 import { Readable } from "stream";
@@ -114,7 +115,7 @@ suite("SwiftTestingOutputParser Suite", () => {
         ]);
     });
 
-    test("Failed test with one issue", async () => {
+    async function performTestFailure(messages: EventMessage[]) {
         const testRunState = new TestRunState(["MyTests.MyTests/testFail()"], true);
         const issueLocation = {
             _filePath: "file:///some/file.swift",
@@ -124,25 +125,23 @@ suite("SwiftTestingOutputParser Suite", () => {
         const events = new TestEventStream([
             testEvent("runStarted"),
             testEvent("testCaseStarted", "MyTests.MyTests/testFail()"),
-            testEvent(
-                "issueRecorded",
-                "MyTests.MyTests/testFail()",
-                [{ text: "Expectation failed: bar == foo", symbol: TestSymbol.fail }],
-                issueLocation
-            ),
+            testEvent("issueRecorded", "MyTests.MyTests/testFail()", messages, issueLocation),
             testEvent("testCaseEnded", "MyTests.MyTests/testFail()"),
             testEvent("runEnded"),
         ]);
 
         await outputParser.watch("file:///mock/named/pipe", testRunState, events);
 
+        const renderedMessages = messages.map(message => MessageRenderer.render(message));
+        const fullFailureMessage = renderedMessages.join("\n");
+
         assert.deepEqual(testRunState.tests, [
             {
                 name: "MyTests.MyTests/testFail()",
                 status: TestStatus.failed,
                 issues: [
                     {
-                        message: "Expectation failed: bar == foo",
+                        message: fullFailureMessage,
                         location: new vscode.Location(
                             vscode.Uri.file(issueLocation._filePath),
                             new vscode.Position(issueLocation.line - 1, issueLocation?.column ?? 0)
@@ -154,11 +153,24 @@ suite("SwiftTestingOutputParser Suite", () => {
                 timing: {
                     timestamp: 0,
                 },
-                output: [
-                    `\u001b[91m${SymbolRenderer.symbol(TestSymbol.fail)}\u001b[0m Expectation failed: bar == foo\r\n`,
-                ],
+                output: renderedMessages.map(message => `${message}\r\n`),
             },
         ]);
+    }
+
+    test("Failed with an issue that has a comment", async () => {
+        await performTestFailure([
+            { text: "Expectation failed: bar == foo", symbol: TestSymbol.fail },
+            { symbol: TestSymbol.details, text: "// One" },
+            { symbol: TestSymbol.details, text: "// Two" },
+            { symbol: TestSymbol.details, text: "// Three" },
+        ]);
+    });
+
+    test("Failed test with one issue", async () => {
+        await performTestFailure([
+            { text: "Expectation failed: bar == foo", symbol: TestSymbol.fail },
+        ]);
     });
 
     test("Parameterized test", async () => {
diff --git a/test/suite/testexplorer/TestExplorerIntegration.test.ts b/test/suite/testexplorer/TestExplorerIntegration.test.ts
@@ -31,6 +31,10 @@ import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
 import { Version } from "../../../src/utilities/version";
 import { TestKind } from "../../../src/TestExplorer/TestKind";
 import { mockNamespace } from "../../unit-tests/MockUtils";
+import {
+    MessageRenderer,
+    TestSymbol,
+} from "../../../src/TestExplorer/TestParsers/SwiftTestingOutputParser";
 
 suite("Test Explorer Suite", function () {
     const MAX_TEST_RUN_TIME_MINUTES = 5;
@@ -196,7 +200,12 @@ suite("Test Explorer Suite", function () {
                 failed: [
                     {
                         test: "PackageTests.testWithKnownIssueAndUnknownIssue()",
-                        issues: ["Expectation failed: 2 == 3"],
+                        issues: [
+                            MessageRenderer.render({
+                                symbol: TestSymbol.fail,
+                                text: "Expectation failed: 2 == 3",
+                            }),
+                        ],
                     },
                 ],
             });
@@ -387,7 +396,12 @@ suite("Test Explorer Suite", function () {
                         failed: [
                             {
                                 test: "PackageTests.topLevelTestFailing()",
-                                issues: ["Expectation failed: 1 == 2"],
+                                issues: [
+                                    MessageRenderer.render({
+                                        symbol: TestSymbol.fail,
+                                        text: "Expectation failed: 1 == 2",
+                                    }),
+                                ],
                             },
                         ],
                     });
@@ -399,13 +413,16 @@ suite("Test Explorer Suite", function () {
                         runProfile,
                         "PackageTests.MixedSwiftTestingSuite"
                     );
+
                     assertTestResults(testRun, {
                         passed: ["PackageTests.MixedSwiftTestingSuite/testPassing()"],
                         skipped: ["PackageTests.MixedSwiftTestingSuite/testDisabled()"],
                         failed: [
                             {
                                 test: "PackageTests.MixedSwiftTestingSuite/testFailing()",
-                                issues: ["testFailing() \u{203A} Expectation failed: 1 == 2"],
+                                issues: [
+                                    `testFailing() \u{203A} ${MessageRenderer.render({ symbol: TestSymbol.fail, text: "Expectation failed: 1 == 2" })}`,
+                                ],
                             },
                             {
                                 issues: [],
@@ -429,7 +446,12 @@ suite("Test Explorer Suite", function () {
                         ],
                         failed: [
                             {
-                                issues: ["2 \u{203A} Expectation failed: (arg → 2) != 2"],
+                                issues: [
+                                    `2 \u{203A} ${MessageRenderer.render({
+                                        symbol: TestSymbol.fail,
+                                        text: "Expectation failed: (arg → 2) != 2",
+                                    })}`,
+                                ],
                                 test: "PackageTests.parameterizedTest(_:)/PackageTests.swift:51:2/argumentIDs: Optional([Testing.Test.Case.Argument.ID(bytes: [50])])",
                             },
                             {
@@ -453,7 +475,9 @@ suite("Test Explorer Suite", function () {
                         failed: [
                             {
                                 test: "PackageTests.MixedSwiftTestingSuite/testFailing()",
-                                issues: ["testFailing() \u{203A} Expectation failed: 1 == 2"],
+                                issues: [
+                                    `testFailing() \u{203A} ${MessageRenderer.render({ symbol: TestSymbol.fail, text: "Expectation failed: 1 == 2" })}`,
+                                ],
                             },
                             {
                                 issues: [],
@@ -480,7 +504,9 @@ suite("Test Explorer Suite", function () {
                         failed: [
                             {
                                 test: "PackageTests.MixedSwiftTestingSuite/testFailing()",
-                                issues: ["testFailing() \u{203A} Expectation failed: 1 == 2"],
+                                issues: [
+                                    `testFailing() \u{203A} ${MessageRenderer.render({ symbol: TestSymbol.fail, text: "Expectation failed: 1 == 2" })}`,
+                                ],
                             },
                             {
                                 issues: [],
