Add "Run Test Multiple Times"

package.json

@@ -201,6 +201,16 @@
         "command": "swift.clearDiagnosticsCollection",
         "title": "Clear Diagnostics Collection",
         "category": "Swift"
+      },
+      {
+        "command": "swift.runTestsMultipleTimes",
+        "title": "Run Multiple Times...",
+        "category": "Swift"
+      },
+      {
+        "command": "swift.runTestsUntilFailure",
+        "title": "Run Until Failure...",
+        "category": "Swift"
       }
     ],
     "configuration": [
@@ -598,6 +608,18 @@
       }
     ],
     "menus": {
+      "testing/item/context": [
+        {
+          "command": "swift.runTestsMultipleTimes",
+          "group": "testExtras",
+          "when": "testId"
+        },
+        {
+          "command": "swift.runTestsUntilFailure",
+          "group": "testExtras",
+          "when": "testId"
+        }
+      ],
       "commandPalette": [
         {
           "command": "swift.createNewProject",

src/TestExplorer/TestExplorer.ts

@@ -41,7 +41,7 @@ export class TestExplorer {
     private onTestItemsDidChangeEmitter = new vscode.EventEmitter<vscode.TestController>();
     public onTestItemsDidChange: vscode.Event<vscode.TestController>;
 
-    private onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
+    public onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
     public onCreateTestRun: vscode.Event<TestRunProxy>;
 
     constructor(public folderContext: FolderContext) {

src/TestExplorer/TestRunner.ts

@@ -52,20 +52,28 @@ export class TestRunProxy {
     private runStarted: boolean = false;
     private queuedOutput: string[] = [];
     private _testItems: vscode.TestItem[];
+    private iteration: number | undefined;
     public coverage: TestCoverage;
 
+    public testRunCompleteEmitter = new vscode.EventEmitter<void>();
+    public onTestRunComplete: vscode.Event<void>;
+
     // Allows for introspection on the state of TestItems after a test run.
-    public runState = {
-        failed: [] as {
-            test: vscode.TestItem;
-            message: vscode.TestMessage | readonly vscode.TestMessage[];
-        }[],
-        passed: [] as vscode.TestItem[],
-        skipped: [] as vscode.TestItem[],
-        errored: [] as vscode.TestItem[],
-        unknown: 0,
-        output: [] as string[],
-    };
+    public runState = TestRunProxy.initialTestRunState();
+
+    private static initialTestRunState() {
+        return {
+            failed: [] as {
+                test: vscode.TestItem;
+                message: vscode.TestMessage | readonly vscode.TestMessage[];
+            }[],
+            passed: [] as vscode.TestItem[],
+            skipped: [] as vscode.TestItem[],
+            errored: [] as vscode.TestItem[],
+            unknown: 0,
+            output: [] as string[],
+        };
+    }
 
     public get testItems(): vscode.TestItem[] {
         return this._testItems;
@@ -79,6 +87,7 @@ export class TestRunProxy {
     ) {
         this._testItems = args.testItems;
         this.coverage = new TestCoverage(folderContext);
+        this.onTestRunComplete = this.testRunCompleteEmitter.event;
     }
 
     public testRunStarted = () => {
@@ -194,20 +203,37 @@ export class TestRunProxy {
 
     public async end() {
         this.testRun?.end();
+        this.testRunCompleteEmitter.fire();
+    }
+
+    public setIteration(iteration: number) {
+        this.runState = TestRunProxy.initialTestRunState();
+        this.iteration = iteration;
     }
 
     public appendOutput(output: string) {
+        const tranformedOutput = this.prependIterationToOutput(output);
         if (this.testRun) {
-            this.testRun.appendOutput(output);
-            this.runState.output.push(output);
+            this.testRun.appendOutput(tranformedOutput);
+            this.runState.output.push(tranformedOutput);
         } else {
-            this.queuedOutput.push(output);
+            this.queuedOutput.push(tranformedOutput);
         }
     }
 
     public appendOutputToTest(output: string, test: vscode.TestItem, location?: vscode.Location) {
-        this.testRun?.appendOutput(output, location, test);
-        this.runState.output.push(output);
+        const tranformedOutput = this.prependIterationToOutput(output);
+        this.testRun?.appendOutput(tranformedOutput, location, test);
+        this.runState.output.push(tranformedOutput);
+    }
+
+    private prependIterationToOutput(output: string): string {
+        if (this.iteration === undefined) {
+            return output;
+        }
+        const itr = this.iteration + 1;
+        const lines = output.match(/[^\r\n]*[\r\n]*/g);
+        return lines?.map(line => (line ? `\x1b[34mRun ${itr}\x1b[0m ${line}` : "")).join("") ?? "";
     }
 
     public async computeCoverage() {
@@ -222,7 +248,7 @@ export class TestRunProxy {
 
 /** Class used to run tests */
 export class TestRunner {
-    private testRun: TestRunProxy;
+    public testRun: TestRunProxy;
     private testArgs: TestRunArguments;
     private xcTestOutputParser: IXCTestOutputParser;
     private swiftTestOutputParser: SwiftTestingOutputParser;
@@ -256,6 +282,20 @@ export class TestRunner {
         );
     }
 
+    /**
+     * When performing a "Run test multiple times" run set the iteration
+     * so it can be shown in the logs.
+     * @param iteration The iteration counter
+     */
+    public setIteration(iteration: number) {
+        // The SwiftTestingOutputParser holds state and needs to be reset between iterations.
+        this.swiftTestOutputParser = new SwiftTestingOutputParser(
+            this.testRun.testRunStarted,
+            this.testRun.addParameterizedTestCase
+        );
+        this.testRun.setIteration(iteration);
+    }
+
     /**
      * If the request has no test items to include in the run,
      * default to usig all the items in the `TestController`.
@@ -587,6 +627,7 @@ export class TestRunner {
             task.execution.onDidWrite(str => {
                 const replaced = str
                     .replace("[1/1] Planning build", "") // Work around SPM still emitting progress when doing --no-build.
+                    .replace(/\[1\/1\] Write swift-version-.*/gm, "")
                     .replace(
                         /LLVM Profile Error: Failed to write file "default.profraw": Operation not permitted\r\n/gm,
                         ""
@@ -1023,6 +1064,7 @@ export class TestRunnerTestRunState implements ITestRunState {
             return;
         }
         const testItem = this.testRun.testItems[index];
+        this.issues.delete(index);
         this.testRun.started(testItem);
         this.currentTestItem = testItem;
         this.startTimes.set(index, startTime);

src/commands.ts

@@ -33,6 +33,8 @@ import { SwiftProjectTemplate } from "./toolchain/toolchain";
 import { showToolchainSelectionQuickPick, showToolchainError } from "./ui/ToolchainSelection";
 import { captureDiagnostics } from "./commands/captureDiagnostics";
 import { reindexProjectRequest } from "./sourcekit-lsp/lspExtensions";
+import { TestRunner, TestRunnerTestRunState } from "./TestExplorer/TestRunner";
+import { TestKind } from "./TestExplorer/TestKind";
 
 /**
  * References:
@@ -734,6 +736,54 @@ function openInExternalEditor(packageNode: PackageNode) {
     }
 }
 
+async function runTestMultipleTimes(
+    ctx: WorkspaceContext,
+    test: vscode.TestItem,
+    untilFailure: boolean
+) {
+    const str = await vscode.window.showInputBox({
+        prompt: "Label: ",
+        placeHolder: `${untilFailure ? "Maximum " : ""}# of times to run`,
+        validateInput: value => (/^[1-9]\d*$/.test(value) ? undefined : "Enter an integer value"),
+    });
+
+    if (!str || !ctx.currentFolder?.testExplorer) {
+        return;
+    }
+
+    const numExecutions = parseInt(str);
+    const testExplorer = ctx.currentFolder.testExplorer;
+    const runner = new TestRunner(
+        TestKind.standard,
+        new vscode.TestRunRequest([test]),
+        ctx.currentFolder,
+        testExplorer.controller
+    );
+
+    testExplorer.onDidCreateTestRunEmitter.fire(runner.testRun);
+
+    const testRunState = new TestRunnerTestRunState(runner.testRun);
+    const token = new vscode.CancellationTokenSource();
+
+    vscode.commands.executeCommand("workbench.panel.testResults.view.focus");
+
+    for (let i = 0; i < numExecutions; i++) {
+        runner.setIteration(i);
+        runner.testRun.appendOutput(`\x1b[36mBeginning Test Iteration #${i + 1}\x1b[0m\n`);
+
+        await runner.runSession(token.token, testRunState);
+
+        if (
+            untilFailure &&
+            (runner.testRun.runState.failed.length > 0 ||
+                runner.testRun.runState.errored.length > 0)
+        ) {
+            break;
+        }
+    }
+    runner.testRun.end();
+}
+
 /**
  * Switches the target SDK to the platform selected in a QuickPick UI.
  */
@@ -841,6 +891,12 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
         vscode.commands.registerCommand("swift.run", () => runBuild(ctx)),
         vscode.commands.registerCommand("swift.debug", () => debugBuild(ctx)),
         vscode.commands.registerCommand("swift.cleanBuild", () => cleanBuild(ctx)),
+        vscode.commands.registerCommand("swift.runTestsMultipleTimes", item =>
+            runTestMultipleTimes(ctx, item, false)
+        ),
+        vscode.commands.registerCommand("swift.runTestsUntilFailure", item =>
+            runTestMultipleTimes(ctx, item, true)
+        ),
         // Note: This is only available on macOS (gated in `package.json`) because its the only OS that has the iOS SDK available.
         vscode.commands.registerCommand("swift.switchPlatform", () => switchPlatform()),
         vscode.commands.registerCommand("swift.resetPackage", () => resetPackage(ctx)),

test/suite/testexplorer/TestExplorerIntegration.test.ts

@@ -15,6 +15,7 @@
 import * as vscode from "vscode";
 import * as assert from "assert";
 import { beforeEach } from "mocha";
+import { when, anything } from "ts-mockito";
 import { testAssetUri } from "../../fixtures";
 import { globalWorkspaceContextPromise } from "../extension.test";
 import { TestExplorer } from "../../../src/TestExplorer/TestExplorer";
@@ -29,6 +30,7 @@ import { WorkspaceContext } from "../../../src/WorkspaceContext";
 import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
 import { Version } from "../../../src/utilities/version";
 import { TestKind } from "../../../src/TestExplorer/TestKind";
+import { mockNamespace } from "../../unit-tests/MockUtils";
 
 suite("Test Explorer Suite", function () {
     const MAX_TEST_RUN_TIME_MINUTES = 5;
@@ -49,18 +51,10 @@ suite("Test Explorer Suite", function () {
         )[0];
     }
 
-    async function runTest(
+    async function gatherTests(
         controller: vscode.TestController,
-        runProfile: TestKind,
         ...tests: string[]
-    ): Promise<TestRunProxy> {
-        const targetProfile = testExplorer.testRunProfiles.find(
-            profile => profile.label === runProfile
-        );
-        if (!targetProfile) {
-            throw new Error(`Unable to find run profile named ${runProfile}`);
-        }
-
+    ): Promise<vscode.TestItem[]> {
         const testItems = tests.map(test => {
             const testItem = getTestItem(controller, test);
             if (!testItem) {
@@ -79,6 +73,21 @@ suite("Test Explorer Suite", function () {
             return testItem;
         });
 
+        return testItems;
+    }
+
+    async function runTest(
+        controller: vscode.TestController,
+        runProfile: TestKind,
+        ...tests: string[]
+    ): Promise<TestRunProxy> {
+        const targetProfile = testExplorer.testRunProfiles.find(
+            profile => profile.label === runProfile
+        );
+        if (!targetProfile) {
+            throw new Error(`Unable to find run profile named ${runProfile}`);
+        }
+        const testItems = await gatherTests(controller, ...tests);
         const request = new vscode.TestRunRequest(testItems);
 
         return (
@@ -233,6 +242,37 @@ suite("Test Explorer Suite", function () {
                 ],
             });
         });
+
+        suite("Runs multiple", function () {
+            const numIterations = 5;
+            const windowMock = mockNamespace(vscode, "window");
+
+            test("@slow runs an swift-testing test multiple times", async function () {
+                const testItems = await gatherTests(
+                    testExplorer.controller,
+                    "PackageTests.MixedXCTestSuite/testPassing"
+                );
+
+                await testExplorer.folderContext.workspaceContext.focusFolder(
+                    testExplorer.folderContext
+                );
+
+                // Stub the showInputBox method to return the input text
+                when(windowMock.showInputBox(anything())).thenReturn(
+                    Promise.resolve(`${numIterations}`)
+                );
+
+                vscode.commands.executeCommand("swift.runTestsMultipleTimes", testItems[0]);
+
+                const testRun = await eventPromise(testExplorer.onCreateTestRun);
+
+                await eventPromise(testRun.onTestRunComplete);
+
+                assertTestResults(testRun, {
+                    passed: ["PackageTests.MixedXCTestSuite/testPassing"],
+                });
+            });
+        });
     });
 
     suite("XCTest", () => {
@@ -262,6 +302,37 @@ suite("Test Explorer Suite", function () {
                 passed: ["PackageTests.DebugReleaseTestSuite/testRelease"],
             });
         });
+
+        suite("Runs multiple", function () {
+            const numIterations = 5;
+            const windowMock = mockNamespace(vscode, "window");
+
+            test("@slow runs an XCTest multiple times", async function () {
+                const testItems = await gatherTests(
+                    testExplorer.controller,
+                    "PackageTests.topLevelTestPassing()"
+                );
+
+                await testExplorer.folderContext.workspaceContext.focusFolder(
+                    testExplorer.folderContext
+                );
+
+                // Stub the showInputBox method to return the input text
+                when(windowMock.showInputBox(anything())).thenReturn(
+                    Promise.resolve(`${numIterations}`)
+                );
+
+                vscode.commands.executeCommand("swift.runTestsMultipleTimes", testItems[0]);
+
+                const testRun = await eventPromise(testExplorer.onCreateTestRun);
+
+                await eventPromise(testRun.onTestRunComplete);
+
+                assertTestResults(testRun, {
+                    passed: ["PackageTests.topLevelTestPassing()"],
+                });
+            });
+        });
     });
 
     // Do coverage last as it does a full rebuild, causing the stage after it to have to rebuild as well.