Add "Run Test Multiple Times"

Adds two new items to the context menu when you right click a test in
the test explorer:

- Run Multiple Times
- Run Until Failure

Selecting either of these promts with a text input where the user can
input how many times they want to run the test(s).

If the user selected Run Until Failure the tests will be run a maximum
number of times, stopping at the first iteration that produces a
failure.

A current limitation in VS Code is if you have multiple tests selected
in the Test Explorer (by shift or ctrl clicking them) only the first is
passed to the command.

Issue: swiftlang#832

Author: plemarquand

package.json

@@ -201,6 +201,16 @@
         "command": "swift.clearDiagnosticsCollection",
         "title": "Clear Diagnostics Collection",
         "category": "Swift"
+      },
+      {
+        "command": "swift.runTestsMultipleTimes",
+        "title": "Run Multiple Times...",
+        "category": "Swift"
+      },
+      {
+        "command": "swift.runTestsUntilFailure",
+        "title": "Run Until Failure...",
+        "category": "Swift"
       }
     ],
     "configuration": [
@@ -598,6 +608,18 @@
       }
     ],
     "menus": {
+      "testing/item/context": [
+        {
+          "command": "swift.runTestsMultipleTimes",
+          "group": "testExtras",
+          "when": "testId"
+        },
+        {
+          "command": "swift.runTestsUntilFailure",
+          "group": "testExtras",
+          "when": "testId"
+        }
+      ],
       "commandPalette": [
         {
           "command": "swift.createNewProject",

src/TestExplorer/TestExplorer.ts

@@ -41,7 +41,7 @@ export class TestExplorer {
     private onTestItemsDidChangeEmitter = new vscode.EventEmitter<vscode.TestController>();
     public onTestItemsDidChange: vscode.Event<vscode.TestController>;
 
-    private onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
+    public onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
     public onCreateTestRun: vscode.Event<TestRunProxy>;
 
     constructor(public folderContext: FolderContext) {

src/TestExplorer/TestRunner.ts

@@ -52,20 +52,28 @@ export class TestRunProxy {
     private runStarted: boolean = false;
     private queuedOutput: string[] = [];
     private _testItems: vscode.TestItem[];
+    private iteration: number | undefined;
     public coverage: TestCoverage;
 
+    public testRunCompleteEmitter = new vscode.EventEmitter<void>();
+    public onTestRunComplete: vscode.Event<void>;
+
     // Allows for introspection on the state of TestItems after a test run.
-    public runState = {
-        failed: [] as {
-            test: vscode.TestItem;
-            message: vscode.TestMessage | readonly vscode.TestMessage[];
-        }[],
-        passed: [] as vscode.TestItem[],
-        skipped: [] as vscode.TestItem[],
-        errored: [] as vscode.TestItem[],
-        unknown: 0,
-        output: [] as string[],
-    };
+    public runState = TestRunProxy.initialTestRunState();
+
+    private static initialTestRunState() {
+        return {
+            failed: [] as {
+                test: vscode.TestItem;
+                message: vscode.TestMessage | readonly vscode.TestMessage[];
+            }[],
+            passed: [] as vscode.TestItem[],
+            skipped: [] as vscode.TestItem[],
+            errored: [] as vscode.TestItem[],
+            unknown: 0,
+            output: [] as string[],
+        };
+    }
 
     public get testItems(): vscode.TestItem[] {
         return this._testItems;
@@ -79,6 +87,7 @@ export class TestRunProxy {
     ) {
         this._testItems = args.testItems;
         this.coverage = new TestCoverage(folderContext);
+        this.onTestRunComplete = this.testRunCompleteEmitter.event;
     }
 
     public testRunStarted = () => {
@@ -194,20 +203,37 @@ export class TestRunProxy {
 
     public async end() {
         this.testRun?.end();
+        this.testRunCompleteEmitter.fire();
+    }
+
+    public setIteration(iteration: number) {
+        this.runState = TestRunProxy.initialTestRunState();
+        this.iteration = iteration;
     }
 
     public appendOutput(output: string) {
+        const tranformedOutput = this.prependIterationToOutput(output);
         if (this.testRun) {
-            this.testRun.appendOutput(output);
-            this.runState.output.push(output);
+            this.testRun.appendOutput(tranformedOutput);
+            this.runState.output.push(tranformedOutput);
         } else {
-            this.queuedOutput.push(output);
+            this.queuedOutput.push(tranformedOutput);
         }
     }
 
     public appendOutputToTest(output: string, test: vscode.TestItem, location?: vscode.Location) {
-        this.testRun?.appendOutput(output, location, test);
-        this.runState.output.push(output);
+        const tranformedOutput = this.prependIterationToOutput(output);
+        this.testRun?.appendOutput(tranformedOutput, location, test);
+        this.runState.output.push(tranformedOutput);
+    }
+
+    private prependIterationToOutput(output: string): string {
+        if (this.iteration === undefined) {
+            return output;
+        }
+        const itr = this.iteration + 1;
+        const lines = output.match(/[^\r\n]*[\r\n]*/g);
+        return lines?.map(line => (line ? `\x1b[34mRun ${itr}\x1b[0m ${line}` : "")).join("") ?? "";
     }
 
     public async computeCoverage() {
@@ -222,7 +248,7 @@ export class TestRunProxy {
 
 /** Class used to run tests */
 export class TestRunner {
-    private testRun: TestRunProxy;
+    public testRun: TestRunProxy;
     private testArgs: TestRunArguments;
     private xcTestOutputParser: IXCTestOutputParser;
     private swiftTestOutputParser: SwiftTestingOutputParser;
@@ -256,6 +282,20 @@ export class TestRunner {
         );
     }
 
+    /**
+     * When performing a "Run test multiple times" run set the iteration
+     * so it can be shown in the logs.
+     * @param iteration The iteration counter
+     */
+    public setIteration(iteration: number) {
+        // The SwiftTestingOutputParser holds state and needs to be reset between iterations.
+        this.swiftTestOutputParser = new SwiftTestingOutputParser(
+            this.testRun.testRunStarted,
+            this.testRun.addParameterizedTestCase
+        );
+        this.testRun.setIteration(iteration);
+    }
+
     /**
      * If the request has no test items to include in the run,
      * default to usig all the items in the `TestController`.
@@ -587,6 +627,7 @@ export class TestRunner {
             task.execution.onDidWrite(str => {
                 const replaced = str
                     .replace("[1/1] Planning build", "") // Work around SPM still emitting progress when doing --no-build.
+                    .replace(/\[1\/1\] Write swift-version-.*/gm, "")
                     .replace(
                         /LLVM Profile Error: Failed to write file "default.profraw": Operation not permitted\r\n/gm,
                         ""
@@ -1051,7 +1092,7 @@ export class TestRunnerTestRunState implements ITestRunState {
         }
 
         const isSuite = test.children.size > 0;
-        const issues = isSuite ? this.childrensIssues(test) : this.issues.get(index) ?? [];
+        const issues = isSuite ? this.childrensIssues(test) : this.pluckIssues(index) ?? [];
         if (issues.length > 0) {
             const allUnknownIssues = issues.filter(({ isKnown }) => !isKnown);
             if (allUnknownIssues.length === 0) {
@@ -1076,14 +1117,24 @@ export class TestRunnerTestRunState implements ITestRunState {
         this.currentTestItem = undefined;
     }
 
+    // Removes and returns the issues at the supplied test index.
+    private pluckIssues(index: number): {
+        isKnown: boolean;
+        message: vscode.TestMessage;
+    }[] {
+        const issues = this.issues.get(index) ?? [];
+        this.issues.delete(index);
+        return issues;
+    }
+
     // Gather the issues of test children into a flat collection.
     private childrensIssues(test: vscode.TestItem): {
         isKnown: boolean;
         message: vscode.TestMessage;
     }[] {
         const index = this.getTestItemIndex(test.id);
         return [
-            ...(this.issues.get(index) ?? []),
+            ...(this.pluckIssues(index) ?? []),
             ...reduceTestItemChildren(
                 test.children,
                 (acc, test) => [

src/commands.ts

@@ -33,6 +33,8 @@ import { SwiftProjectTemplate } from "./toolchain/toolchain";
 import { showToolchainSelectionQuickPick, showToolchainError } from "./ui/ToolchainSelection";
 import { captureDiagnostics } from "./commands/captureDiagnostics";
 import { reindexProjectRequest } from "./sourcekit-lsp/lspExtensions";
+import { TestRunner, TestRunnerTestRunState } from "./TestExplorer/TestRunner";
+import { TestKind } from "./TestExplorer/TestKind";
 
 /**
  * References:
@@ -734,6 +736,54 @@ function openInExternalEditor(packageNode: PackageNode) {
     }
 }
 
+async function runTestMultipleTimes(
+    ctx: WorkspaceContext,
+    test: vscode.TestItem,
+    untilFailure: boolean
+) {
+    const str = await vscode.window.showInputBox({
+        prompt: "Label: ",
+        placeHolder: `${untilFailure ? "Maximum " : ""}# of times to run`,
+        validateInput: value => (/^[1-9]\d*$/.test(value) ? undefined : "Enter an integer value"),
+    });
+
+    if (!str || !ctx.currentFolder?.testExplorer) {
+        return;
+    }
+
+    const numExecutions = parseInt(str);
+    const testExplorer = ctx.currentFolder.testExplorer;
+    const runner = new TestRunner(
+        TestKind.standard,
+        new vscode.TestRunRequest([test]),
+        ctx.currentFolder,
+        testExplorer.controller
+    );
+
+    testExplorer.onDidCreateTestRunEmitter.fire(runner.testRun);
+
+    const testRunState = new TestRunnerTestRunState(runner.testRun);
+    const token = new vscode.CancellationTokenSource();
+
+    vscode.commands.executeCommand("workbench.panel.testResults.view.focus");
+
+    for (let i = 0; i < numExecutions; i++) {
+        runner.setIteration(i);
+        runner.testRun.appendOutput(`\x1b[36mBeginning Test Iteration #${i + 1}\x1b[0m\n`);
+
+        await runner.runSession(token.token, testRunState);
+
+        if (
+            untilFailure &&
+            (runner.testRun.runState.failed.length > 0 ||
+                runner.testRun.runState.errored.length > 0)
+        ) {
+            break;
+        }
+    }
+    runner.testRun.end();
+}
+
 /**
  * Switches the target SDK to the platform selected in a QuickPick UI.
  */
@@ -841,6 +891,12 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
         vscode.commands.registerCommand("swift.run", () => runBuild(ctx)),
         vscode.commands.registerCommand("swift.debug", () => debugBuild(ctx)),
         vscode.commands.registerCommand("swift.cleanBuild", () => cleanBuild(ctx)),
+        vscode.commands.registerCommand("swift.runTestsMultipleTimes", item =>
+            runTestMultipleTimes(ctx, item, false)
+        ),
+        vscode.commands.registerCommand("swift.runTestsUntilFailure", item =>
+            runTestMultipleTimes(ctx, item, true)
+        ),
         // Note: This is only available on macOS (gated in `package.json`) because its the only OS that has the iOS SDK available.
         vscode.commands.registerCommand("swift.switchPlatform", () => switchPlatform()),
         vscode.commands.registerCommand("swift.resetPackage", () => resetPackage(ctx)),