Add tests for Run Tests Multiple Times (#1041)

Author: plemarquand

src/TestExplorer/TestRunner.ts

@@ -45,6 +45,18 @@ export enum TestLibrary {
     swiftTesting = "swift-testing",
 }
 
+export interface TestRunState {
+    failed: {
+        test: vscode.TestItem;
+        message: vscode.TestMessage | readonly vscode.TestMessage[];
+    }[];
+    passed: vscode.TestItem[];
+    skipped: vscode.TestItem[];
+    errored: vscode.TestItem[];
+    unknown: number;
+    output: string[];
+}
+
 export class TestRunProxy {
     private testRun?: vscode.TestRun;
     private addedTestItems: { testClass: TestClass; parentIndex: number }[] = [];
@@ -60,17 +72,14 @@ export class TestRunProxy {
     // Allows for introspection on the state of TestItems after a test run.
     public runState = TestRunProxy.initialTestRunState();
 
-    private static initialTestRunState() {
+    public static initialTestRunState(): TestRunState {
         return {
-            failed: [] as {
-                test: vscode.TestItem;
-                message: vscode.TestMessage | readonly vscode.TestMessage[];
-            }[],
-            passed: [] as vscode.TestItem[],
-            skipped: [] as vscode.TestItem[],
-            errored: [] as vscode.TestItem[],
+            failed: [],
+            passed: [],
+            skipped: [],
+            errored: [],
             unknown: 0,
-            output: [] as string[],
+            output: [],
         };
     }
 
@@ -464,7 +473,10 @@ export class TestRunner {
     }
 
     /** Run test session without attaching to a debugger */
-    async runSession(token: vscode.CancellationToken, runState: TestRunnerTestRunState) {
+    async runSession(
+        token: vscode.CancellationToken,
+        runState: TestRunnerTestRunState
+    ): Promise<TestRunState> {
         // Run swift-testing first, then XCTest.
         // swift-testing being parallel by default should help these run faster.
         if (this.testArgs.hasSwiftTestingTests) {
@@ -486,13 +498,13 @@ export class TestRunner {
                 );
 
                 if (testBuildConfig === null) {
-                    return;
+                    return this.testRun.runState;
                 }
 
                 const outputStream = this.testOutputWritable(TestLibrary.swiftTesting, runState);
                 if (token.isCancellationRequested) {
                     outputStream.end();
-                    return;
+                    return this.testRun.runState;
                 }
 
                 // Watch the pipe for JSONL output and parse the events into test explorer updates.
@@ -518,13 +530,13 @@ export class TestRunner {
                 true
             );
             if (testBuildConfig === null) {
-                return;
+                return this.testRun.runState;
             }
 
             const parsedOutputStream = this.testOutputWritable(TestLibrary.xctest, runState);
             if (token.isCancellationRequested) {
                 parsedOutputStream.end();
-                return;
+                return this.testRun.runState;
             }
 
             // XCTestRuns are started immediately
@@ -539,6 +551,8 @@ export class TestRunner {
                 TestLibrary.xctest
             );
         }
+
+        return this.testRun.runState;
     }
 
     private async launchTests(

src/commands.ts

@@ -76,12 +76,16 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
         vscode.commands.registerCommand("swift.run", () => runBuild(ctx)),
         vscode.commands.registerCommand("swift.debug", () => debugBuild(ctx)),
         vscode.commands.registerCommand("swift.cleanBuild", () => cleanBuild(ctx)),
-        vscode.commands.registerCommand("swift.runTestsMultipleTimes", item =>
-            runTestMultipleTimes(ctx, item, false)
-        ),
-        vscode.commands.registerCommand("swift.runTestsUntilFailure", item =>
-            runTestMultipleTimes(ctx, item, true)
-        ),
+        vscode.commands.registerCommand("swift.runTestsMultipleTimes", item => {
+            if (ctx.currentFolder) {
+                runTestMultipleTimes(ctx.currentFolder, item, false);
+            }
+        }),
+        vscode.commands.registerCommand("swift.runTestsUntilFailure", item => {
+            if (ctx.currentFolder) {
+                runTestMultipleTimes(ctx.currentFolder, item, true);
+            }
+        }),
         // Note: This is only available on macOS (gated in `package.json`) because its the only OS that has the iOS SDK available.
         vscode.commands.registerCommand("swift.switchPlatform", () => switchPlatform()),
         vscode.commands.registerCommand("swift.resetPackage", () => resetPackage(ctx)),

src/commands/testMultipleTimes.ts

@@ -14,8 +14,8 @@
 
 import * as vscode from "vscode";
 import { TestKind } from "../TestExplorer/TestKind";
-import { TestRunner, TestRunnerTestRunState } from "../TestExplorer/TestRunner";
-import { WorkspaceContext } from "../WorkspaceContext";
+import { TestRunner, TestRunnerTestRunState, TestRunState } from "../TestExplorer/TestRunner";
+import { FolderContext } from "../FolderContext";
 
 /**
  * Runs the supplied TestItem a number of times. The user is prompted with a dialog
@@ -25,26 +25,27 @@ import { WorkspaceContext } from "../WorkspaceContext";
  * @param untilFailure If `true` stop running the test if it fails
  */
 export async function runTestMultipleTimes(
-    ctx: WorkspaceContext,
+    currentFolder: FolderContext,
     test: vscode.TestItem,
-    untilFailure: boolean
+    untilFailure: boolean,
+    testRunner?: () => Promise<TestRunState>
 ) {
     const str = await vscode.window.showInputBox({
         prompt: "Label: ",
         placeHolder: `${untilFailure ? "Maximum " : ""}# of times to run`,
         validateInput: value => (/^[1-9]\d*$/.test(value) ? undefined : "Enter an integer value"),
     });
 
-    if (!str || !ctx.currentFolder?.testExplorer) {
+    if (!str || !currentFolder.testExplorer) {
         return;
     }
 
     const numExecutions = parseInt(str);
-    const testExplorer = ctx.currentFolder.testExplorer;
+    const testExplorer = currentFolder.testExplorer;
     const runner = new TestRunner(
         TestKind.standard,
         new vscode.TestRunRequest([test]),
-        ctx.currentFolder,
+        currentFolder,
         testExplorer.controller
     );
 
@@ -55,19 +56,22 @@ export async function runTestMultipleTimes(
 
     vscode.commands.executeCommand("workbench.panel.testResults.view.focus");
 
+    const runStates: TestRunState[] = [];
     for (let i = 0; i < numExecutions; i++) {
         runner.setIteration(i);
         runner.testRun.appendOutput(`\x1b[36mBeginning Test Iteration #${i + 1}\x1b[0m\n`);
 
-        await runner.runSession(token.token, testRunState);
+        const runState = await (testRunner !== undefined
+            ? testRunner()
+            : runner.runSession(token.token, testRunState));
 
-        if (
-            untilFailure &&
-            (runner.testRun.runState.failed.length > 0 ||
-                runner.testRun.runState.errored.length > 0)
-        ) {
+        runStates.push(runState);
+
+        if (untilFailure && (runState.failed.length > 0 || runState.errored.length > 0)) {
             break;
         }
     }
     runner.testRun.end();
+
+    return runStates;
 }

test/suite/commands/runTestMultipleTimes.test.ts

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the VS Code Swift open source project
+//
+// Copyright (c) 2024 the VS Code Swift project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of VS Code Swift project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import * as vscode from "vscode";
+import * as assert from "assert";
+import { anything, when } from "ts-mockito";
+import { runTestMultipleTimes } from "../../../src/commands/testMultipleTimes";
+import { mockNamespace } from "../../unit-tests/MockUtils";
+import { SwiftToolchain } from "../../../src/toolchain/toolchain";
+import { WorkspaceContext } from "../../../src/WorkspaceContext";
+import { FolderContext } from "../../../src/FolderContext";
+import { SwiftOutputChannel } from "../../../src/ui/SwiftOutputChannel";
+import { testAssetWorkspaceFolder } from "../../fixtures";
+import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
+
+suite("Test Multiple Times Command Test Suite", () => {
+    const windowMock = mockNamespace(vscode, "window");
+
+    let folderContext: FolderContext;
+    let testItem: vscode.TestItem;
+
+    suiteSetup(async () => {
+        const toolchain = await SwiftToolchain.create();
+        const workspaceContext = await WorkspaceContext.create(
+            new SwiftOutputChannel("Swift"),
+            toolchain
+        );
+        const workspaceFolder = testAssetWorkspaceFolder("diagnostics");
+        folderContext = await workspaceContext.addPackageFolder(
+            workspaceFolder.uri,
+            workspaceFolder
+        );
+        folderContext.addTestExplorer();
+
+        const item = folderContext.testExplorer?.controller.createTestItem(
+            "testId",
+            "Test Item For Testing"
+        );
+
+        assert.ok(item);
+        testItem = item;
+    });
+
+    test("Runs successfully after testing 0 times", async () => {
+        when(windowMock.showInputBox(anything())).thenReturn(Promise.resolve("0"));
+        const runState = await runTestMultipleTimes(folderContext, testItem, false);
+        assert.deepStrictEqual(runState, []);
+    });
+
+    test("Runs successfully after testing 3 times", async () => {
+        when(windowMock.showInputBox(anything())).thenReturn(Promise.resolve("3"));
+
+        const runState = await runTestMultipleTimes(folderContext, testItem, false, () =>
+            Promise.resolve(TestRunProxy.initialTestRunState())
+        );
+
+        assert.deepStrictEqual(runState, [
+            TestRunProxy.initialTestRunState(),
+            TestRunProxy.initialTestRunState(),
+            TestRunProxy.initialTestRunState(),
+        ]);
+    });
+
+    test("Stops after a failure on the 2nd iteration ", async () => {
+        when(windowMock.showInputBox(anything())).thenReturn(Promise.resolve("3"));
+
+        const failure = {
+            ...TestRunProxy.initialTestRunState(),
+            failed: [{ test: testItem, message: new vscode.TestMessage("oh no") }],
+        };
+        let ctr = 0;
+        const runState = await runTestMultipleTimes(folderContext, testItem, true, () => {
+            ctr += 1;
+            if (ctr === 2) {
+                return Promise.resolve(failure);
+            }
+            return Promise.resolve(TestRunProxy.initialTestRunState());
+        });
+
+        assert.deepStrictEqual(runState, [TestRunProxy.initialTestRunState(), failure]);
+    });
+});