Add "Run All/Debug All" dropdown picker for parametrized tests (#8757)

* Register new command for running parametrized tests

* Show "Run All" or "Debug All" in dropdown picker on parametrized tests

* Use CommandSource as input to second call of getNamesAndValues

In the picker unit tests two calls to getNamesAndValues are used to
extract values from enums. The second call used a generic with type
'CommandSource' but as input a Type object. To be more meaningful this
has been changed to 'CommandSource' so that the test checks different
TestItem types against different command sources. Otherwise it would
test TestItem types against TestItem types.

Furthermore extracting name and value of CommandSource.commandpalette
with getNamesAndValues was not successful and returned undefined. This
has been fixed by assigning CommandSource.commandpalette directly to
commandSource.value when this command source is evaluated.

* Add test for Tests_Run_Parametrized command

* Add news entry for #5608

Author: phloose

news/1 Enhancements/5608.md

@@ -0,0 +1 @@
+Add QuickPick dropdown option _Run All/Debug All_ when clicking on a Code Lens for a parametrized test to be able to run/debug all belonging test variants at once. Thanks to [Philipp Loose](https://github.com/phloose)

src/client/common/application/commands.ts

@@ -97,6 +97,7 @@ export interface ICommandNameArgumentTypeMapping extends ICommandNameWithoutArgu
     [Commands.Tests_Run]: [undefined | TestWorkspaceFolder, undefined | CommandSource, undefined | Uri, undefined | TestsToRun];
     // When command is invoked from a tree node, first argument is the node data.
     [Commands.Tests_Debug]: [undefined | TestWorkspaceFolder, undefined | CommandSource, undefined | Uri, undefined | TestsToRun];
+    [Commands.Tests_Run_Parametrized]: [undefined, undefined | CommandSource, Uri, TestFunction[], boolean];
     // When command is invoked from a tree node, first argument is the node data.
     [Commands.Tests_Discover]: [undefined | TestWorkspaceFolder, undefined | CommandSource, undefined | Uri];
     [Commands.Tests_Run_Failed]: [undefined, CommandSource, Uri];

src/client/common/constants.ts

@@ -32,6 +32,7 @@ export namespace Commands {
     export const Tests_Run_Failed = 'python.runFailedTests';
     export const Sort_Imports = 'python.sortImports';
     export const Tests_Run = 'python.runtests';
+    export const Tests_Run_Parametrized = 'python.runParametrizedTests';
     export const Tests_Debug = 'python.debugtests';
     export const Tests_Ask_To_Stop_Test = 'python.askToStopTests';
     export const Tests_Ask_To_Stop_Discovery = 'python.askToStopTestDiscovery';

src/client/testing/display/picker.ts

@@ -65,9 +65,9 @@ export class TestDisplay implements ITestDisplay {
             return fn.parentTestFile.name === testFile.name &&
                 testFunctions.some(testFunc => testFunc.nameToRun === fn.testFunction.nameToRun);
         });
-
-        this.appShell.showQuickPick(buildItemsForFunctions(rootDirectory, flattenedFunctions, undefined, undefined, debug),
-            { matchOnDescription: true, matchOnDetail: true })
+        const runAllItem = buildRunAllParametrizedItem(flattenedFunctions, debug);
+        const functionItems = buildItemsForFunctions(rootDirectory, flattenedFunctions, undefined, undefined, debug);
+        this.appShell.showQuickPick(runAllItem.concat(...functionItems), { matchOnDescription: true, matchOnDetail: true })
             .then(testItem => testItem ? onItemSelected(this.commandManager, cmdSource, wkspace, testItem, debug) : Promise.resolve());
     }
 }
@@ -84,7 +84,8 @@ export enum Type {
     Null = 8,
     SelectAndRunMethod = 9,
     DebugMethod = 10,
-    Configure = 11
+    Configure = 11,
+    RunParametrized = 12
 }
 const statusIconMapping = new Map<TestStatus, string>();
 statusIconMapping.set(TestStatus.Pass, constants.Octicons.Test_Pass);
@@ -95,6 +96,7 @@ statusIconMapping.set(TestStatus.Skipped, constants.Octicons.Test_Skip);
 type TestItem = QuickPickItem & {
     type: Type;
     fn?: FlattenedTestFunction;
+    fns?: TestFunction[];
 };
 
 type TestFileItem = QuickPickItem & {
@@ -150,6 +152,18 @@ const statusSortPrefix = {
     [TestStatus.Unknown]: undefined
 };
 
+function buildRunAllParametrizedItem(tests: FlattenedTestFunction[], debug: boolean = false): TestItem[] {
+    const testFunctions: TestFunction[] = [];
+    tests.forEach(fn => {
+        testFunctions.push(fn.testFunction);
+    });
+    return [{
+        description: '',
+        label: debug ? 'Debug All' : 'Run All',
+        type: Type.RunParametrized,
+        fns: testFunctions
+    }];
+}
 function buildItemsForFunctions(rootDirectory: string, tests: FlattenedTestFunction[], sortBasedOnResults: boolean = false, displayStatusIcons: boolean = false, debug: boolean = false): TestItem[] {
     const functionItems: TestItem[] = [];
     tests.forEach(fn => {
@@ -218,6 +232,9 @@ export function onItemSelected(commandManager: ICommandManager, cmdSource: Comma
         case Type.RunAll: {
             return commandManager.executeCommand(constants.Commands.Tests_Run, undefined, cmdSource, wkspace, undefined);
         }
+        case Type.RunParametrized: {
+            return commandManager.executeCommand(constants.Commands.Tests_Run_Parametrized, undefined, cmdSource, wkspace, selection.fns!, debug!);
+        }
         case Type.ReDiscover: {
             return commandManager.executeCommand(constants.Commands.Tests_Discover, undefined, cmdSource, wkspace);
         }

src/client/testing/main.ts

@@ -225,6 +225,13 @@ export class UnitTestManagementService implements ITestManagementService, Dispos
         const testDisplay = this.serviceContainer.get<ITestDisplay>(ITestDisplay);
         testDisplay.displayFunctionTestPickerUI(cmdSource, testManager.workspaceFolder, testManager.workingDirectory, file, testFunctions, debug);
     }
+    public async runParametrizedTests(cmdSource: CommandSource, resource: Uri, testFunctions: TestFunction[], debug?: boolean) {
+        const testManager = await this.getTestManager(true, resource);
+        if (!testManager) {
+            return;
+        }
+        await this.runTestsImpl(cmdSource, resource, { testFunction: testFunctions }, undefined, debug);
+    }
     public viewOutput(_cmdSource: CommandSource) {
         sendTelemetryEvent(EventName.UNITTEST_VIEW_OUTPUT);
         this.outputChannel.show();
@@ -398,6 +405,10 @@ export class UnitTestManagementService implements ITestManagementService, Dispos
                 constants.Commands.Tests_Picker_UI_Debug,
                 (_, cmdSource: CommandSource = CommandSource.commandPalette, file: Uri, testFunctions: TestFunction[]) => this.displayPickerUI(cmdSource, file, testFunctions, true)
             ),
+            commandManager.registerCommand(
+                constants.Commands.Tests_Run_Parametrized,
+                (_, cmdSource: CommandSource = CommandSource.commandPalette, resource: Uri, testFunctions: TestFunction[], debug: boolean) => this.runParametrizedTests(cmdSource, resource, testFunctions, debug)
+            ),
             commandManager.registerCommand(constants.Commands.Tests_Stop, (_, resource: Uri) => this.stopTests(resource)),
             commandManager.registerCommand(constants.Commands.Tests_ViewOutput, (_, cmdSource: CommandSource = CommandSource.commandPalette) => this.viewOutput(cmdSource)),
             commandManager.registerCommand(constants.Commands.Tests_Ask_To_Stop_Discovery, () => this.displayStopUI('Stop discovering tests')),

src/test/testing/display/picker.unit.test.ts

@@ -16,14 +16,29 @@ import { onItemSelected, Type } from '../../../client/testing/display/picker';
 
 suite('Unit Tests - Picker (execution of commands)', () => {
     getNamesAndValues<Type>(Type).forEach(item => {
-        getNamesAndValues<CommandSource>(Type).forEach(commandSource => {
+        getNamesAndValues<CommandSource>(CommandSource).forEach(commandSource => {
             [true, false].forEach(debug => {
                 test(`Invoking command for selection ${item.name} from ${commandSource.name} (${debug ? 'Debug' : 'No debug'})`, async () => {
                     const commandManager = mock(CommandManager);
                     const workspaceUri = Uri.file(__filename);
 
                     const testFunction = 'some test Function';
-                    const selection = { type: item.value, fn: { testFunction } };
+                    const testFunctions = [{
+                        name: 'some_name',
+                        nameToRun: 'some_name_to_run',
+                        time: 0,
+                        resource: workspaceUri
+                    }];
+                    const selection = { type: item.value, fn: { testFunction }, fns: testFunctions };
+
+                    // Getting the value of CommandSource.commandPalette in getNamesAndValues(CommandSource)
+                    // fails because the names and values object is build by accessing the CommandSource enum
+                    // properties by value. In case of commandpalette the property is commandPalette and the
+                    // respective value is commandpalette which do not match and thus return undefined for value.
+                    if (commandSource.name === 'commandpalette') {
+                        commandSource.value = CommandSource.commandPalette;
+                    }
+
                     onItemSelected(instance(commandManager), commandSource.value, workspaceUri, selection as any, debug);
 
                     switch (selection.type) {
@@ -40,6 +55,10 @@ suite('Unit Tests - Picker (execution of commands)', () => {
                             verify(commandManager.executeCommand(Commands.Tests_Run, undefined, commandSource.value, workspaceUri, undefined)).once();
                             return;
                         }
+                        case Type.RunParametrized: {
+                            verify(commandManager.executeCommand(Commands.Tests_Run_Parametrized, undefined, commandSource.value, workspaceUri, selection.fns, debug)).once();
+                            return;
+                        }
                         case Type.ReDiscover: {
                             verify(commandManager.executeCommand(Commands.Tests_Discover, undefined, commandSource.value, workspaceUri)).once();
                             return;