Use native VSCode coverage APIs, cover swift-testing tests (#807)

* Use native VSCode coverage APIs, cover swift-testing

Move to the new VSCode coverage APIs that were introduced in VSCode
version 1.88. This allows us to remove the coverage renderer and
coverage report files. This functionality is now built in to the test
explorer and VSCode's document rendering.

This patch also adds support for producing test coverage from
swift-testing tests. XCTests and swift-testing tests are each run in
their respective binaries and then the coverage data is merged into a
profdata file before being exported to LCOV format.

Instead of a build followed by a test run with --skip-build, use a
SwiftExecution task to do swift test with the appropriate arguments.
This shows the build progress in the status bar.

Finally we transform the LCOV JSON into the coverage structures for
VSCode and associate this data with the test run.

Author: plemarquand

.gitignore

@@ -5,3 +5,4 @@ default.profraw
 *.vsix
 .vscode-test
 .build
+assets/test/**/Package.resolved

docs/images/coverage-render.png

@@ -2,17 +2,14 @@
 
 Test coverage is a measurement of how much of your code is tested by your tests. It defines how many lines of code were actually run when you ran your tests and how many were not. When a line of code is not run by your tests it will not have been tested and perhaps you need to extend your tests.
 
-The Swift extension has an option to run your tests and record what code has been hit or missed by your tests.
+The Swift extension integrates with VSCode's Code Coverage APIs to record what code has been hit or missed by your tests.
 
 ![](images/coverage-run.png)
 
-Once this is run an overview report will be displayed listing all the source files in your project and how many lines were hit by tests, how many lines were missed, how lines of source code there is in total and a percentage of those that were hit. You can click on each file to open that file in Visual Studio Code. If you close the report you can always get it back by running the command `Show Test Coverage Report`. There is also a setting `Coverage: Display Report after Run` to control whether the test coverage report is shown immediately after running tests.
+Once you've performed a code coverage run a coverage report will be displayed in a section of the primary side bar. This report lists all the source files in your project and what percentage of lines were hit by tests. You can click on each file to open that file in the code editor. If you close the report you can always get it back by running the command `Test: Open Coverage`.
 
 ![](images/coverage-report.png)
 
-If you would like a more detailed view of the results there is a command `Toggle Display of Test Coverage Results` to toggle an in editor view of the coverage results. This will color the background of hit and missed lines of code with different colours. By default a hit line of code gets a green background and a missed line get a red background, although these are editable in the settings. 
+After generating code coverage lines numbers in covered files will be coloured red or green depending on if they ran during the test run. Hovering over the line numbers shows how many times each line was run. Hitting the "Toggle Inline Coverage" link that appears when hovering over the line numbers will keep this information visible.
 
 ![](images/coverage-render.png)
-
-An additional UI status item is displayed in the status bar at the bottom of the screen showing the hit percentage for the currently open file. This status item can be set to be always visible or only when coverage information is available. If you have it set to be visible all the time it can be used as a button to toggle the display of the in editor coverage results.  
-

docs/images/coverage-report.png

@@ -10,7 +10,7 @@
     "url": "https://github.com/swift-server/vscode-swift"
   },
   "engines": {
-    "vscode": "^1.71.0"
+    "vscode": "^1.88.0"
   },
   "categories": [
     "Programming Languages",
@@ -160,16 +160,6 @@
         "title": "Insert Function Comment",
         "category": "Swift"
       },
-      {
-        "command": "swift.showTestCoverageReport",
-        "title": "Show Test Coverage Report",
-        "category": "Swift"
-      },
-      {
-        "command": "swift.toggleTestCoverage",
-        "title": "Toggle Display of Test Coverage Results",
-        "category": "Swift"
-      },
       {
         "command": "swift.attachDebugger",
         "title": "Attach to Process...",
@@ -453,47 +443,6 @@
           }
         }
       },
-      {
-        "title": "Test Coverage",
-        "properties": {
-          "swift.coverage.displayReportAfterRun": {
-            "type": "boolean",
-            "default": true,
-            "description": "Should test coverage report be shown after running tests",
-            "order": 1
-          },
-          "swift.coverage.alwaysShowStatusItem": {
-            "type": "boolean",
-            "default": true,
-            "description": "Always show the test coverage status item. If this is set to true the status item can be clicked on to toggle test coverage display on and off.",
-            "order": 2
-          },
-          "swift.coverage.colors.lightMode.hit": {
-            "type": "string",
-            "default": "#c0ffc0",
-            "description": "Light mode theme background color for line of code hit during test coverage.",
-            "order": 3
-          },
-          "swift.coverage.colors.lightMode.miss": {
-            "type": "string",
-            "default": "#ffc0c0",
-            "description": "Light mode theme background color for line of code missed during test coverage.",
-            "order": 4
-          },
-          "swift.coverage.colors.darkMode.hit": {
-            "type": "string",
-            "default": "#003000",
-            "description": "Dark mode theme background color for line of code hit during test coverage.",
-            "order": 5
-          },
-          "swift.coverage.colors.darkMode.miss": {
-            "type": "string",
-            "default": "#400000",
-            "description": "Dark mode theme background color for line of code missed during test coverage.",
-            "order": 6
-          }
-        }
-      },
       {
         "title": "Debugger",
         "properties": {
@@ -1177,7 +1126,7 @@
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.19.33",
     "@types/plist": "^3.0.5",
-    "@types/vscode": "^1.71.0",
+    "@types/vscode": "^1.88.0",
     "@typescript-eslint/eslint-plugin": "^7.8.0",
     "@typescript-eslint/parser": "^7.8.0",
     "@vscode/test-electron": "^2.3.8",

docs/images/coverage-run.png

@@ -21,7 +21,6 @@ import { TestExplorer } from "./TestExplorer/TestExplorer";
 import { WorkspaceContext, FolderEvent } from "./WorkspaceContext";
 import { BackgroundCompilation } from "./BackgroundCompilation";
 import { TaskQueue } from "./tasks/TaskQueue";
-import { LcovResults } from "./coverage/LcovResults";
 import { isPathInsidePath } from "./utilities/utilities";
 
 export class FolderContext implements vscode.Disposable {
@@ -30,7 +29,6 @@ export class FolderContext implements vscode.Disposable {
     public hasResolveErrors = false;
     public testExplorer?: TestExplorer;
     public taskQueue: TaskQueue;
-    public lcovResults: LcovResults;
 
     /**
      * FolderContext constructor
@@ -49,15 +47,13 @@ export class FolderContext implements vscode.Disposable {
         this.packageWatcher.install();
         this.backgroundCompilation = new BackgroundCompilation(this);
         this.taskQueue = new TaskQueue(this);
-        this.lcovResults = new LcovResults(this);
     }
 
     /** dispose of any thing FolderContext holds */
     dispose() {
         this.linuxMain?.dispose();
         this.packageWatcher.dispose();
         this.testExplorer?.dispose();
-        this.lcovResults.dispose();
     }
 
     /**

docs/test-coverage.md

@@ -14,6 +14,7 @@
 
 import { ITestRunState } from "./TestRunState";
 import { sourceLocationToVSCodeLocation } from "../../utilities/utilities";
+import { MarkdownString, Location } from "vscode";
 
 /** Regex for parsing XCTest output */
 interface TestRegex {
@@ -67,7 +68,69 @@ export const nonDarwinTestRegex = {
     failedSuite: /^Test Suite '(.*)' failed/,
 };
 
-export class XCTestOutputParser {
+export interface IXCTestOutputParser {
+    parseResult(output: string, runState: ITestRunState): void;
+}
+
+export class ParallelXCTestOutputParser implements IXCTestOutputParser {
+    private outputParser: XCTestOutputParser;
+
+    /**
+     * Create an ParallelXCTestOutputParser.
+     * Optional regex can be supplied for tests.
+     */
+    constructor(
+        private hasMultiLineParallelTestOutput: boolean,
+        regex?: TestRegex
+    ) {
+        this.outputParser = new XCTestOutputParser(regex);
+    }
+
+    public parseResult(output: string, runState: ITestRunState) {
+        // From 5.7 to 5.10 running with the --parallel option dumps the test results out
+        // to the console with no newlines, so it isn't possible to distinguish where errors
+        // begin and end. Consequently we can't record them, and so we manually mark them
+        // as passed or failed here with a manufactured issue.
+        // Don't attempt to parse the console output of parallel tests between 5.7 and 5.10
+        // as it doesn't have newlines. You might get lucky and find the output is split
+        // in the right spot, but more often than not we wont be able to parse it.
+        if (!this.hasMultiLineParallelTestOutput) {
+            return;
+        }
+
+        // For parallel XCTest runs we get pass/fail results from the xunit XML
+        // produced at the end of the run, but we still want to monitor the output
+        // for the individual assertion failures. Wrap the run state and only forward
+        // along the issues captured during a test run, and let the `TestXUnitParser`
+        // handle marking tests as completed.
+        this.outputParser.parseResult(output, new ParallelXCTestRunStateProxy(runState));
+    }
+}
+
+/* eslint-disable @typescript-eslint/no-unused-vars */
+class ParallelXCTestRunStateProxy implements ITestRunState {
+    constructor(private runState: ITestRunState) {}
+
+    getTestItemIndex(id: string, filename: string | undefined): number {
+        return this.runState.getTestItemIndex(id, filename);
+    }
+    recordIssue(
+        index: number,
+        message: string | MarkdownString,
+        location?: Location | undefined
+    ): void {
+        this.runState.recordIssue(index, message, location);
+    }
+    started(index: number, startTime?: number | undefined): void {}
+    completed(index: number, timing: { duration: number } | { timestamp: number }): void {}
+    skipped(index: number): void {}
+    startedSuite(name: string): void {}
+    passedSuite(name: string): void {}
+    failedSuite(name: string): void {}
+}
+/* eslint-enable @typescript-eslint/no-unused-vars */
+
+export class XCTestOutputParser implements IXCTestOutputParser {
     private regex: TestRegex;
 
     /**
@@ -93,7 +156,7 @@ export class XCTestOutputParser {
             lines.pop();
         }
         // if submitted text does not end with a newline then pop that off and store in excess
-        // for next call of parseResultDarwin
+        // for next call of parseResult
         if (output2[output2.length - 1] !== "\n") {
             runState.excess = lines.pop();
         } else {