Add support for swift-testing (#775)

* Add support for swift-testing

Support running swift-testing tests the same as XCTests. If a test run
has both types, swift-testing tests will be run first followed by
XCTests.

First a list of XCTests and swift-testing tests to run is parsed from
the test request. Test type is determined by a tag on each
`vscode.TestItem[]`, either `"XCTest"` or `"swift-testing"`.

swift-testing tests are launched by running the binary named
<PackageName>PackageTests.swift-testing inside the build debug folder.
This binary is run with the `--experimental-event-stream-output` flag
which forwards test events (test started, complete, issue recorded, etc)
to a named pipe. The `SwiftTestingOutputParser` watches this pipe for
events as the tests are being run and translates them in to
`ITestRunner` calls to record test progress in VSCode.

There are different named pipe reader implementations between
macOS/Linux and Windows.

Author: plemarquand

src/TestExplorer/TestDiscovery.ts

@@ -16,6 +16,7 @@ import * as vscode from "vscode";
 import { FolderContext } from "../FolderContext";
 import { TargetType } from "../SwiftPackage";
 import { LSPTestItem } from "../sourcekit-lsp/lspExtensions";
+import { reduceTestItemChildren } from "./TestUtils";
 
 /** Test class definition */
 export interface TestClass extends Omit<Omit<LSPTestItem, "location">, "children"> {
@@ -76,7 +77,11 @@ export function updateTests(
             (!filterFile || testItem.uri?.fsPath === filterFile.fsPath)
         ) {
             const collection = testItem.parent ? testItem.parent.children : testController.items;
-            collection.delete(testItem.id);
+
+            // TODO: This needs to take in to account parameterized tests with no URI, when they're added.
+            if (testItem.children.size === 0) {
+                collection.delete(testItem.id);
+            }
         }
     }
 
@@ -113,6 +118,24 @@ function createIncomingTestLookup(
     return dictionary;
 }
 
+/**
+ * Merges the TestItems recursively from the `existingItem` in to the `newItem`
+ */
+function deepMergeTestItemChildren(existingItem: vscode.TestItem, newItem: vscode.TestItem) {
+    reduceTestItemChildren(
+        existingItem.children,
+        (collection, testItem: vscode.TestItem) => {
+            const existing = collection.get(testItem.id);
+            if (existing) {
+                deepMergeTestItemChildren(existing, testItem);
+            }
+            collection.add(testItem);
+            return collection;
+        },
+        newItem.children
+    );
+}
+
 /**
  * Updates the existing `vscode.TestItem` if it exists with the same ID as the `TestClass`,
  * otherwise creates an add a new one. The location on the returned vscode.TestItem is always updated.
@@ -122,12 +145,6 @@ function upsertTestItem(
     testItem: TestClass,
     parent?: vscode.TestItem
 ) {
-    // This is a temporary gate on adding swift-testing tests until there is code to
-    // run them. See https://github.com/swift-server/vscode-swift/issues/757
-    if (testItem.style === "swift-testing") {
-        return;
-    }
-
     const collection = parent?.children ?? testController.items;
     const existingItem = collection.get(testItem.id);
     let newItem: vscode.TestItem;
@@ -148,11 +165,26 @@ function upsertTestItem(
         newItem = existingItem;
     }
 
+    // At this point all the test items that should have been deleted are out of the tree.
+    // Its possible we're dropping a whole branch of test items on top of an existing one,
+    // and we want to merge these branches instead of the new one replacing the existing one.
+    if (existingItem) {
+        deepMergeTestItemChildren(existingItem, newItem);
+    }
+
     // Manually add the test style as a tag so we can filter by test type.
     newItem.tags = [{ id: testItem.style }, ...testItem.tags];
     newItem.label = testItem.label;
     newItem.range = testItem.location?.range;
 
+    if (testItem.sortText) {
+        newItem.sortText = testItem.sortText;
+    } else if (!testItem.location) {
+        // TestItems without a location should be sorted to the top.
+        const zeros = ``.padStart(8, "0");
+        newItem.sortText = `${zeros}:${testItem.label}`;
+    }
+
     // Performs an upsert based on whether a test item exists in the collection with the same id.
     // If no parent is provided operate on the testController's root items.
     collection.add(newItem);

src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts

@@ -0,0 +1,193 @@
+import * as readline from "readline";
+import { Readable } from "stream";
+import {
+    INamedPipeReader,
+    UnixNamedPipeReader,
+    WindowsNamedPipeReader,
+} from "./TestEventStreamReader";
+import { ITestRunState } from "./TestRunState";
+
+// All events produced by a swift-testing run will be one of these three types.
+export type SwiftTestEvent = MetadataRecord | TestRecord | EventRecord;
+
+interface VersionedRecord {
+    version: number;
+}
+
+interface MetadataRecord extends VersionedRecord {
+    kind: "metadata";
+    payload: Metadata;
+}
+
+interface TestRecord extends VersionedRecord {
+    kind: "test";
+    payload: Test;
+}
+
+export type EventRecordPayload =
+    | RunStarted
+    | TestStarted
+    | TestEnded
+    | TestCaseStarted
+    | TestCaseEnded
+    | IssueRecorded
+    | TestSkipped
+    | RunEnded;
+
+export interface EventRecord extends VersionedRecord {
+    kind: "event";
+    payload: EventRecordPayload;
+}
+
+interface Metadata {
+    [key: string]: object; // Currently unstructured content
+}
+
+interface Test {
+    kind: "suite" | "function" | "parameterizedFunction";
+    id: string;
+    name: string;
+    _testCases?: TestCase[];
+    sourceLocation: SourceLocation;
+}
+
+interface TestCase {
+    id: string;
+    displayName: string;
+}
+
+// Event types
+interface RunStarted {
+    kind: "runStarted";
+}
+
+interface RunEnded {
+    kind: "runEnded";
+}
+
+interface Instant {
+    absolute: number;
+    since1970: number;
+}
+
+interface BaseEvent {
+    instant: Instant;
+    messages: EventMessage[];
+    testID: string;
+}
+
+interface TestStarted extends BaseEvent {
+    kind: "testStarted";
+}
+
+interface TestEnded extends BaseEvent {
+    kind: "testEnded";
+}
+
+interface TestCaseStarted extends BaseEvent {
+    kind: "testCaseStarted";
+}
+
+interface TestCaseEnded extends BaseEvent {
+    kind: "testCaseEnded";
+}
+
+interface TestSkipped extends BaseEvent {
+    kind: "testSkipped";
+}
+
+interface IssueRecorded extends BaseEvent {
+    kind: "issueRecorded";
+    issue: {
+        sourceLocation: SourceLocation;
+    };
+}
+
+export interface EventMessage {
+    text: string;
+}
+
+export interface SourceLocation {
+    _filePath: string;
+    line: number;
+    column: number;
+}
+
+export class SwiftTestingOutputParser {
+    private completionMap = new Map<number, boolean>();
+
+    /**
+     * Watches for test events on the named pipe at the supplied path.
+     * As events are read they are parsed and recorded in the test run state.
+     */
+    public async watch(
+        path: string,
+        runState: ITestRunState,
+        pipeReader?: INamedPipeReader
+    ): Promise<void> {
+        // Creates a reader based on the platform unless being provided in a test context.
+        const reader = pipeReader ?? this.createReader(path);
+        const readlinePipe = new Readable({
+            read() {},
+        });
+
+        // Use readline to automatically chunk the data into lines,
+        // and then take each line and parse it as JSON.
+        const rl = readline.createInterface({
+            input: readlinePipe,
+            crlfDelay: Infinity,
+        });
+
+        rl.on("line", line => this.parse(JSON.parse(line), runState));
+
+        reader.start(readlinePipe);
+    }
+
+    private createReader(path: string): INamedPipeReader {
+        return process.platform === "win32"
+            ? new WindowsNamedPipeReader(path)
+            : new UnixNamedPipeReader(path);
+    }
+
+    private testName(id: string): string {
+        const nameMatcher = /^(.*\(.*\))\/(.*)\.swift:\d+:\d+$/;
+        const matches = id.match(nameMatcher);
+        return !matches ? id : matches[1];
+    }
+
+    private parse(item: SwiftTestEvent, runState: ITestRunState) {
+        if (item.kind === "event") {
+            if (item.payload.kind === "testCaseStarted" || item.payload.kind === "testStarted") {
+                const testName = this.testName(item.payload.testID);
+                const testIndex = runState.getTestItemIndex(testName, undefined);
+                runState.started(testIndex, item.payload.instant.absolute);
+            } else if (item.payload.kind === "testSkipped") {
+                const testName = this.testName(item.payload.testID);
+                const testIndex = runState.getTestItemIndex(testName, undefined);
+                runState.skipped(testIndex);
+            } else if (item.payload.kind === "issueRecorded") {
+                const testName = this.testName(item.payload.testID);
+                const testIndex = runState.getTestItemIndex(testName, undefined);
+                const sourceLocation = item.payload.issue.sourceLocation;
+                item.payload.messages.forEach(message => {
+                    runState.recordIssue(testIndex, message.text, {
+                        file: sourceLocation._filePath,
+                        line: sourceLocation.line,
+                        column: sourceLocation.column,
+                    });
+                });
+            } else if (item.payload.kind === "testCaseEnded" || item.payload.kind === "testEnded") {
+                const testName = this.testName(item.payload.testID);
+                const testIndex = runState.getTestItemIndex(testName, undefined);
+
+                // When running a single test the testEnded and testCaseEnded events
+                // have the same ID, and so we'd end the same test twice.
+                if (this.completionMap.get(testIndex)) {
+                    return;
+                }
+                this.completionMap.set(testIndex, true);
+                runState.completed(testIndex, { timestamp: item.payload.instant.absolute });
+            }
+        }
+    }
+}

src/TestExplorer/TestParsers/TestEventStreamReader.ts

@@ -0,0 +1,63 @@
+import * as fs from "fs";
+import * as net from "net";
+import { Readable } from "stream";
+
+export interface INamedPipeReader {
+    start(readable: Readable): Promise<void>;
+}
+
+/**
+ * Reads from a named pipe on Windows and forwards data to a `Readable` stream.
+ * Note that the path must be in the Windows named pipe format of `\\.\pipe\pipename`.
+ */
+export class WindowsNamedPipeReader implements INamedPipeReader {
+    constructor(private path: string) {}
+
+    public async start(readable: Readable) {
+        return new Promise<void>((resolve, reject) => {
+            try {
+                const server = net.createServer(function (stream) {
+                    stream.on("data", data => readable.push(data));
+                    stream.on("error", () => server.close());
+                    stream.on("end", function () {
+                        readable.push(null);
+                        server.close();
+                    });
+                });
+
+                server.listen(this.path, () => resolve());
+            } catch (error) {
+                reject(error);
+            }
+        });
+    }
+}
+
+/**
+ * Reads from a unix FIFO pipe and forwards data to a `Readable` stream.
+ * Note that the pipe at the supplied path should be created with `mkfifo`
+ * before calling `start()`.
+ */
+export class UnixNamedPipeReader implements INamedPipeReader {
+    constructor(private path: string) {}
+
+    public async start(readable: Readable) {
+        return new Promise<void>((resolve, reject) => {
+            fs.open(this.path, fs.constants.O_RDONLY | fs.constants.O_NONBLOCK, (err, fd) => {
+                try {
+                    const pipe = new net.Socket({ fd, readable: true });
+                    pipe.on("data", data => readable.push(data));
+                    pipe.on("error", () => fs.close(fd));
+                    pipe.on("end", () => {
+                        readable.push(null);
+                        fs.close(fd);
+                    });
+
+                    resolve();
+                } catch (error) {
+                    reject(error);
+                }
+            });
+        });
+    }
+}

src/TestExplorer/TestParsers/TestRunState.ts

@@ -0,0 +1,47 @@
+import { MarkdownString } from "vscode";
+
+/**
+ * Interface for setting this test runs state
+ */
+export interface ITestRunState {
+    // excess data from previous parse that was not processed
+    excess?: string;
+    // failed test state
+    failedTest?: {
+        testIndex: number;
+        message: string;
+        file: string;
+        lineNumber: number;
+        complete: boolean;
+    };
+
+    // get test item index from test name on non Darwin platforms
+    getTestItemIndex(id: string, filename: string | undefined): number;
+
+    // set test index to be started
+    started(index: number, startTime?: number): void;
+
+    // set test index to have passed.
+    // If a start time was provided to `started` then the duration is computed as endTime - startTime,
+    // otherwise the time passed is assumed to be the duration.
+    completed(index: number, timing: { duration: number } | { timestamp: number }): void;
+
+    // record an issue against a test
+    recordIssue(
+        index: number,
+        message: string | MarkdownString,
+        location?: { file: string; line: number; column?: number }
+    ): void;
+
+    // set test index to have been skipped
+    skipped(index: number): void;
+
+    // started suite
+    startedSuite(name: string): void;
+
+    // passed suite
+    passedSuite(name: string): void;
+
+    // failed suite
+    failedSuite(name: string): void;
+}