Use XCTestRun to execute tests and report results

A major goal for swift-corelibs-xctest is API parity with Apple XCTest.
This adds the largest missing API in swift-corelibs-xctest: `XCTestRun`.

In Apple XCTest, `XCTestRun` is responsible for keeping track of the
result of a test run. It's an integral part of how Apple XCTest works.
swift-corelibs-xctest, on the other hand, used a global array of
`XCTRun` structs to keep track of how many tests passed/failed.

While it may have been possible to tack on `XCTestRun` to the
swift-corelibs-xctest mechanism for failure reporting, this commit
instead fully integrates it. As a result, the changes are widespread:
gone is `XCTFailureHandler`, `XCTRun`, and other internal structures.
In their place, welcome the Apple XCTest public APIs: the `XCTest` abstract
class, `XCTestRun`, and its subclasses `XCTestCaseRun` and
`XCTestSuiteRun`.

In conjunction with the new `XCTestSuite`-related observation methods
from swiftlang#84,
test reporting is now done exclusively through `XCTestObservation`.
As a result, test output is now nearly identical to Apple XCTest.

Author: modocache

Sources/XCTest/PrintObserver.swift

@@ -0,0 +1,77 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  PrintObserver.swift
+//  Prints test progress to stdout.
+//
+
+#if os(Linux) || os(FreeBSD)
+    import Foundation
+#else
+    import SwiftFoundation
+#endif
+
+/// Prints textual representations of each XCTestObservation event to stdout.
+/// Mirrors the Apple XCTest output exactly.
+internal class PrintObserver: XCTestObservation {
+    func testBundleWillStart(testBundle: NSBundle) {}
+
+    func testSuiteWillStart(testSuite: XCTestSuite) {
+        printAndFlush("Test Suite '\(testSuite.name)' started at \(dateFormatter.stringFromDate(testSuite.testRun!.startDate!))")
+    }
+
+    func testCaseWillStart(testCase: XCTestCase) {
+        printAndFlush("Test Case '\(testCase.name)' started at \(dateFormatter.stringFromDate(testCase.testRun!.startDate!))")
+    }
+
+    func testCase(testCase: XCTestCase, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: UInt) {
+        let file = filePath ?? "<unknown>"
+        printAndFlush("\(file):\(lineNumber): error: \(description)")
+    }
+
+    func testCaseDidFinish(testCase: XCTestCase) {
+        let testRun = testCase.testRun!
+        let verb = testRun.hasSucceeded ? "passed" : "failed"
+        // FIXME: Apple XCTest does not print a period after "(N seconds)".
+        //        The trailing period here should be removed and the functional
+        //        test suite should be updated.
+        printAndFlush("Test Case '\(testCase.name)' \(verb) (\(formatTimeInterval(testRun.totalDuration)) seconds).")
+    }
+
+    func testSuiteDidFinish(testSuite: XCTestSuite) {
+        let testRun = testSuite.testRun!
+        let verb = testRun.hasSucceeded ? "passed" : "failed"
+        printAndFlush("Test Suite '\(testSuite.name)' \(verb) at \(dateFormatter.stringFromDate(testRun.stopDate!))")
+
+        let tests = testRun.executionCount == 1 ? "test" : "tests"
+        let failures = testRun.totalFailureCount == 1 ? "failure" : "failures"
+        printAndFlush(
+            "\t Executed \(testRun.executionCount) \(tests), " +
+            "with \(testRun.totalFailureCount) \(failures) (\(testRun.unexpectedExceptionCount) unexpected) " +
+            "in \(formatTimeInterval(testRun.testDuration)) (\(formatTimeInterval(testRun.totalDuration))) seconds"
+        )
+    }
+
+    func testBundleDidFinish(testBundle: NSBundle) {}
+
+    private lazy var dateFormatter: NSDateFormatter = {
+        let formatter = NSDateFormatter()
+        formatter.dateFormat = "HH:mm:ss.SSS"
+        return formatter
+    }()
+
+    private func printAndFlush(message: String) {
+        print(message)
+        fflush(stdout)
+    }
+
+    private func formatTimeInterval(timeInterval: NSTimeInterval) -> String {
+        return String(round(timeInterval * 1000.0) / 1000.0)
+    }
+}

Sources/XCTest/XCAbstractTest.swift

@@ -26,11 +26,46 @@ public class XCTest {
         fatalError("Must be overridden by subclasses.")
     }
 
+    /// The `XCTestRun` subclass that will be instantiated when the test is run
+    /// to hold the test's results. Must be overridden by subclasses.
+    public var testRunClass: AnyClass {
+        fatalError("Must be overridden by subclasses.")
+    }
+
+    /// The test run object that executed the test, an instance of
+    /// testRunClass. If the test has not yet been run, this will be nil.
+    /// - Note: FIXME: This property is meant to be `internal(set)`. It is
+    ///   publicly settable for now due to a Swift compiler bug on Linux. To
+    ///   ensure compatibility of tests between swift-corelibs-xctest and Apple
+    ///   XCTest, you should not set this property. See
+    ///   https://bugs.swift.org/browse/SR-1129 for details.
+    public public(set) var testRun: XCTestRun? = nil
+
+    /// The method through which tests are executed. Must be overridden by
+    /// subclasses.
+    public func performTest(run: XCTestRun) {
+        fatalError("Must be overridden by subclasses.")
+    }
+
+    /// Creates an instance of the `testRunClass` and passes it as a parameter
+    /// to `performTest()`.
+    public func runTest() {
+        guard let testRunType = testRunClass as? XCTestRun.Type else {
+            fatalError("XCTest.testRunClass must be a kind of XCTestRun.")
+        }
+        testRun = testRunType.init(test: self)
+        performTest(testRun!)
+    }
+
     /// Setup method called before the invocation of each test method in the
     /// class.
     public func setUp() {}
 
     /// Teardown method called after the invocation of each test method in the
     /// class.
     public func tearDown() {}
+
+    // FIXME: This initializer is required due to a Swift compiler bug on Linux.
+    //        It should be removed once the bug is fixed.
+    public init() {}
 }

Sources/XCTest/XCTAssert.swift

@@ -87,8 +87,12 @@ private func _XCTEvaluateAssertion(assertion: _XCTAssertion, @autoclosure messag
     case .Success:
         return
     default:
-        if let handler = XCTFailureHandler {
-            handler(XCTFailure(message: message(), failureDescription: result.failureDescription(assertion), expected: result.expected, file: file, line: line))
+        if let currentTestCase = XCTCurrentTestCase {
+            currentTestCase.recordFailureWithDescription(
+                "\(result.failureDescription(assertion)) - \(message())",
+                inFile: String(file),
+                atLine: line,
+                expected: result.expected)
         }
     }
 }

Sources/XCTest/XCTestCase.swift

@@ -24,7 +24,12 @@
 /// - seealso: `XCTMain`
 public typealias XCTestCaseEntry = (testCaseClass: XCTestCase.Type, allTests: [(String, XCTestCase throws -> Void)])
 
+// A global pointer to the currently running test case. This is required in
+// order for XCTAssert functions to report failures.
+internal var XCTCurrentTestCase: XCTestCase?
+
 public class XCTestCase: XCTest {
+    private let testClosure: XCTestCase throws -> Void
 
     /// The name of the test case, consisting of its class name and the method
     /// name it will run.
@@ -35,11 +40,79 @@ public class XCTestCase: XCTest {
     /// - Note: FIXME: This property should be readonly, but currently has to
     ///   be publicly settable due to a Swift compiler bug on Linux. To ensure
     ///   compatibility of tests between swift-corelibs-xctest and Apple XCTest,
-    ///   this property should not be modified.
+    ///   this property should not be modified. See
+    ///   https://bugs.swift.org/browse/SR-1129 for details.
     public var _name: String
 
-    public required override init() {
-        _name = "\(self.dynamicType).<unknown>"
+    public override var testRunClass: AnyClass {
+        return XCTestCaseRun.self
+    }
+
+    public override func performTest(run: XCTestRun) {
+        guard let testRun = run as? XCTestCaseRun else {
+            fatalError("Wrong XCTestRun class.")
+        }
+
+        XCTCurrentTestCase = self
+        testRun.start()
+        invokeTest()
+        failIfExpectationsNotWaitedFor(XCTAllExpectations)
+        XCTAllExpectations = []
+        testRun.stop()
+        XCTCurrentTestCase = nil
+    }
+
+    /// The designated initializer for SwiftXCTest's XCTestCase.
+    /// - Note: Like the designated initializer for Apple XCTest's XCTestCase,
+    ///   `-[XCTestCase initWithInvocation:]`, it's rare for anyone outside of
+    ///   XCTest itself to call this initializer.
+    public required init(name: String, testClosure: XCTestCase throws -> Void) {
+        _name = "\(self.dynamicType).\(name)"
+        self.testClosure = testClosure
+    }
+
+    /// Invoking a test performs its setUp, invocation, and tearDown. In
+    /// general this should not be called directly.
+    public func invokeTest() {
+        setUp()
+        do {
+            try testClosure(self)
+        } catch {
+            recordFailureWithDescription(
+                "threw error \"\(error)\"",
+                inFile: "<EXPR>",
+                atLine: 0,
+                expected: false)
+        }
+        tearDown()
+    }
+
+    /// Records a failure in the execution of the test and is used by all test
+    /// assertions.
+    /// - Parameter description: The description of the failure being reported.
+    /// - Parameter filePath: The file path to the source file where the failure
+    ///   being reported was encountered.
+    /// - Parameter lineNumber: The line number in the source file at filePath
+    ///   where the failure being reported was encountered.
+    /// - Parameter expected: `true` if the failure being reported was the
+    ///   result of a failed assertion, `false` if it was the result of an
+    ///   uncaught exception.
+    public func recordFailureWithDescription(description: String, inFile filePath: String, atLine lineNumber: UInt, expected: Bool) {
+        testRun?.recordFailureWithDescription(
+            "\(name) : \(description)",
+            inFile: filePath,
+            atLine: lineNumber,
+            expected: expected)
+
+        // FIXME: Apple XCTest does not throw a fatal error and crash the test
+        //        process, it merely prevents the remainder of a testClosure
+        //        from execting after it's been determined that it has already
+        //        failed. The following behavior is incorrect.
+        // FIXME: No regression tests exist for this feature. We may break it
+        //        without ever realizing.
+        if !continueAfterFailure {
+            fatalError("Terminating execution due to test failure")
+        }
     }
 }
 
@@ -81,90 +154,15 @@ extension XCTestCase {
         }
     }
 
-    internal static func invokeTests(tests: [(String, XCTestCase throws -> Void)]) {
-        let observationCenter = XCTestObservationCenter.shared()
-
-        var totalDuration = 0.0
-        var totalFailures = 0
-        var unexpectedFailures = 0
-        let overallDuration = measureTimeExecutingBlock {
-            for (name, test) in tests {
-                let testCase = self.init()
-                testCase._name = "\(testCase.dynamicType).\(name)"
-
-                var failures = [XCTFailure]()
-                XCTFailureHandler = { failure in
-                    observationCenter.testCase(testCase,
-                                               didFailWithDescription: failure.failureMessage,
-                                               inFile: String(failure.file),
-                                               atLine: failure.line)
-
-                    if !testCase.continueAfterFailure {
-                        failure.emit(testCase.name)
-                        fatalError("Terminating execution due to test failure", file: failure.file, line: failure.line)
-                    } else {
-                        failures.append(failure)
-                    }
-                }
-
-                XCTPrint("Test Case '\(testCase.name)' started.")
-
-                observationCenter.testCaseWillStart(testCase)
-
-                testCase.setUp()
-
-                let duration = measureTimeExecutingBlock {
-                    do {
-                        try test(testCase)
-                    } catch {
-                        let unexpectedFailure = XCTFailure(message: "", failureDescription: "threw error \"\(error)\"", expected: false, file: "<EXPR>", line: 0)
-                        XCTFailureHandler!(unexpectedFailure)
-                    }
-                }
-
-                testCase.tearDown()
-                testCase.failIfExpectationsNotWaitedFor(XCTAllExpectations)
-                XCTAllExpectations = []
-
-                observationCenter.testCaseDidFinish(testCase)
-
-                totalDuration += duration
-
-                var result = "passed"
-                for failure in failures {
-                    failure.emit(testCase.name)
-                    totalFailures += 1
-                    if !failure.expected {
-                        unexpectedFailures += 1
-                    }
-                    result = failures.count > 0 ? "failed" : "passed"
-                }
-
-                XCTPrint("Test Case '\(testCase.name)' \(result) (\(printableStringForTimeInterval(duration)) seconds).")
-                XCTAllRuns.append(XCTRun(duration: duration, method: testCase.name, passed: failures.count == 0, failures: failures))
-                XCTFailureHandler = nil
-            }
-        }
-
-        let testCountSuffix = (tests.count == 1) ? "" : "s"
-        let failureSuffix = (totalFailures == 1) ? "" : "s"
-
-        XCTPrint("Executed \(tests.count) test\(testCountSuffix), with \(totalFailures) failure\(failureSuffix) (\(unexpectedFailures) unexpected) in \(printableStringForTimeInterval(totalDuration)) (\(printableStringForTimeInterval(overallDuration))) seconds")
-    }
-
     /// It is an API violation to create expectations but not wait for them to
     /// be completed. Notify the user of a mistake via a test failure.
     private func failIfExpectationsNotWaitedFor(expectations: [XCTestExpectation]) {
         if expectations.count > 0 {
-            let failure = XCTFailure(
-                message: "Failed due to unwaited expectations.",
-                failureDescription: "",
-                expected: false,
-                file: expectations.last!.file,
-                line: expectations.last!.line)
-            if let failureHandler = XCTFailureHandler {
-                failureHandler(failure)
-            }
+            recordFailureWithDescription(
+                "Failed due to unwaited expectations.",
+                inFile: String(expectations.last!.file),
+                atLine: expectations.last!.line,
+                expected: false)
         }
     }
 
@@ -232,15 +230,11 @@ extension XCTestCase {
         //        and executes the rest of the test. This discrepancy should be
         //        fixed.
         if XCTAllExpectations.count == 0 {
-            let failure = XCTFailure(
-                message: "call made to wait without any expectations having been set.",
-                failureDescription: "API violation",
-                expected: false,
-                file: file,
-                line: line)
-            if let failureHandler = XCTFailureHandler {
-                failureHandler(failure)
-            }
+            recordFailureWithDescription(
+                "API violation - call made to wait without any expectations having been set.",
+                inFile: String(file),
+                atLine: line,
+                expected: false)
             return
         }
 
@@ -280,15 +274,11 @@ extension XCTestCase {
             // Not all expectations were fulfilled. Append a failure
             // to the array of expectation-based failures.
             let descriptions = unfulfilledDescriptions.joined(separator: ", ")
-            let failure = XCTFailure(
-                message: "Exceeded timeout of \(timeout) seconds, with unfulfilled expectations: \(descriptions)",
-                failureDescription: "Asynchronous wait failed",
-                expected: true,
-                file: file,
-                line: line)
-            if let failureHandler = XCTFailureHandler {
-                failureHandler(failure)
-            }
+            recordFailureWithDescription(
+                "Asynchronous wait failed - Exceeded timeout of \(timeout) seconds, with unfulfilled expectations: \(descriptions)",
+                inFile: String(file),
+                atLine: line,
+                expected: true)
         }
 
         // We've recorded all the failures; clear the expectations that