[XCTestCase] Add asynchronous testing API

modocache · modocache · commit 61112cf4f6b8 · 2016-02-07T01:40:14.000-05:00
Mirror the Objective-C XCTest API to add asynchronous testing support,
primarily via two methods:

- `XCTestCase.expectationWithDescription()`
- `XCTestCase.waitForExpectationsWithTimeout()`

These methods' APIs are identical to their Objective-C SDK overlay
counterparts, with the following exceptions:

1. If expectations are created but not waited for, Objective-C XCTest
   generates a test failure. In order to display a failure on the exact
   line of the last expectation that was created, Objective-C XCTest
   symbolicates the call stack. It does so using a private framework
   called `CoreSymbolication.framework`. We don't have this at our
   disposal for swift-corelibs-xctest, so instead we take `file` and
   `line` as parameters, defaulting to `__FILE__` and `__LINE__`. By
   doing so, we provide the same failure highlighting functionality
   while also maintaining a (nearly) identical API.
2. If `waitForExpectationsWithTimeout()` fails, Objective-C XCTest
   generates a test failure on the exact line that method was called,
   using the same technique from (1). For the same reason, this method
   also takes `file` and `line` parameters in swift-corelibs-xctest.
3. `waitForExpectationsWithTimeout()` takes a handler callback, which
   in Objective-C XCTest is passed an instance of `NSError`.
   swift-corelibs-xctest doesn't depend upon swift-corelibs-foundation,
   and so does not have access to `NSError`. Instead, it defines a shim,
   `XCTError`.

In order to implement the asynchronous APIs, swift-corelibs-xctest now
has a dependency upon swift-corelibs-foundation. Bump minimum deployment
targets for XCTest to the same required by Foundation.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 .DS_Store
 xcuserdata
+*.xcscmblueprint
 .build/
 Output/
diff --git a/Sources/XCTest/XCTestCase.swift b/Sources/XCTest/XCTestCase.swift
@@ -11,11 +11,25 @@
 //  Base protocol (and extension with default methods) for test cases
 //
 
+#if os(Linux) || os(FreeBSD)
+    import Foundation
+#else
+    import SwiftFoundation
+#endif
+
 public protocol XCTestCase : XCTestCaseProvider {
     func setUp()
     func tearDown()
 }
 
+/// A block to be invoked when a call to wait times out or has had all
+/// associated expectations fulfilled.
+///
+/// - Parameter error: If the wait timed out or a failure was raised while
+///   waiting, the error's code will specify the type of failure. Otherwise
+///   error will be nil.
+public typealias XCWaitCompletionHandler = (NSError?) -> ()
+
 extension XCTestCase {
     
     public var continueAfterFailure: Bool {
@@ -61,6 +75,23 @@ extension XCTestCase {
 
                 tearDown()
 
+                // It is an API violation to create expectations but not wait
+                // for them to be completed. Notify the user of a mistake via
+                // a test failure.
+                if XCTAllExpectations.count > 0 {
+                    let failure = XCTFailure(
+                        message: "Failed due to unwaited expectations.",
+                        failureDescription: "",
+                        expected: true,
+                        file: XCTLatestExpectationLocation.file,
+                        line: XCTLatestExpectationLocation.line)
+                    XCTAllExpectationFailures.append(failure)
+                }
+
+                failures += XCTAllExpectationFailures
+                XCTLatestExpectationLocation = (file: "", line: 0)
+                XCTAllExpectationFailures = []
+
                 totalDuration += duration
 
                 var result = "passed"
@@ -98,4 +129,137 @@ extension XCTestCase {
     public func tearDown() {
         
     }
+
+    /// Creates and returns an expectation associated with the test case.
+    ///
+    /// - Parameter description: This string will be displayed in the test log
+    ///   to help diagnose failures.
+    /// - Parameter file: The file name to use in the error message if
+    ///   this expectation is not waited for. Default is the file
+    ///   containing the call to this method. It is rare to provide this
+    ///   parameter when calling this method.
+    /// - Parameter line: The line number to use in the error message if the
+    ///   this expectation is not waited for. Default is the line
+    ///   number of the call to this method in the calling file. It is rare to
+    ///   provide this parameter when calling this method.
+    ///
+    /// - Note: Whereas Objective-C XCTest determines the file and line
+    ///   number of expectations that are created by using symbolication, this
+    ///   implementation opts to take `file` and `line` as parameters instead.
+    ///   As a result, the interface to these methods are not exactly identical
+    ///   between these environments.
+    public func expectationWithDescription(description: String, file: StaticString = __FILE__, line: UInt = __LINE__) -> XCTestExpectation {
+        let expectation = XCTestExpectation(description: description)
+        XCTAllExpectations.append(expectation)
+        XCTLatestExpectationLocation = (file: file, line: line)
+        return expectation
+    }
+
+    /// Creates a point of synchronization in the flow of a test. Only one
+    /// "wait" can be active at any given time, but multiple discrete sequences
+    /// of { expectations -> wait } can be chained together.
+    ///
+    /// - Parameter timeout: The amount of time within which all expectation
+    ///   must be fulfilled.
+    /// - Parameter file: The file name to use in the error message if
+    ///   expectations are not met before the given timeout. Default is the file
+    ///   containing the call to this method. It is rare to provide this
+    ///   parameter when calling this method.
+    /// - Parameter line: The line number to use in the error message if the
+    ///   expectations are not met before the given timeout. Default is the line
+    ///   number of the call to this method in the calling file. It is rare to
+    ///   provide this parameter when calling this method.
+    /// - Parameter handler: If provided, the handler will be invoked both on
+    ///   timeout or fulfillment of all expectations. Timeout is always treated
+    ///   as a test failure.
+    ///
+    /// - Note: Whereas Objective-C XCTest determines the file and line
+    ///   number of the "wait" call using symbolication, this implementation
+    ///   opts to take `file` and `line` as parameters instead. As a result,
+    ///   the interface to these methods are not exactly identical between
+    ///   these environments.
+    public func waitForExpectationsWithTimeout(timeout: Double, file: StaticString = __FILE__, line: UInt = __LINE__, handler: XCWaitCompletionHandler?) {
+        // Mirror Objective-C XCTest behavior; display a test failure when
+        // users wait without having first set expectations.
+        if XCTAllExpectations.count == 0 {
+            let failure = XCTFailure(
+                message: "call made to wait without any expectations having been set",
+                failureDescription: "API violation",
+                expected: false,
+                file: file,
+                line: line)
+            XCTAllExpectationFailures.append(failure)
+            return
+        }
+
+        // Objective-C XCTest outputs the descriptions of every unfulfilled
+        // expectation. We gather them into this array, which is also used
+        // to determine failure--a non-empty array meets expectations weren't
+        // met.
+        var unfulfilledDescriptions = [String]()
+
+        // We continue checking whether expectations have been fulfilled until
+        // the specified timeout has been reached.
+        let runLoop = NSRunLoop.currentRunLoop()
+        let timeoutDate = NSDate(timeIntervalSinceNow: timeout)
+        while NSDate().compare(timeoutDate) == NSComparisonResult.OrderedAscending {
+            unfulfilledDescriptions = []
+            for expectation in XCTAllExpectations {
+                if !expectation.fulfilled {
+                    unfulfilledDescriptions.append(expectation.description)
+                }
+            }
+
+            // If we've met all expectations, then break out of the specified
+            // timeout loop early.
+            if unfulfilledDescriptions.count == 0 {
+                break
+            }
+
+            // Otherwise, wait another fraction of a second.
+            runLoop.runUntilDate(NSDate(timeIntervalSinceNow: 0.01))
+        }
+
+        if unfulfilledDescriptions.count > 0 {
+            // Not all expectations were fulfilled. Append a failure
+            // to the array of expectation-based failures.
+            let descriptions = unfulfilledDescriptions.joinWithSeparator(", ")
+            let failure = XCTFailure(
+                message: "Exceeded timeout of \(timeout) seconds, with unfulfilled expectations: \(descriptions)",
+                failureDescription: "Asynchronous wait failed",
+                expected: true,
+                file: file,
+                line: line)
+            // FIXME: This should be an instance variable on the test case,
+            //        not a global.
+            XCTAllExpectationFailures.append(failure)
+        }
+
+        // We've recorded all the failures; clear the expectations that
+        // were set for this test case, in order to prepare the next test
+        // case to be run.
+        // FIXME: This should be an instance variable on the test case.
+        //        Once this is the case, there is no longer any need
+        //        to reset it to an empty array here.
+        XCTAllExpectations = []
+
+        // The handler is invoked regardless of whether the test passed.
+        if let completionHandler = handler {
+            var error: NSError? = nil
+            if unfulfilledDescriptions.count > 0 {
+                // If the test failed, send an error object.
+                error = NSError(
+                    domain: "org.swift.XCTestErrorDomain",
+                    code: 0,
+                    userInfo: [:])
+            }
+            completionHandler(error)
+        }
+    }
 }
+
+// FIXME: These should be instance variables on XCTestCase;
+//        see: https://github.com/apple/swift-corelibs-xctest/pull/40
+private var XCTLatestExpectationLocation: (file: StaticString, line: UInt) = ("", 0)
+private var XCTAllExpectations = [XCTestExpectation]()
+private var XCTAllExpectationFailures = [XCTFailure]()
diff --git a/Sources/XCTest/XCTestExpectation.swift b/Sources/XCTest/XCTestExpectation.swift
@@ -0,0 +1,29 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  XCTestExpectation.swift
+//  Expectations represent specific conditions in asynchronous testing.
+//
+
+/// Expectations represent specific conditions in asynchronous testing.
+public class XCTestExpectation {
+    internal var fulfilled = false
+    internal let description: String
+
+    internal init(description: String) {
+        self.description = description
+    }
+
+    /// Marks an expectation as having been met. It's an error to call this
+    /// method on an expectation that has already been fulfilled, or when the
+    /// test case that vended the expectation has already completed.
+    public func fulfill() {
+        fulfilled = true
+    }
+}
diff --git a/Tests/Functional/Asynchronous/main.swift b/Tests/Functional/Asynchronous/main.swift
@@ -0,0 +1,136 @@
+// RUN: %{swiftc} %s -o %{built_tests_dir}/Asynchronous
+// RUN: %{built_tests_dir}/Asynchronous > %t || true
+// RUN: %{xctest_checker} %t %s
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnUnfulfilledExpectation_fails' started.
+// CHECK: .*/Tests/Functional/Asynchronous/main.swift:51: error: AsynchronousTestCase.test_waitingForAnUnfulfilledExpectation_fails : Asynchronous wait failed - Exceeded timeout of 0.2 seconds, with unfulfilled expectations: foo
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnUnfulfilledExpectation_fails' failed \(\d+\.\d+ seconds\).
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails' started.
+// CHECK: .*/Tests/Functional/Asynchronous/main.swift:57: error: AsynchronousTestCase.test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails : Asynchronous wait failed - Exceeded timeout of 0.2 seconds, with unfulfilled expectations: bar, baz
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails' failed \(\d+\.\d+ seconds\).
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnImmediatelyFulfilledExpectation_passes' started.
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnImmediatelyFulfilledExpectation_passes' passed \(\d+\.\d+ seconds\).
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnEventuallyFulfilledExpectation_passes' started.
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnEventuallyFulfilledExpectation_passes' passed \(\d+\.\d+ seconds\).
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnExpectationFulfilledAfterTheTimeout_fails' started.
+// CHECK: .*/Tests/Functional/Asynchronous/main.swift:81: error: AsynchronousTestCase.test_waitingForAnExpectationFulfilledAfterTheTimeout_fails : Asynchronous wait failed - Exceeded timeout of 0.1 seconds, with unfulfilled expectations: hog
+// CHECK: Test Case 'AsynchronousTestCase.test_waitingForAnExpectationFulfilledAfterTheTimeout_fails' failed \(\d+\.\d+ seconds\).
+// CHECK: Executed 5 tests, with 3 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreNotFulfilled_handlerCalled_andFails' started.
+// CHECK: .*/Tests/Functional/Asynchronous/main.swift:97: error: HandlerTestCase.test_whenExpectationsAreNotFulfilled_handlerCalled_andFails : Asynchronous wait failed - Exceeded timeout of 0.2 seconds, with unfulfilled expectations: fog
+// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreNotFulfilled_handlerCalled_andFails' failed \(\d+\.\d+ seconds\).
+// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreFulfilled_handlerCalled_andPasses' started.
+// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreFulfilled_handlerCalled_andPasses' passed \(\d+\.\d+ seconds\).
+// CHECK: Executed 2 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: Test Case 'MisuseTestCase.test_whenExpectationsAreMade_butNotWaitedFor_fails' started.
+// CHECK: .*/Tests/Functional/Asynchronous/main.swift:128: error: MisuseTestCase.test_whenExpectationsAreMade_butNotWaitedFor_fails :  - Failed due to unwaited expectations.
+// CHECK: Test Case 'MisuseTestCase.test_whenExpectationsAreMade_butNotWaitedFor_fails' failed \(\d+\.\d+ seconds\).
+// CHECK: Executed 1 test, with 1 failure \(0 unexpected\) in \d+\.\d+\ \(\d+\.\d+\) seconds
+// CHECK: Total executed 8 tests, with 5 failures \(0 unexpected\) in \d+\.\d+\ \(\d+\.\d+\) seconds
+
+#if os(Linux) || os(FreeBSD)
+    import XCTest
+    import Foundation
+#else
+    import SwiftXCTest
+    import SwiftFoundation
+#endif
+
+class AsynchronousTestCase: XCTestCase {
+    var allTests: [(String, () throws -> ())] {
+        return [
+            ("test_waitingForAnUnfulfilledExpectation_fails", test_waitingForAnUnfulfilledExpectation_fails),
+            ("test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails", test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails),
+            ("test_waitingForAnImmediatelyFulfilledExpectation_passes", test_waitingForAnImmediatelyFulfilledExpectation_passes),
+            ("test_waitingForAnEventuallyFulfilledExpectation_passes", test_waitingForAnEventuallyFulfilledExpectation_passes),
+            ("test_waitingForAnExpectationFulfilledAfterTheTimeout_fails", test_waitingForAnExpectationFulfilledAfterTheTimeout_fails),
+        ]
+    }
+
+    func test_waitingForAnUnfulfilledExpectation_fails() {
+        expectationWithDescription("foo")
+        waitForExpectationsWithTimeout(0.2, handler: nil)
+    }
+
+    func test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails() {
+        expectationWithDescription("bar")
+        expectationWithDescription("baz")
+        waitForExpectationsWithTimeout(0.2, handler: nil)
+    }
+
+    func test_waitingForAnImmediatelyFulfilledExpectation_passes() {
+        let expectation = expectationWithDescription("flim")
+        expectation.fulfill()
+        waitForExpectationsWithTimeout(0.2, handler: nil)
+    }
+
+    func test_waitingForAnEventuallyFulfilledExpectation_passes() {
+        let expectation = expectationWithDescription("flam")
+        let timer = NSTimer.scheduledTimer(0.1, repeats: false) { _ in
+            expectation.fulfill()
+        }
+        NSRunLoop.currentRunLoop().addTimer(timer, forMode: NSDefaultRunLoopMode)
+        waitForExpectationsWithTimeout(1.0, handler: nil)
+    }
+
+    func test_waitingForAnExpectationFulfilledAfterTheTimeout_fails() {
+        let expectation = expectationWithDescription("hog")
+        let timer = NSTimer.scheduledTimer(1.0, repeats: false) { _ in
+            expectation.fulfill()
+        }
+        NSRunLoop.currentRunLoop().addTimer(timer, forMode: NSDefaultRunLoopMode)
+        waitForExpectationsWithTimeout(0.1, handler: nil)
+    }
+}
+
+class HandlerTestCase: XCTestCase {
+    var allTests: [(String, () throws -> ())] {
+        return [
+            ("test_whenExpectationsAreNotFulfilled_handlerCalled_andFails", test_whenExpectationsAreNotFulfilled_handlerCalled_andFails),
+            ("test_whenExpectationsAreFulfilled_handlerCalled_andPasses", test_whenExpectationsAreFulfilled_handlerCalled_andPasses),
+        ]
+    }
+
+    func test_whenExpectationsAreNotFulfilled_handlerCalled_andFails() {
+        self.expectationWithDescription("fog")
+
+        var handlerWasCalled = false
+        self.waitForExpectationsWithTimeout(0.2) { error in
+            XCTAssertNotNil(error, "Expectation handlers for unfulfilled expectations should not be nil.")
+            XCTAssertTrue(error!.domain.hasSuffix("XCTestErrorDomain"), "The last component of the error domain should match Objective-C XCTest.")
+            XCTAssertEqual(error!.code, 0, "The error code should match Objective-C XCTest.")
+            handlerWasCalled = true
+        }
+        XCTAssertTrue(handlerWasCalled)
+    }
+
+    func test_whenExpectationsAreFulfilled_handlerCalled_andPasses() {
+        let expectation = self.expectationWithDescription("bog")
+        expectation.fulfill()
+
+        var handlerWasCalled = false
+        self.waitForExpectationsWithTimeout(0.2) { error in
+            XCTAssertNil(error, "Expectation handlers for fulfilled expectations should be nil.")
+            handlerWasCalled = true
+        }
+        XCTAssertTrue(handlerWasCalled)
+    }
+}
+
+class MisuseTestCase: XCTestCase {
+    var allTests: [(String, () throws -> ())] {
+        return [
+            ("test_whenExpectationsAreMade_butNotWaitedFor_fails", test_whenExpectationsAreMade_butNotWaitedFor_fails),
+        ]
+    }
+
+    func test_whenExpectationsAreMade_butNotWaitedFor_fails() {
+        self.expectationWithDescription("the first expectation")
+        self.expectationWithDescription("the second expectation (the file and line number for this one are included in the failure message")
+    }
+}
+
+XCTMain([
+    AsynchronousTestCase(),
+    HandlerTestCase(),
+    MisuseTestCase(),
+])
diff --git a/XCTest.xcodeproj/project.pbxproj b/XCTest.xcodeproj/project.pbxproj
diff --git a/XCTest.xcworkspace/contents.xcworkspacedata b/XCTest.xcworkspace/contents.xcworkspacedata
