Merge pull request #40 from briancroom/allow_setup_teardown_as_override

[SR-460] Provide XCTestCase a class to allow overriding setUp() and tearDown()

Author: Mike Ferris

README.md

@@ -58,14 +58,14 @@ You may add tests for XCTest by including them in the `Tests/Functional/` direct
 
 ### Additional Considerations for Swift on Linux
 
-When running on the Objective-C runtime, XCTest is able to find all of your tests by simply asking the runtime for the subclasses of `XCTestCase`. It then finds the methods that start with the string `test`. This functionality is not currently present when running on the Swift runtime. Therefore, you must currently provide an additional property called `allTests` in your `XCTestCase` subclass. This method lists all of the tests in the test class. The rest of your test case subclass still contains your test methods.
+When running on the Objective-C runtime, XCTest is able to find all of your tests by simply asking the runtime for the subclasses of `XCTestCase`. It then finds the methods that start with the string `test`. This functionality is not currently present when running on the Swift runtime. Therefore, you must currently provide an additional property, conventionally named `allTests`, in your `XCTestCase` subclass. This method lists all of the tests in the test class. The rest of your test case subclass still contains your test methods.
 
 ```swift
 class TestNSURL : XCTestCase {
-    var allTests : [(String, () -> Void)] {
+    static var allTests : [(String, TestNSURL -> () throws -> Void)] {
         return [
             ("test_URLStrings", test_URLStrings),
-            ("test_fileURLWithPath_relativeToURL", test_fileURLWithPath_relativeToURL ),
+            ("test_fileURLWithPath_relativeToURL", test_fileURLWithPath_relativeToURL),
             ("test_fileURLWithPath", test_fileURLWithPath),
             ("test_fileURLWithPath_isDirectory", test_fileURLWithPath_isDirectory),
             // Other tests go here
@@ -81,10 +81,10 @@ class TestNSURL : XCTestCase {
 }
 ```
 
-Also, this version of XCTest does not use the external test runner binary. Instead, create your own executable which links `libXCTest.so`. In your `main.swift`, invoke the `XCTMain` function with an array of instances of the test cases that you wish to run. For example:
+Also, this version of XCTest does not use the external test runner binary. Instead, create your own executable which links `libXCTest.so`. In your `main.swift`, invoke the `XCTMain` function with an array of the test cases classes that you wish to run, wrapped by the `testCase` helper function. For example:
 
 ```swift
-XCTMain([TestNSString(), TestNSArray(), TestNSDictionary()])
+XCTMain([testCase(TestNSString.allTests), testCase(TestNSArray.allTests), testCase(TestNSDictionary.allTests)])
 ```
 
 The `XCTMain` function does not return, and will cause your test app to exit with either `0` for success or `1` for failure.

Sources/XCTest/XCTAssert.swift

@@ -104,7 +104,7 @@ private func _XCTEvaluateAssertion(assertion: _XCTAssertion, @autoclosure messag
 /// to it evaluates to false.
 ///
 /// - Requires: This and all other XCTAssert* functions must be called from
-///   within a test method, as indicated by `XCTestCaseProvider.allTests`.
+///   within a test method, as passed to `XCTMain`.
 ///   Assertion failures that occur outside of a test method will *not* be
 ///   reported as failures.
 ///

Sources/XCTest/XCTestCase.swift

@@ -8,12 +8,44 @@
 //
 //
 //  XCTestCase.swift
-//  Base protocol (and extension with default methods) for test cases
+//  Base class for test cases
 //
 
-public protocol XCTestCase : XCTestCaseProvider {
-    func setUp()
-    func tearDown()
+/// This is a compound type used by `XCTMain` to represent tests to run. It combines an
+/// `XCTestCase` subclass type with the list of test methods to invoke on the test case.
+/// This type is intended to be produced by the `testCase` helper function.
+/// - seealso: `testCase`
+/// - seealso: `XCTMain`
+public typealias XCTestCaseEntry = (testCaseClass: XCTestCase.Type, allTests: [(String, XCTestCase throws -> Void)])
+
+public class XCTestCase {
+
+    public required init() {
+    }
+
+    public func setUp() {
+    }
+
+    public func tearDown() {
+    }
+}
+
+/// Wrapper function allowing an array of static test case methods to fit
+/// the signature required by `XCTMain`
+/// - seealso: `XCTMain`
+public func testCase<T: XCTestCase>(allTests: [(String, T -> () throws -> Void)]) -> XCTestCaseEntry {
+    let tests: [(String, XCTestCase throws -> Void)] = allTests.map({ ($0.0, test($0.1)) })
+    return (T.self, tests)
+}
+
+private func test<T: XCTestCase>(testFunc: T -> () throws -> Void) -> XCTestCase throws -> Void {
+    return { testCaseType in
+        guard let testCase: T = testCaseType as? T else {
+            fatalError("Attempt to invoke test on class \(T.self) with incompatible instance type \(testCaseType.dynamicType)")
+        }
+
+        try testFunc(testCase)()
+    }
 }
 
 extension XCTestCase {
@@ -26,55 +58,55 @@ extension XCTestCase {
             // TODO: When using the Objective-C runtime, XCTest is able to throw an exception from an assert and then catch it at the frame above the test method. This enables the framework to effectively stop all execution in the current test. There is no such facility in Swift. Until we figure out how to get a compatible behavior, we have decided to hard-code the value of 'true' for continue after failure.
         }
     }
-    
-    public func invokeTest() {
-        let tests = self.allTests
+
+    internal static func invokeTests(tests: [(String, XCTestCase throws -> Void)]) {
         var totalDuration = 0.0
         var totalFailures = 0
         var unexpectedFailures = 0
         let overallDuration = measureTimeExecutingBlock {
             for (name, test) in tests {
-                let method = "\(self.dynamicType).\(name)"
+                let testCase = self.init()
+                let fullName = "\(testCase.dynamicType).\(name)"
 
                 var failures = [XCTFailure]()
                 XCTFailureHandler = { failure in
-                    if !self.continueAfterFailure {
-                        failure.emit(method)
+                    if !testCase.continueAfterFailure {
+                        failure.emit(fullName)
                         fatalError("Terminating execution due to test failure", file: failure.file, line: failure.line)
                     } else {
                         failures.append(failure)
                     }
                 }
 
-                XCTPrint("Test Case '\(method)' started.")
+                XCTPrint("Test Case '\(fullName)' started.")
 
-                setUp()
+                testCase.setUp()
 
                 let duration = measureTimeExecutingBlock {
                     do {
-                        try test()
+                        try test(testCase)
                     } catch {
                         let unexpectedFailure = XCTFailure(message: "", failureDescription: "threw error \"\(error)\"", expected: false, file: "<EXPR>", line: 0)
                         XCTFailureHandler!(unexpectedFailure)
                     }
                 }
 
-                tearDown()
+                testCase.tearDown()
 
                 totalDuration += duration
 
                 var result = "passed"
                 for failure in failures {
-                    failure.emit(method)
+                    failure.emit(fullName)
                     totalFailures += 1
                     if !failure.expected {
                         unexpectedFailures += 1
                     }
                     result = failures.count > 0 ? "failed" : "passed"
                 }
 
-                XCTPrint("Test Case '\(method)' \(result) (\(printableStringForTimeInterval(duration)) seconds).")
-                XCTAllRuns.append(XCTRun(duration: duration, method: method, passed: failures.count == 0, failures: failures))
+                XCTPrint("Test Case '\(fullName)' \(result) (\(printableStringForTimeInterval(duration)) seconds).")
+                XCTAllRuns.append(XCTRun(duration: duration, method: fullName, passed: failures.count == 0, failures: failures))
                 XCTFailureHandler = nil
             }
         }
@@ -90,12 +122,4 @@ extension XCTestCase {
 
         XCTPrint("Executed \(tests.count) test\(testCountSuffix), with \(totalFailures) failure\(failureSuffix) (\(unexpectedFailures) unexpected) in \(printableStringForTimeInterval(totalDuration)) (\(printableStringForTimeInterval(overallDuration))) seconds")
     }
-    
-    public func setUp() {
-        
-    }
-    
-    public func tearDown() {
-        
-    }
 }

Sources/XCTest/XCTestCaseProvider.swift

@@ -48,11 +48,31 @@ internal struct XCTRun {
 /// Starts a test run for the specified test cases.
 ///
 /// This function will not return. If the test cases pass, then it will call `exit(0)`. If there is a failure, then it will call `exit(1)`.
-/// - Parameter testCases: An array of test cases to run.
-@noreturn public func XCTMain(testCases: [XCTestCase]) {
+/// Example usage:
+///
+///     class TestFoo: XCTestCase {
+///         static var allTests : [(String, TestFoo -> () throws -> Void)] {
+///             return [
+///                 ("test_foo", test_foo),
+///                 ("test_bar", test_bar),
+///             ]
+///         }
+///
+///         func test_foo() {
+///             // Test things...
+///         }
+///
+///         // etc...
+///     }
+///
+///     XCTMain([ testCase(TestFoo.allTests) ])
+///
+/// - Parameter testCases: An array of test cases run, each produced by a call to the `testCase` function
+/// - seealso: `testCase`
+@noreturn public func XCTMain(testCases: [XCTestCaseEntry]) {
     let overallDuration = measureTimeExecutingBlock {
-        for testCase in testCases {
-            testCase.invokeTest()
+        for (testCase, tests) in testCases {
+            testCase.invokeTests(tests)
         }
     }
 

Sources/XCTest/XCTestMain.swift

@@ -27,7 +27,7 @@
 #endif
 
 class ErrorHandling: XCTestCase {
-    var allTests: [(String, () throws -> ())] {
+    static var allTests: [(String, ErrorHandling -> () throws -> Void)] {
         return [
             // Tests for XCTAssertThrowsError
             ("test_shouldButDoesNotThrowErrorInAssertion", test_shouldButDoesNotThrowErrorInAssertion),
@@ -39,7 +39,7 @@ class ErrorHandling: XCTestCase {
             ("test_canButDoesNotThrowErrorFromTestMethod", test_canButDoesNotThrowErrorFromTestMethod),
 
             // Tests for throwing assertion expressions
-            ("test_assertionExpressionCanThrow", test_assertionExpressionCanThrow), 
+            ("test_assertionExpressionCanThrow", test_assertionExpressionCanThrow),
         ]
     }
 
@@ -103,4 +103,4 @@ class ErrorHandling: XCTestCase {
     }
 }
 
-XCTMain([ErrorHandling()])
+XCTMain([testCase(ErrorHandling.allTests)])

Tests/Functional/ErrorHandling/main.swift

@@ -22,7 +22,7 @@
 #endif
 
 class PassingTestCase: XCTestCase {
-    var allTests: [(String, () throws -> ())] {
+    static var allTests: [(String, PassingTestCase -> () throws -> Void)] {
         return [
             ("test_passes", test_passes),
         ]
@@ -34,7 +34,7 @@ class PassingTestCase: XCTestCase {
 }
 
 class FailingTestCase: XCTestCase {
-    var allTests: [(String, () throws -> ())] {
+    static var allTests: [(String, FailingTestCase -> () throws -> Void)] {
         return [
             ("test_passes", test_passes),
             ("test_fails", test_fails),
@@ -56,6 +56,6 @@ class FailingTestCase: XCTestCase {
 }
 
 XCTMain([
-    PassingTestCase(),
-    FailingTestCase(),
+    testCase(PassingTestCase.allTests),
+    testCase(FailingTestCase.allTests),
 ])

Tests/Functional/FailingTestSuite/main.swift

@@ -78,7 +78,7 @@
 
 // Regression test for https://github.com/apple/swift-corelibs-xctest/pull/22
 class FailureMessagesTestCase: XCTestCase {
-    var allTests : [(String, () throws -> Void)] {
+    static var allTests: [(String, FailureMessagesTestCase -> () throws -> Void)] {
         return [
             ("testAssert", testAssert),
             ("testAssertEqualOptionals", testAssertEqualOptionals),
@@ -194,4 +194,4 @@ class FailureMessagesTestCase: XCTestCase {
     }
 }
 
-XCTMain([FailureMessagesTestCase()])
+XCTMain([testCase(FailureMessagesTestCase.allTests)])

Tests/Functional/FailureMessagesTestCase/main.swift

@@ -22,7 +22,7 @@
 
 // Regression test for https://github.com/apple/swift-corelibs-xctest/pull/7
 class NegativeAccuracyTestCase: XCTestCase {
-    var allTests: [(String, () throws -> ())] {
+    static var allTests: [(String, NegativeAccuracyTestCase -> () throws -> Void)] {
         return [
             ("test_equalWithAccuracy_passes", test_equalWithAccuracy_passes),
             ("test_equalWithAccuracy_fails", test_equalWithAccuracy_fails),
@@ -48,4 +48,4 @@ class NegativeAccuracyTestCase: XCTestCase {
     }
 }
 
-XCTMain([NegativeAccuracyTestCase()])
+XCTMain([testCase(NegativeAccuracyTestCase.allTests)])

Tests/Functional/NegativeAccuracyTestCase/main.swift

@@ -14,9 +14,9 @@
 #endif
 
 class SingleFailingTestCase: XCTestCase {
-    var allTests: [(String, () throws -> ())] {
+    static var allTests: [(String, SingleFailingTestCase -> () throws -> Void)] {
         return [
-            ("test_fails", test_fails),
+            ("test_fails", test_fails)
         ]
     }
 
@@ -25,4 +25,4 @@ class SingleFailingTestCase: XCTestCase {
     }
 }
 
-XCTMain([SingleFailingTestCase()])
+XCTMain([testCase(SingleFailingTestCase.allTests)])

Tests/Functional/SingleFailingTestCase/main.swift

@@ -0,0 +1,72 @@
+// RUN: %{swiftc} %s -o %{built_tests_dir}/TestCaseLifecycle
+// RUN: %{built_tests_dir}/TestCaseLifecycle > %t || true
+// RUN: %{xctest_checker} %t %s
+// CHECK: Test Case 'SetUpTearDownTestCase.test_hasValueFromSetUp' started.
+// CHECK: In setUp\(\)
+// CHECK: In test_hasValueFromSetUp\(\)
+// CHECK: In tearDown\(\)
+// CHECK: Test Case 'SetUpTearDownTestCase.test_hasValueFromSetUp' passed \(\d+\.\d+ seconds\).
+// CHECK: Executed 1 test, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: Test Case 'NewInstanceForEachTestTestCase.test_hasInitializedValue' started.
+// CHECK: Test Case 'NewInstanceForEachTestTestCase.test_hasInitializedValue' passed \(\d+\.\d+ seconds\).
+// CHECK: Test Case 'NewInstanceForEachTestTestCase.test_hasInitializedValueInAnotherTest' started.
+// CHECK: Test Case 'NewInstanceForEachTestTestCase.test_hasInitializedValueInAnotherTest' passed \(\d+\.\d+ seconds\).
+// CHECK: Executed 2 tests, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: Total executed 3 tests, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+
+#if os(Linux) || os(FreeBSD)
+    import XCTest
+#else
+    import SwiftXCTest
+#endif
+
+class SetUpTearDownTestCase: XCTestCase {
+    static var allTests: [(String, SetUpTearDownTestCase -> () throws -> Void)] {
+        return [
+            ("test_hasValueFromSetUp", test_hasValueFromSetUp),
+        ]
+    }
+
+    var value = 0
+
+    override func setUp() {
+        super.setUp()
+        print("In \(__FUNCTION__)")
+        value = 42
+    }
+
+    override func tearDown() {
+        super.tearDown()
+        print("In \(__FUNCTION__)")
+    }
+
+    func test_hasValueFromSetUp() {
+        print("In \(__FUNCTION__)")
+        XCTAssertEqual(value, 42)
+    }
+}
+
+class NewInstanceForEachTestTestCase: XCTestCase {
+    static var allTests: [(String, NewInstanceForEachTestTestCase -> () throws -> Void)] {
+        return [
+            ("test_hasInitializedValue", test_hasInitializedValue),
+            ("test_hasInitializedValueInAnotherTest", test_hasInitializedValueInAnotherTest),
+        ]
+    }
+
+    var value = 1
+
+    func test_hasInitializedValue() {
+        XCTAssertEqual(value, 1)
+        value += 1
+    }
+
+    func test_hasInitializedValueInAnotherTest() {
+        XCTAssertEqual(value, 1)
+    }
+}
+
+XCTMain([
+    testCase(SetUpTearDownTestCase.allTests),
+    testCase(NewInstanceForEachTestTestCase.allTests)
+])