Merge pull request #297 from stmontgomery/XCTSkip

Introduce XCTSkip and related APIs for skipping tests

Author: stmontgomery

CMakeLists.txt

@@ -41,6 +41,7 @@ add_library(XCTest
   Sources/XCTest/Public/XCTestObservationCenter.swift
   Sources/XCTest/Public/XCTestCase+Performance.swift
   Sources/XCTest/Public/XCTAssert.swift
+  Sources/XCTest/Public/XCTSkip.swift
   Sources/XCTest/Public/Asynchronous/XCTNSNotificationExpectation.swift
   Sources/XCTest/Public/Asynchronous/XCTNSPredicateExpectation.swift
   Sources/XCTest/Public/Asynchronous/XCTWaiter+Validation.swift

Sources/XCTest/Private/IgnoredErrors.swift

@@ -10,16 +10,58 @@
 //  IgnoredErrors.swift
 //
 
-/// The user info key used by errors so that they are ignored by the XCTest library.
-internal let XCTestErrorUserInfoKeyShouldIgnore = "XCTestErrorUserInfoKeyShouldIgnore"
+protocol XCTCustomErrorHandling: Error {
 
-/// The error type thrown by `XCTUnwrap` on assertion failure.
-internal struct XCTestErrorWhileUnwrappingOptional: Error, CustomNSError {
-    static var errorDomain: String = XCTestErrorDomain
+    /// Whether this error should be recorded as a test failure when it is caught. Default: true.
+    var shouldRecordAsTestFailure: Bool { get }
+
+    /// Whether this error should cause the test invocation to be skipped when it is caught during a throwing setUp method. Default: true.
+    var shouldSkipTestInvocation: Bool { get }
 
-    var errorCode: Int = 105
+    /// Whether this error should be recorded as a test skip when it is caught during a test invocation. Default: false.
+    var shouldRecordAsTestSkip: Bool { get }
+
+}
+
+extension XCTCustomErrorHandling {
+
+    var shouldRecordAsTestFailure: Bool {
+        true
+    }
 
-    var errorUserInfo: [String : Any] {
-        return [XCTestErrorUserInfoKeyShouldIgnore: true]
+    var shouldSkipTestInvocation: Bool {
+        true
     }
+
+    var shouldRecordAsTestSkip: Bool {
+        false
+    }
+
+}
+
+extension Error {
+
+    var xct_shouldRecordAsTestFailure: Bool {
+        (self as? XCTCustomErrorHandling)?.shouldRecordAsTestFailure ?? true
+    }
+
+    var xct_shouldSkipTestInvocation: Bool {
+        (self as? XCTCustomErrorHandling)?.shouldSkipTestInvocation ?? true
+    }
+
+    var xct_shouldRecordAsTestSkip: Bool {
+        (self as? XCTCustomErrorHandling)?.shouldRecordAsTestSkip ?? false
+    }
+
+}
+
+/// The error type thrown by `XCTUnwrap` on assertion failure.
+internal struct XCTestErrorWhileUnwrappingOptional: Error, XCTCustomErrorHandling {
+
+    var shouldRecordAsTestFailure: Bool {
+        // Don't record this error as a test failure, because XCTUnwrap
+        // internally records the failure before throwing this error
+        false
+    }
+
 }

Sources/XCTest/Private/PrintObserver.swift

@@ -31,7 +31,18 @@ internal class PrintObserver: XCTestObservation {
 
     func testCaseDidFinish(_ testCase: XCTestCase) {
         let testRun = testCase.testRun!
-        let verb = testRun.hasSucceeded ? "passed" : "failed"
+
+        let verb: String
+        if testRun.hasSucceeded {
+            if testRun.hasBeenSkipped {
+                verb = "skipped"
+            } else {
+                verb = "passed"
+            }
+        } else {
+            verb = "failed"
+        }
+
         printAndFlush("Test Case '\(testCase.name)' \(verb) (\(formatTimeInterval(testRun.totalDuration)) seconds)")
     }
 
@@ -41,11 +52,16 @@ internal class PrintObserver: XCTestObservation {
         printAndFlush("Test Suite '\(testSuite.name)' \(verb) at \(dateFormatter.string(from: testRun.stopDate!))")
 
         let tests = testRun.executionCount == 1 ? "test" : "tests"
+        let skipped = testRun.skipCount > 0 ? "\(testRun.skipCount) test\(testRun.skipCount != 1 ? "s" : "") skipped and " : ""
         let failures = testRun.totalFailureCount == 1 ? "failure" : "failures"
-        printAndFlush(
-            "\t Executed \(testRun.executionCount) \(tests), " +
-            "with \(testRun.totalFailureCount) \(failures) (\(testRun.unexpectedExceptionCount) unexpected) " +
-            "in \(formatTimeInterval(testRun.testDuration)) (\(formatTimeInterval(testRun.totalDuration))) seconds"
+
+        printAndFlush("""
+            \t Executed \(testRun.executionCount) \(tests), \
+            with \(skipped)\
+            \(testRun.totalFailureCount) \(failures) \
+            (\(testRun.unexpectedExceptionCount) unexpected) \
+            in \(formatTimeInterval(testRun.testDuration)) (\(formatTimeInterval(testRun.totalDuration))) seconds
+            """
         )
     }
 
@@ -70,6 +86,12 @@ internal class PrintObserver: XCTestObservation {
 }
 
 extension PrintObserver: XCTestInternalObservation {
+    func testCase(_ testCase: XCTestCase, wasSkippedWithDescription description: String, at sourceLocation: SourceLocation?) {
+        let file = sourceLocation?.file ?? "<unknown>"
+        let line = sourceLocation?.line ?? 0
+        printAndFlush("\(file):\(line): \(testCase.name) : \(description)")
+    }
+
     func testCase(_ testCase: XCTestCase, didMeasurePerformanceResults results: String, file: StaticString, line: Int) {
         printAndFlush("\(file):\(line): Test Case '\(testCase.name)' measured \(results)")
     }

Sources/XCTest/Private/XCTestInternalObservation.swift

@@ -15,6 +15,8 @@
 /// Expanded version of `XCTestObservation` used internally to respond to
 /// additional events not publicly exposed.
 internal protocol XCTestInternalObservation: XCTestObservation {
+    func testCase(_ testCase: XCTestCase, wasSkippedWithDescription description: String, at sourceLocation: SourceLocation?)
+
     /// Called when a test case finishes measuring performance and has results
     /// to report
     /// - Parameter testCase: The test case that did the measurements.
@@ -28,5 +30,6 @@ internal protocol XCTestInternalObservation: XCTestObservation {
 
 // All `XCInternalTestObservation` methods are optional, so empty default implementations are provided
 internal extension XCTestInternalObservation {
+    func testCase(_ testCase: XCTestCase, wasSkippedWithDescription description: String, at sourceLocation: SourceLocation?) {}
     func testCase(_ testCase: XCTestCase, didMeasurePerformanceResults results: String, file: StaticString, line: Int) {}
 }

Sources/XCTest/Public/XCAbstractTest.swift

@@ -52,6 +52,10 @@ open class XCTest {
         perform(testRun!)
     }
 
+    /// Setup method called before the invocation of `setUp` and the test method
+    /// for each test method in the class.
+    open func setUpWithError() throws {}
+
     /// Setup method called before the invocation of each test method in the
     /// class.
     open func setUp() {}
@@ -60,6 +64,10 @@ open class XCTest {
     /// class.
     open func tearDown() {}
 
+    /// Teardown method called after the invocation of the test method and `tearDown`
+    /// for each test method in the class.
+    open func tearDownWithError() throws {}
+
     // FIXME: This initializer is required due to a Swift compiler bug on Linux.
     //        It should be removed once the bug is fixed.
     public init() {}

Sources/XCTest/Public/XCTSkip.swift

@@ -0,0 +1,116 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  XCTSkip.swift
+//  APIs for skipping tests
+//
+
+/// An error which causes the current test to cease executing
+/// and be marked as skipped when it is thrown.
+public struct XCTSkip: Error {
+
+    /// The user-supplied message related to this skip, if specified.
+    public let message: String?
+
+    /// A complete description of the skip. Includes the string-ified expression and user-supplied message when possible.
+    let summary: String
+
+    /// An explanation of why the skip has occurred.
+    ///
+    /// - Note: May be nil if the skip was unconditional.
+    private let explanation: String?
+
+    /// The source code location where the skip occurred.
+    let sourceLocation: SourceLocation?
+
+    private init(explanation: String?, message: String?, sourceLocation: SourceLocation?) {
+        self.explanation = explanation
+        self.message = message
+        self.sourceLocation = sourceLocation
+
+        var summary = "Test skipped"
+        if let explanation = explanation {
+            summary += ": \(explanation)"
+        }
+        if let message = message, !message.isEmpty {
+            summary += " - \(message)"
+        }
+        self.summary = summary
+    }
+
+    public init(_ message: @autoclosure () -> String? = nil, file: StaticString = #file, line: UInt = #line) {
+        self.init(explanation: nil, message: message(), sourceLocation: SourceLocation(file: file, line: line))
+    }
+
+    fileprivate init(expectedValue: Bool, message: String?, file: StaticString, line: UInt) {
+        let explanation = expectedValue
+            ? "required true value but got false"
+            : "required false value but got true"
+        self.init(explanation: explanation, message: message, sourceLocation: SourceLocation(file: file, line: line))
+    }
+
+    internal init(error: Error, message: String?, sourceLocation: SourceLocation?) {
+        let explanation = #"threw error "\#(error)""#
+        self.init(explanation: explanation, message: message, sourceLocation: sourceLocation)
+    }
+
+}
+
+extension XCTSkip: XCTCustomErrorHandling {
+
+    var shouldRecordAsTestFailure: Bool {
+        // Don't record this error as a test failure since it's a test skip
+        false
+    }
+
+    var shouldRecordAsTestSkip: Bool {
+        true
+    }
+
+}
+
+/// Evaluates a boolean expression and, if it is true, throws an error which
+/// causes the current test to cease executing and be marked as skipped.
+public func XCTSkipIf(
+    _ expression: @autoclosure () throws -> Bool,
+    _ message: @autoclosure () -> String? = nil,
+    file: StaticString = #file, line: UInt = #line
+) throws {
+    try skipIfEqual(expression(), true, message(), file: file, line: line)
+}
+
+/// Evaluates a boolean expression and, if it is false, throws an error which
+/// causes the current test to cease executing and be marked as skipped.
+public func XCTSkipUnless(
+    _ expression: @autoclosure () throws -> Bool,
+    _ message: @autoclosure () -> String? = nil,
+    file: StaticString = #file, line: UInt = #line
+) throws {
+    try skipIfEqual(expression(), false, message(), file: file, line: line)
+}
+
+private func skipIfEqual(
+    _ expression: @autoclosure () throws -> Bool,
+    _ expectedValue: Bool,
+    _ message: @autoclosure () -> String?,
+    file: StaticString, line: UInt
+) throws {
+    let expressionValue: Bool
+
+    do {
+        // evaluate the expression exactly once
+        expressionValue = try expression()
+    } catch {
+        throw XCTSkip(error: error, message: message(), sourceLocation: SourceLocation(file: file, line: line))
+    }
+
+    if expressionValue == expectedValue {
+        throw XCTSkip(expectedValue: expectedValue, message: message(), file: file, line: line)
+    }
+}

Sources/XCTest/Public/XCTestCase.swift

@@ -34,6 +34,8 @@ internal var XCTCurrentTestCase: XCTestCase?
 open class XCTestCase: XCTest {
     private let testClosure: XCTestCaseClosure
 
+    private var skip: XCTSkip?
+
     /// The name of the test case, consisting of its class name and the method
     /// name it will run.
     open override var name: String {
@@ -114,26 +116,31 @@ open class XCTestCase: XCTest {
     /// Invoking a test performs its setUp, invocation, and tearDown. In
     /// general this should not be called directly.
     open func invokeTest() {
-        setUp()
+        performSetUpSequence()
+
         do {
-            try testClosure(self)
+            if skip == nil {
+                try testClosure(self)
+            }
         } catch {
-            var shouldIgnore = false
-            if let userInfo = (error as? CustomNSError)?.errorUserInfo,
-                let shouldIgnoreValue = userInfo[XCTestErrorUserInfoKeyShouldIgnore] as? NSNumber {
-                shouldIgnore = shouldIgnoreValue.boolValue
+            if error.xct_shouldRecordAsTestFailure {
+                recordFailure(for: error)
             }
 
-            if !shouldIgnore {
-                recordFailure(
-                    withDescription: "threw error \"\(error)\"",
-                    inFile: "<EXPR>",
-                    atLine: 0,
-                    expected: false)
+            if error.xct_shouldRecordAsTestSkip {
+                if let skip = error as? XCTSkip {
+                    self.skip = skip
+                } else {
+                    self.skip = XCTSkip(error: error, message: nil, sourceLocation: nil)
+                }
             }
         }
-        runTeardownBlocks()
-        tearDown()
+
+        if let skip = skip {
+            testRun?.recordSkip(description: skip.summary, sourceLocation: skip.sourceLocation)
+        }
+
+        performTearDownSequence()
     }
 
     /// Records a failure in the execution of the test and is used by all test
@@ -171,6 +178,15 @@ open class XCTestCase: XCTest {
         recordFailure(withDescription: description, inFile: sourceLocation.file, atLine: Int(sourceLocation.line), expected: expected)
     }
 
+    // Convenience for recording a failure for a caught Error
+    private func recordFailure(for error: Error) {
+        recordFailure(
+            withDescription: "threw error \"\(error)\"",
+            inFile: "<EXPR>",
+            atLine: 0,
+            expected: false)
+    }
+
     /// Setup method called before the invocation of any test method in the
     /// class.
     open class func setUp() {}
@@ -192,6 +208,40 @@ open class XCTestCase: XCTest {
         }
     }
 
+    private func performSetUpSequence() {
+        do {
+            try setUpWithError()
+        } catch {
+            if error.xct_shouldRecordAsTestFailure {
+                recordFailure(for: error)
+            }
+
+            if error.xct_shouldSkipTestInvocation {
+                if let skip = error as? XCTSkip {
+                    self.skip = skip
+                } else {
+                    self.skip = XCTSkip(error: error, message: nil, sourceLocation: nil)
+                }
+            }
+        }
+
+        setUp()
+    }
+
+    private func performTearDownSequence() {
+        runTeardownBlocks()
+
+        tearDown()
+
+        do {
+            try tearDownWithError()
+        } catch {
+            if error.xct_shouldRecordAsTestFailure {
+                recordFailure(for: error)
+            }
+        }
+    }
+
     private func runTeardownBlocks() {
         let blocks = teardownBlocksQueue.sync { () -> [() -> Void] in
             self.teardownBlocksDequeued = true