[XCTestObservation] Add XCTestSuite announcements

Sources/XCTest/XCAbstractTest.swift

@@ -0,0 +1,36 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  XCAbstractTest.swift
+//  An abstract base class that XCTestCase and XCTestSuite inherit from.
+//  The purpose of this class is to mirror the design of Apple XCTest.
+//
+
+/// An abstract base class for testing. `XCTestCase` and `XCTestSuite` extend
+/// `XCTest` to provide for creating, managing, and executing tests. Most
+/// developers will not need to subclass `XCTest` directly.
+public class XCTest {
+    /// Test's name. Must be overridden by subclasses.
+    public var name: String {
+        fatalError("Must be overridden by subclasses.")
+    }
+
+    /// Number of test cases. Must be overridden by subclasses.
+    public var testCaseCount: UInt {
+        fatalError("Must be overridden by subclasses.")
+    }
+
+    /// Setup method called before the invocation of each test method in the
+    /// class.
+    public func setUp() {}
+
+    /// Teardown method called after the invocation of each test method in the
+    /// class.
+    public func tearDown() {}
+}

Sources/XCTest/XCTestCase.swift

@@ -24,23 +24,23 @@
 /// - seealso: `XCTMain`
 public typealias XCTestCaseEntry = (testCaseClass: XCTestCase.Type, allTests: [(String, XCTestCase throws -> Void)])
 
-public class XCTestCase {
+public class XCTestCase: XCTest {
 
-    /// The name of the test case, consisting of its class name and the method name it will run.
-    /// - Note: FIXME: This property should be readonly, but currently has to be publicly settable due to a
-    ///         toolchain bug on Linux. To ensure compatibility of tests between
-    ///         swift-corelibs-xctest and Apple XCTest, this property should not be modified.
-    ///         See https://bugs.swift.org/browse/SR-1129 for details.
-    public var name: String
-
-    public required init() {
-        name = "\(self.dynamicType).<unknown>"
-    }
-
-    public func setUp() {
+    /// The name of the test case, consisting of its class name and the method
+    /// name it will run.
+    public override var name: String {
+        return _name
     }
-
-    public func tearDown() {
+    /// A private setter for the name of this test case.
+    /// - Note: FIXME: This property should be readonly, but currently has to
+    ///   be publicly settable due to a Swift compiler bug on Linux. To ensure
+    ///   compatibility of tests between swift-corelibs-xctest and Apple XCTest,
+    ///   this property should not be modified. See
+    ///   https://bugs.swift.org/browse/SR-1129 for details.
+    public var _name: String
+
+    public required override init() {
+        _name = "\(self.dynamicType).<unknown>"
     }
 }
 
@@ -91,7 +91,7 @@ extension XCTestCase {
         let overallDuration = measureTimeExecutingBlock {
             for (name, test) in tests {
                 let testCase = self.init()
-                testCase.name = "\(testCase.dynamicType).\(name)"
+                testCase._name = "\(testCase.dynamicType).\(name)"
 
                 var failures = [XCTFailure]()
                 XCTFailureHandler = { failure in

Sources/XCTest/XCTestMain.swift

@@ -83,11 +83,47 @@ internal struct XCTRun {
     let testBundle = NSBundle.mainBundle()
     observationCenter.testBundleWillStart(testBundle)
 
-    let filter = TestFiltering()
+    // Apple XCTest behaves differently if tests have been filtered:
+    // - The root `XCTestSuite` is named "Selected tests" instead of
+    //   "All tests".
+    // - An `XCTestSuite` representing the .xctest test bundle is not included.
+    let selectedTestName = ArgumentParser().selectedTestName
+    var rootTestSuites = [XCTestSuite]()
+    if selectedTestName == nil {
+        rootTestSuites.append(XCTestSuite(name: "All tests"))
+        rootTestSuites.append(XCTestSuite(name: "\(testBundle.bundlePath.lastPathComponent).xctest"))
+    } else {
+        rootTestSuites.append(XCTestSuite(name: "Selected tests"))
+    }
+
+    let filter = TestFiltering(selectedTestName: selectedTestName)
+    let filteredTestCases = TestFiltering.filterTests(testCases, filter: filter.selectedTestFilter)
+
+    // When `XCTestSuite` objects are announced, they need to already include
+    // the correct tests.
+    for (testCase, _) in filteredTestCases {
+        let testCaseSuite = XCTestSuite(name: "\(testCase.init().dynamicType)")
+        rootTestSuites.last!.addTest(testCaseSuite)
+    }
+
+    for suite in rootTestSuites {
+        observationCenter.testSuiteWillStart(suite)
+    }
 
     let overallDuration = measureTimeExecutingBlock {
-        for (testCase, tests) in TestFiltering.filterTests(testCases, filter: filter.selectedTestFilter) {
+        // FIXME: This was the simplest implementation that didn't involve
+        //        changing how swift-corelibs-xctest builds up and executes a
+        //        collection of test cases. However, instead of using an index
+        //        to enumerate both the `XCTestSuite` and the test cases at
+        //        once, we should enumerate the test cases in the `XCTestSuite`.
+        //        `XCTestSuite` should be responsible for representing the
+        //        collection of tests we execute here.
+        for (index, test) in filteredTestCases.enumerated() {
+            let (testCase, tests) = test
+            let testSuite = rootTestSuites.last!.tests[index] as! XCTestSuite
+            observationCenter.testSuiteWillStart(testSuite)
             testCase.invokeTests(tests)
+            observationCenter.testSuiteDidFinish(testSuite)
         }
     }
 
@@ -102,6 +138,10 @@ internal struct XCTRun {
         failureSuffix = ""
     }
 
+    for suite in rootTestSuites.reversed() {
+        observationCenter.testSuiteDidFinish(suite)
+    }
+
     XCTPrint("Total executed \(XCTAllRuns.count) test\(testCountSuffix), with \(totalFailures) failure\(failureSuffix) (\(totalUnexpectedFailures) unexpected) in \(printableStringForTimeInterval(totalDuration)) (\(printableStringForTimeInterval(overallDuration))) seconds")
     observationCenter.testBundleDidFinish(testBundle)
     exit(totalFailures > 0 ? 1 : 0)

Sources/XCTest/XCTestObservation.swift

@@ -27,6 +27,11 @@ public protocol XCTestObservation: class {
     ///   executed.
     func testBundleWillStart(testBundle: NSBundle)
 
+    /// Sent when a test suite starts executing.
+    /// - Parameter testSuite: The test suite that started. Additional
+    ///   information can be retrieved from the associated XCTestRun.
+    func testSuiteWillStart(testSuite: XCTestSuite)
+
     /// Called just before a test begins executing.
     /// - Parameter testCase: The test case that is about to start. Its `name`
     ///   property can be used to identify it.
@@ -47,6 +52,11 @@ public protocol XCTestObservation: class {
     ///   can be used to identify it.
     func testCaseDidFinish(testCase: XCTestCase)
 
+    /// Sent when a test suite finishes executing.
+    /// - Parameter testSuite: The test suite that finished. Additional
+    ///   information can be retrieved from the associated XCTestRun.
+    func testSuiteDidFinish(testSuite: XCTestSuite)
+
     /// Sent immediately after all tests have finished as a hook for any
     /// post-testing activity. The test process will generally exit after this
     /// method returns, so if there is long running and/or asynchronous work to
@@ -59,7 +69,11 @@ public protocol XCTestObservation: class {
 
 // All `XCTestObservation` methods are optional, so empty default implementations are provided
 public extension XCTestObservation {
+    func testBundleWillStart(testBundle: NSBundle) {}
+    func testSuiteWillStart(testSuite: XCTestSuite) {}
     func testCaseWillStart(testCase: XCTestCase) {}
     func testCase(testCase: XCTestCase, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: UInt) {}
     func testCaseDidFinish(testCase: XCTestCase) {}
+    func testSuiteDidFinish(testSuite: XCTestSuite) {}
+    func testBundleDidFinish(testBundle: NSBundle) {}
 }

Sources/XCTest/XCTestObservationCenter.swift

@@ -47,6 +47,10 @@ public class XCTestObservationCenter {
         forEachObserver { $0.testBundleWillStart(testBundle) }
     }
 
+    internal func testSuiteWillStart(testSuite: XCTestSuite) {
+        forEachObserver { $0.testSuiteWillStart(testSuite) }
+    }
+
     internal func testCaseWillStart(testCase: XCTestCase) {
         forEachObserver { $0.testCaseWillStart(testCase) }
     }
@@ -59,6 +63,10 @@ public class XCTestObservationCenter {
         forEachObserver { $0.testCaseDidFinish(testCase) }
     }
 
+    internal func testSuiteDidFinish(testSuite: XCTestSuite) {
+        forEachObserver { $0.testSuiteDidFinish(testSuite) }
+    }
+
     internal func testBundleDidFinish(testBundle: NSBundle) {
         forEachObserver { $0.testBundleDidFinish(testBundle) }
     }

Sources/XCTest/XCTestSuite.swift

@@ -0,0 +1,58 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  XCTestSuite.swift
+//  A collection of test cases.
+//
+
+/// A concrete subclass of XCTest, XCTestSuite is a collection of test cases.
+/// Suites are usually managed by the IDE, but XCTestSuite also provides API
+/// for dynamic test and suite management:
+///
+///     XCTestSuite *suite = [XCTestSuite testSuiteWithName:@"My tests"];
+///     [suite addTest:[MathTest testCaseWithSelector:@selector(testAdd)]];
+///     [suite addTest:[MathTest testCaseWithSelector:@selector(testDivideByZero)]];
+///
+/// Alternatively, a test suite can extract the tests to be run automatically.
+/// To do so, pass the class of your test case class to the suite's constructor:
+///
+///     XCTestSuite *suite = [XCTestSuite testSuiteForTestCaseClass:[MathTest class]];
+///
+/// This creates a suite with all the methods starting with "test" that take no
+/// arguments. Also, a test suite of all the test cases found in the runtime
+/// can be created automatically:
+///
+///     XCTestSuite *suite = [XCTestSuite defaultTestSuite];
+///
+/// This creates a suite of suites with all the XCTestCase subclasses methods
+/// that start with "test" and take no arguments.
+public class XCTestSuite: XCTest {
+    public private(set) var tests = [XCTest]()
+
+    /// The name of this test suite.
+    override public var name: String {
+        return _name
+    }
+    private let _name: String
+
+    /// The number of test cases in this suite.
+    public override var testCaseCount: UInt {
+        return UInt(tests.count)
+    }
+
+    public init(name: String) {
+        _name = name
+    }
+
+    /// Adds a test (either an `XCTestSuite` or an `XCTestCase` to this
+    /// collection.
+    public func addTest(test: XCTest) {
+        tests.append(test)
+    }
+}

Tests/Functional/Observation/main.swift → Tests/Functional/Observation/All/main.swift

@@ -1,5 +1,5 @@
-// RUN: %{swiftc} %s -o %{built_tests_dir}/Observation
-// RUN: %{built_tests_dir}/Observation > %t || true
+// RUN: %{swiftc} %s -o %{built_tests_dir}/All
+// RUN: %{built_tests_dir}/All > %t || true
 // RUN: %{xctest_checker} %t %s
 
 #if os(Linux) || os(FreeBSD)
@@ -12,15 +12,21 @@
 
 class Observer: XCTestObservation {
     var startedBundlePaths = [String]()
+    var startedTestSuites = [XCTestSuite]()
     var startedTestCaseNames = [String]()
     var failureDescriptions = [String]()
     var finishedTestCaseNames = [String]()
+    var finishedTestSuites = [XCTestSuite]()
     var finishedBundlePaths = [String]()
 
     func testBundleWillStart(testBundle: NSBundle) {
         startedBundlePaths.append(testBundle.bundlePath)
     }
 
+    func testSuiteWillStart(testSuite: XCTestSuite) {
+        startedTestSuites.append(testSuite)
+    }
+
     func testCaseWillStart(testCase: XCTestCase) {
         startedTestCaseNames.append(testCase.name)
     }
@@ -33,6 +39,10 @@ class Observer: XCTestObservation {
         finishedTestCaseNames.append(testCase.name)
     }
 
+    func testSuiteDidFinish(testSuite: XCTestSuite) {
+        print("In \(#function): \(testSuite.name)")
+    }
+
     func testBundleDidFinish(testBundle: NSBundle) {
         print("In \(#function)")
     }
@@ -51,10 +61,13 @@ class Observation: XCTestCase {
     }
 
 // CHECK: Test Case 'Observation.test_one' started.
-// CHECK: .*/Observation/main.swift:\d+: error: Observation.test_one : failed - fail!
+// CHECK: .*/Observation/All/main.swift:\d+: error: Observation.test_one : failed - fail!
 // CHECK: Test Case 'Observation.test_one' failed \(\d+\.\d+ seconds\).
     func test_one() {
         XCTAssertEqual(observer.startedBundlePaths.count, 1)
+        XCTAssertEqual(
+            observer.startedTestSuites.count, 3,
+            "Three test suites should have started: 'All tests', 'tmp.xctest', and 'Observation'.")
         XCTAssertEqual(observer.startedTestCaseNames, ["Observation.test_one"])
         XCTAssertEqual(observer.failureDescriptions, [])
         XCTAssertEqual(observer.finishedTestCaseNames, [])
@@ -68,6 +81,9 @@ class Observation: XCTestCase {
 // CHECK: Test Case 'Observation.test_two' passed \(\d+\.\d+ seconds\).
     func test_two() {
         XCTAssertEqual(observer.startedBundlePaths.count, 1)
+        XCTAssertEqual(
+            observer.startedTestSuites.count, 3,
+            "Three test suites should have started: 'All tests', 'tmp.xctest', and 'Observation'.")
         XCTAssertEqual(observer.startedTestCaseNames, ["Observation.test_one", "Observation.test_two"])
         XCTAssertEqual(observer.finishedTestCaseNames,["Observation.test_one"])
         XCTAssertEqual(observer.finishedBundlePaths.count, 0)
@@ -90,5 +106,8 @@ class Observation: XCTestCase {
 XCTMain([testCase(Observation.allTests)])
 
 // CHECK: Executed 3 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: In testSuiteDidFinish: Observation
+// CHECK: In testSuiteDidFinish: .*\.xctest
+// CHECK: In testSuiteDidFinish: All tests
 // CHECK: Total executed 3 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
 // CHECK: In testBundleDidFinish