Skip to content

Commit 15f9272

Browse files
modocachemodocache
committed
[XCTestObservation] Add XCTestSuite announcements
Apple XCTest's `XCTestObservation` protocol includes methods that announce when an `XCTestSuite` will begin executing, or has finished executing. Add these same announcements to swift-corelibs-xctest. swift-corelibs-xctest did not already defined `XCTestSuite`, so add it. Apple XCTest allows `XCTestObservation` listeners to determine the number of passing and failing tests by accessing an `XCTestSuite`'s `testRun` property, but this is not yet supported in swift-corelibs-xctest.
1 parent db3643a commit 15f9272

File tree

9 files changed

+281
-20
lines changed

9 files changed

+281
-20
lines changed

‎Sources/XCTest/XCAbstractTest.swift

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// This source file is part of the Swift.org open source project
2+
//
3+
// Copyright (c) 2016 Apple Inc. and the Swift project authors
4+
// Licensed under Apache License v2.0 with Runtime Library Exception
5+
//
6+
// See http://swift.org/LICENSE.txt for license information
7+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
8+
//
9+
//
10+
// XCAbstractTest.swift
11+
// An abstract base class that XCTestCase and XCTestSuite inherit from.
12+
// The purpose of this class is to mirror the design of Apple XCTest.
13+
//
14+
15+
/// An abstract base class for testing. `XCTestCase` and `XCTestSuite` extend
16+
/// `XCTest` to provide for creating, managing, and executing tests. Most
17+
/// developers will not need to subclass `XCTest` directly.
18+
public class XCTest {
19+
/// Test's name. Must be overridden by subclasses.
20+
public var name: String {
21+
fatalError("Must be overridden by subclasses.")
22+
}
23+
24+
/// Number of test cases. Must be overridden by subclasses.
25+
public var testCaseCount: UInt {
26+
fatalError("Must be overridden by subclasses.")
27+
}
28+
29+
/// Setup method called before the invocation of each test method in the
30+
/// class.
31+
public func setUp() {}
32+
33+
/// Teardown method called after the invocation of each test method in the
34+
/// class.
35+
public func tearDown() {}
36+
}

‎Sources/XCTest/XCTestCase.swift

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,22 +24,22 @@
2424
/// - seealso: `XCTMain`
2525
public typealias XCTestCaseEntry = (testCaseClass: XCTestCase.Type, allTests: [(String, XCTestCase throws -> Void)])
2626

27-
public class XCTestCase {
27+
public class XCTestCase: XCTest {
2828

29-
/// The name of the test case, consisting of its class name and the method name it will run.
30-
/// - Note: FIXME: This property should be readonly, but currently has to be publicly settable due to a
31-
/// toolchain bug on Linux. To ensure compatibility of tests between
32-
/// swift-corelibs-xctest and Apple XCTest, this property should not be modified.
33-
public var name: String
34-
35-
public required init() {
36-
name = "\(self.dynamicType).<unknown>"
37-
}
38-
39-
public func setUp() {
29+
/// The name of the test case, consisting of its class name and the method
30+
/// name it will run.
31+
public override var name: String {
32+
return _name
4033
}
41-
42-
public func tearDown() {
34+
/// A private setter for the name of this test case.
35+
/// - Note: FIXME: This property should be readonly, but currently has to
36+
/// be publicly settable due to a Swift compiler bug on Linux. To ensure
37+
/// compatibility of tests between swift-corelibs-xctest and Apple XCTest,
38+
/// this property should not be modified.
39+
public var _name: String
40+
41+
public required override init() {
42+
_name = "\(self.dynamicType).<unknown>"
4343
}
4444
}
4545

@@ -90,7 +90,7 @@ extension XCTestCase {
9090
let overallDuration = measureTimeExecutingBlock {
9191
for (name, test) in tests {
9292
let testCase = self.init()
93-
testCase.name = "\(testCase.dynamicType).\(name)"
93+
testCase._name = "\(testCase.dynamicType).\(name)"
9494

9595
var failures = [XCTFailure]()
9696
XCTFailureHandler = { failure in

‎Sources/XCTest/XCTestMain.swift

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,47 @@ internal struct XCTRun {
8383
let testBundle = NSBundle.mainBundle()
8484
observationCenter.testBundleWillStart(testBundle)
8585

86-
let filter = TestFiltering()
86+
// Apple XCTest behaves differently if tests have been filtered:
87+
// - The root `XCTestSuite` is named "Selected tests" instead of
88+
// "All tests".
89+
// - An `XCTestSuite` representing the .xctest test bundle is not included.
90+
let selectedTestName = ArgumentParser().selectedTestName
91+
var rootTestSuites = [XCTestSuite]()
92+
if selectedTestName == nil {
93+
rootTestSuites.append(XCTestSuite(name: "All tests"))
94+
rootTestSuites.append(XCTestSuite(name: "\(testBundle.bundlePath.lastPathComponent).xctest"))
95+
} else {
96+
rootTestSuites.append(XCTestSuite(name: "Selected tests"))
97+
}
98+
99+
let filter = TestFiltering(selectedTestName: selectedTestName)
100+
let filteredTestCases = TestFiltering.filterTests(testCases, filter: filter.selectedTestFilter)
101+
102+
// When `XCTestSuite` objects are announced, they need to already include
103+
// the correct tests.
104+
for (testCase, _) in filteredTestCases {
105+
let testCaseSuite = XCTestSuite(name: "\(testCase.init().dynamicType)")
106+
rootTestSuites.last!.addTest(testCaseSuite)
107+
}
108+
109+
for suite in rootTestSuites {
110+
observationCenter.testSuiteWillStart(suite)
111+
}
87112

88113
let overallDuration = measureTimeExecutingBlock {
89-
for (testCase, tests) in TestFiltering.filterTests(testCases, filter: filter.selectedTestFilter) {
114+
// FIXME: This was the simplest implementation that didn't involve
115+
// changing how swift-corelibs-xctest builds up and executes a
116+
// collection of test cases. However, instead of using an index
117+
// to enumerate both the `XCTestSuite` and the test cases at
118+
// once, we should enumerate the test cases in the `XCTestSuite`.
119+
// `XCTestSuite` should be responsible for representing the
120+
// collection of tests we execute here.
121+
for (index, test) in filteredTestCases.enumerated() {
122+
let (testCase, tests) = test
123+
let testSuite = rootTestSuites.last!.tests[index] as! XCTestSuite
124+
observationCenter.testSuiteWillStart(testSuite)
90125
testCase.invokeTests(tests)
126+
observationCenter.testSuiteDidFinish(testSuite)
91127
}
92128
}
93129

@@ -102,6 +138,10 @@ internal struct XCTRun {
102138
failureSuffix = ""
103139
}
104140

141+
for suite in rootTestSuites.reversed() {
142+
observationCenter.testSuiteDidFinish(suite)
143+
}
144+
105145
XCTPrint("Total executed \(XCTAllRuns.count) test\(testCountSuffix), with \(totalFailures) failure\(failureSuffix) (\(totalUnexpectedFailures) unexpected) in \(printableStringForTimeInterval(totalDuration)) (\(printableStringForTimeInterval(overallDuration))) seconds")
106146
observationCenter.testBundleDidFinish(testBundle)
107147
exit(totalFailures > 0 ? 1 : 0)

‎Sources/XCTest/XCTestObservation.swift

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ public protocol XCTestObservation: class {
2727
/// executed.
2828
func testBundleWillStart(testBundle: NSBundle)
2929

30+
/// Sent when a test suite starts executing.
31+
/// - Parameter testSuite: The test suite that started. Additional
32+
/// information can be retrieved from the associated XCTestRun.
33+
func testSuiteWillStart(testSuite: XCTestSuite)
34+
3035
/// Called just before a test begins executing.
3136
/// - Parameter testCase: The test case that is about to start. Its `name`
3237
/// property can be used to identify it.
@@ -47,6 +52,11 @@ public protocol XCTestObservation: class {
4752
/// can be used to identify it.
4853
func testCaseDidFinish(testCase: XCTestCase)
4954

55+
/// Sent when a test suite finishes executing.
56+
/// - Parameter testSuite: The test suite that finished. Additional
57+
/// information can be retrieved from the associated XCTestRun.
58+
func testSuiteDidFinish(testSuite: XCTestSuite)
59+
5060
/// Sent immediately after all tests have finished as a hook for any
5161
/// post-testing activity. The test process will generally exit after this
5262
/// method returns, so if there is long running and/or asynchronous work to

‎Sources/XCTest/XCTestObservationCenter.swift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ public class XCTestObservationCenter {
4747
forEachObserver { $0.testBundleWillStart(testBundle) }
4848
}
4949

50+
internal func testSuiteWillStart(testSuite: XCTestSuite) {
51+
forEachObserver { $0.testSuiteWillStart(testSuite) }
52+
}
53+
5054
internal func testCaseWillStart(testCase: XCTestCase) {
5155
forEachObserver { $0.testCaseWillStart(testCase) }
5256
}
@@ -59,6 +63,10 @@ public class XCTestObservationCenter {
5963
forEachObserver { $0.testCaseDidFinish(testCase) }
6064
}
6165

66+
internal func testSuiteDidFinish(testSuite: XCTestSuite) {
67+
forEachObserver { $0.testSuiteDidFinish(testSuite) }
68+
}
69+
6270
internal func testBundleDidFinish(testBundle: NSBundle) {
6371
forEachObserver { $0.testBundleDidFinish(testBundle) }
6472
}

‎Sources/XCTest/XCTestSuite.swift

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// This source file is part of the Swift.org open source project
2+
//
3+
// Copyright (c) 2016 Apple Inc. and the Swift project authors
4+
// Licensed under Apache License v2.0 with Runtime Library Exception
5+
//
6+
// See http://swift.org/LICENSE.txt for license information
7+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
8+
//
9+
//
10+
// XCTestSuite.swift
11+
// A collection of test cases.
12+
//
13+
14+
/// A concrete subclass of XCTest, XCTestSuite is a collection of test cases.
15+
/// Suites are usually managed by the IDE, but XCTestSuite also provides API
16+
/// for dynamic test and suite management:
17+
///
18+
/// XCTestSuite *suite = [XCTestSuite testSuiteWithName:@"My tests"];
19+
/// [suite addTest:[MathTest testCaseWithSelector:@selector(testAdd)]];
20+
/// [suite addTest:[MathTest testCaseWithSelector:@selector(testDivideByZero)]];
21+
///
22+
/// Alternatively, a test suite can extract the tests to be run automatically.
23+
/// To do so, pass the class of your test case class to the suite's constructor:
24+
///
25+
/// XCTestSuite *suite = [XCTestSuite testSuiteForTestCaseClass:[MathTest class]];
26+
///
27+
/// This creates a suite with all the methods starting with "test" that take no
28+
/// arguments. Also, a test suite of all the test cases found in the runtime
29+
/// can be created automatically:
30+
///
31+
/// XCTestSuite *suite = [XCTestSuite defaultTestSuite];
32+
///
33+
/// This creates a suite of suites with all the XCTestCase subclasses methods
34+
/// that start with "test" and take no arguments.
35+
public class XCTestSuite: XCTest {
36+
public private(set) var tests = [XCTest]()
37+
38+
/// The name of this test suite.
39+
override public var name: String {
40+
return _name
41+
}
42+
/// A private setter for the name of this test suite.
43+
/// - Note: FIXME: This property should be readonly, but currently has to
44+
/// be publicly settable due to a Swift compiler bug on Linux. To ensure
45+
/// compatibility of tests between swift-corelibs-xctest and Apple XCTest,
46+
/// this property should not be modified.
47+
public let _name: String
48+
49+
/// The number of test cases in this suite.
50+
public override var testCaseCount: UInt {
51+
return UInt(tests.count)
52+
}
53+
54+
public init(name: String) {
55+
_name = name
56+
}
57+
58+
/// Adds a test (either an `XCTestSuite` or an `XCTestCase` to this
59+
/// collection.
60+
public func addTest(test: XCTest) {
61+
tests.append(test)
62+
}
63+
}

‎Tests/Functional/Observation/main.swift renamed to ‎Tests/Functional/Observation/All/main.swift

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
// RUN: %{swiftc} %s -o %{built_tests_dir}/Observation
2-
// RUN: %{built_tests_dir}/Observation > %t || true
1+
// RUN: %{swiftc} %s -o %{built_tests_dir}/All
2+
// RUN: %{built_tests_dir}/All > %t || true
33
// RUN: %{xctest_checker} %t %s
44

55
#if os(Linux) || os(FreeBSD)
@@ -12,15 +12,21 @@
1212

1313
class Observer: XCTestObservation {
1414
var startedBundlePaths = [String]()
15+
var startedTestSuites = [XCTestSuite]()
1516
var startedTestCaseNames = [String]()
1617
var failureDescriptions = [String]()
1718
var finishedTestCaseNames = [String]()
19+
var finishedTestSuites = [XCTestSuite]()
1820
var finishedBundlePaths = [String]()
1921

2022
func testBundleWillStart(testBundle: NSBundle) {
2123
startedBundlePaths.append(testBundle.bundlePath)
2224
}
2325

26+
func testSuiteWillStart(testSuite: XCTestSuite) {
27+
startedTestSuites.append(testSuite)
28+
}
29+
2430
func testCaseWillStart(testCase: XCTestCase) {
2531
startedTestCaseNames.append(testCase.name)
2632
}
@@ -33,6 +39,10 @@ class Observer: XCTestObservation {
3339
finishedTestCaseNames.append(testCase.name)
3440
}
3541

42+
func testSuiteDidFinish(testSuite: XCTestSuite) {
43+
print("In \(#function): \(testSuite.name)")
44+
}
45+
3646
func testBundleDidFinish(testBundle: NSBundle) {
3747
print("In \(#function)")
3848
}
@@ -51,10 +61,13 @@ class Observation: XCTestCase {
5161
}
5262

5363
// CHECK: Test Case 'Observation.test_one' started.
54-
// CHECK: .*/Observation/main.swift:\d+: error: Observation.test_one : failed - fail!
64+
// CHECK: .*/Observation/All/main.swift:\d+: error: Observation.test_one : failed - fail!
5565
// CHECK: Test Case 'Observation.test_one' failed \(\d+\.\d+ seconds\).
5666
func test_one() {
5767
XCTAssertEqual(observer.startedBundlePaths.count, 1)
68+
XCTAssertEqual(
69+
observer.startedTestSuites.count, 3,
70+
"Three test suites should have started: 'All tests', 'tmp.xctest', and 'Observation'.")
5871
XCTAssertEqual(observer.startedTestCaseNames, ["Observation.test_one"])
5972
XCTAssertEqual(observer.failureDescriptions, [])
6073
XCTAssertEqual(observer.finishedTestCaseNames, [])
@@ -68,6 +81,9 @@ class Observation: XCTestCase {
6881
// CHECK: Test Case 'Observation.test_two' passed \(\d+\.\d+ seconds\).
6982
func test_two() {
7083
XCTAssertEqual(observer.startedBundlePaths.count, 1)
84+
XCTAssertEqual(
85+
observer.startedTestSuites.count, 3,
86+
"Three test suites should have started: 'All tests', 'tmp.xctest', and 'Observation'.")
7187
XCTAssertEqual(observer.startedTestCaseNames, ["Observation.test_one", "Observation.test_two"])
7288
XCTAssertEqual(observer.finishedTestCaseNames,["Observation.test_one"])
7389
XCTAssertEqual(observer.finishedBundlePaths.count, 0)
@@ -90,5 +106,8 @@ class Observation: XCTestCase {
90106
XCTMain([testCase(Observation.allTests)])
91107

92108
// CHECK: Executed 3 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
109+
// CHECK: In testSuiteDidFinish: Observation
110+
// CHECK: In testSuiteDidFinish: .*\.xctest
111+
// CHECK: In testSuiteDidFinish: All tests
93112
// CHECK: Total executed 3 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
94113
// CHECK: In testBundleDidFinish

0 commit comments

Comments
 (0)