Merge pull request #64 from briancroom/select-test

Allow selecting a particular test or test case to run from the command line

Author: modocache

README.md

@@ -108,6 +108,11 @@ Also, this version of XCTest does not use the external test runner binary. Inste
 XCTMain([testCase(TestNSString.allTests), testCase(TestNSArray.allTests), testCase(TestNSDictionary.allTests)])
 ```
 
-The `XCTMain` function does not return, and will cause your test app to exit with either `0` for success or `1` for failure.
+The `XCTMain` function does not return, and will cause your test app to exit with either `0` for success or `1` for failure. Command line arguments given to the executable can be used to select a particular test or test case to execute. For example:
+
+```sh
+./FooTests FooTestCase/testFoo  # Run a single test method
+./FooTests FooTestCase          # Run all the tests in FooTestCase
+```
 
 We are currently investigating ideas on how to make these additional steps for test discovery automatic when running on the Swift runtime.

Sources/XCTest/ArgumentParser.swift

@@ -0,0 +1,24 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  ArgumentParser.swift
+//  Tools for parsing test execution configuration from command line arguments
+//
+
+internal struct ArgumentParser {
+    private let arguments: [String]
+
+    init(arguments: [String] = Process.arguments) {
+        self.arguments = arguments
+    }
+
+    var selectedTestName: String? {
+        return arguments.count > 1 ? arguments[1] : nil
+    }
+}

Sources/XCTest/TestFiltering.swift

@@ -0,0 +1,78 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  XCTestFiltering.swift
+//  This provides utilities for executing only a subset of the tests provided to XCTMain
+//
+
+internal typealias TestFilter = (XCTestCase.Type, String) -> Bool
+
+internal struct TestFiltering {
+    private let selectedTestName: String?
+
+    init(selectedTestName: String? = ArgumentParser().selectedTestName) {
+        self.selectedTestName = selectedTestName
+    }
+
+    var selectedTestFilter: TestFilter {
+        if let selectedTestName = selectedTestName {
+            if let selectedTest = SelectedTest(selectedTestName: selectedTestName) {
+                return selectedTest.matches
+            } else {
+                return excludeAllFilter()
+            }
+        } else {
+            return includeAllFilter()
+        }
+    }
+
+    private func excludeAllFilter() -> TestFilter {
+        return { _ in false }
+    }
+
+    private func includeAllFilter() -> TestFilter {
+        return { _ in true }
+    }
+
+    static func filterTests(entries: [XCTestCaseEntry], filter: TestFilter) -> [XCTestCaseEntry] {
+        return entries
+            .map({ testCase, tests in
+                return (testCase, tests.filter({ filter(testCase, $0.0) }))
+            })
+            .filter({ testCase, tests in
+                return !tests.isEmpty
+            })
+    }
+}
+
+/// A selected test can be an entire test case, or a single test method
+/// within a test case.
+private struct SelectedTest {
+    let testCaseName: String
+    let testName: String?
+}
+
+private extension SelectedTest {
+    init?(selectedTestName: String) {
+        let components = selectedTestName.characters.split(separator: "/").map(String.init)
+        switch components.count {
+        case 1:
+            testCaseName = components[0]
+            testName = nil
+        case 2:
+            testCaseName = components[0]
+            testName = components[1]
+        default: return nil
+        }
+    }
+
+    func matches(testCase testCase: XCTestCase.Type, testName: String) -> Bool {
+        return String(reflecting: testCase) == testCaseName && (self.testName == nil || testName == self.testName)
+    }
+}

Sources/XCTest/XCTestMain.swift

@@ -67,11 +67,18 @@ internal struct XCTRun {
 ///
 ///     XCTMain([ testCase(TestFoo.allTests) ])
 ///
+/// Command line arguments can be used to select a particular test or test case to execute. For example:
+///
+///     ./FooTests FooTestCase/testFoo  # Run a single test method
+///     ./FooTests FooTestCase          # Run all the tests in FooTestCase
+///
 /// - Parameter testCases: An array of test cases run, each produced by a call to the `testCase` function
 /// - seealso: `testCase`
 @noreturn public func XCTMain(testCases: [XCTestCaseEntry]) {
+    let filter = TestFiltering()
+
     let overallDuration = measureTimeExecutingBlock {
-        for (testCase, tests) in testCases {
+        for (testCase, tests) in TestFiltering.filterTests(testCases, filter: filter.selectedTestFilter) {
             testCase.invokeTests(tests)
         }
     }

Tests/Functional/SelectedTest/main.swift

@@ -0,0 +1,60 @@
+// RUN: %{swiftc} %s -o %{built_tests_dir}/SelectedTest
+// RUN: %{built_tests_dir}/SelectedTest SelectedTest.ExecutedTestCase/test_foo > %T/one_test_method || true
+// RUN: %{built_tests_dir}/SelectedTest SelectedTest.ExecutedTestCase > %T/one_test_case || true
+// RUN: %{built_tests_dir}/SelectedTest > %T/all || true
+// RUN: %{xctest_checker} -p "// CHECK-METHOD:   " %T/one_test_method %s
+// RUN: %{xctest_checker} -p "// CHECK-TESTCASE: " %T/one_test_case %s
+// RUN: %{xctest_checker} -p "// CHECK-ALL:      " %T/all %s
+
+#if os(Linux) || os(FreeBSD)
+    import XCTest
+#else
+    import SwiftXCTest
+#endif
+
+class ExecutedTestCase: XCTestCase {
+    static var allTests: [(String, ExecutedTestCase -> () throws -> Void)] {
+        return [
+            ("test_bar", test_bar),
+            ("test_foo", test_foo),
+        ]
+    }
+
+// CHECK-METHOD:   Test Case 'ExecutedTestCase.test_foo' started.
+// CHECK-METHOD:   Test Case 'ExecutedTestCase.test_foo' passed \(\d+\.\d+ seconds\).
+// CHECK-TESTCASE: Test Case 'ExecutedTestCase.test_bar' started.
+// CHECK-TESTCASE: Test Case 'ExecutedTestCase.test_bar' passed \(\d+\.\d+ seconds\).
+// CHECK-ALL:      Test Case 'ExecutedTestCase.test_bar' started.
+// CHECK-ALL:      Test Case 'ExecutedTestCase.test_bar' passed \(\d+\.\d+ seconds\).
+    func test_bar() {}
+
+// CHECK-TESTCASE: Test Case 'ExecutedTestCase.test_foo' started.
+// CHECK-TESTCASE: Test Case 'ExecutedTestCase.test_foo' passed \(\d+\.\d+ seconds\).
+// CHECK-ALL:      Test Case 'ExecutedTestCase.test_foo' started.
+// CHECK-ALL:      Test Case 'ExecutedTestCase.test_foo' passed \(\d+\.\d+ seconds\).
+    func test_foo() {}
+}
+// CHECK-METHOD:   Executed 1 test, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK-TESTCASE: Executed 2 tests, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK-ALL:      Executed 2 tests, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+
+
+class SkippedTestCase: XCTestCase {
+    static var allTests: [(String, SkippedTestCase -> () throws -> Void)] {
+        return [("test_baz", test_baz)]
+    }
+
+// CHECK-ALL:      Test Case 'SkippedTestCase.test_baz' started.
+// CHECK-ALL:      Test Case 'SkippedTestCase.test_baz' passed \(\d+\.\d+ seconds\).
+    func test_baz() {}
+}
+// CHECK-ALL:      Executed 1 test, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+
+XCTMain([
+    testCase(ExecutedTestCase.allTests),
+    testCase(SkippedTestCase.allTests),
+])
+
+// CHECK-METHOD:   Total executed 1 test, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK-TESTCASE: Total executed 2 tests, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK-ALL:      Total executed 3 tests, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds

XCTest.xcodeproj/project.pbxproj

@@ -7,6 +7,8 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		AE7DD6091C8E81A0006FC722 /* ArgumentParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE7DD6071C8E81A0006FC722 /* ArgumentParser.swift */; };
+		AE7DD60A1C8E81A0006FC722 /* TestFiltering.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE7DD6081C8E81A0006FC722 /* TestFiltering.swift */; };
 		C265F66F1C3AEB6A00520CF9 /* XCTAssert.swift in Sources */ = {isa = PBXBuildFile; fileRef = C265F6691C3AEB6A00520CF9 /* XCTAssert.swift */; };
 		C265F6701C3AEB6A00520CF9 /* XCTestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = C265F66A1C3AEB6A00520CF9 /* XCTestCase.swift */; };
 		C265F6721C3AEB6A00520CF9 /* XCTestMain.swift in Sources */ = {isa = PBXBuildFile; fileRef = C265F66C1C3AEB6A00520CF9 /* XCTestMain.swift */; };
@@ -29,6 +31,8 @@
 /* Begin PBXFileReference section */
 		5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = SwiftXCTest.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		AE7DD6061C8DC6C0006FC722 /* Functional */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Functional; sourceTree = "<group>"; };
+		AE7DD6071C8E81A0006FC722 /* ArgumentParser.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ArgumentParser.swift; sourceTree = "<group>"; };
+		AE7DD6081C8E81A0006FC722 /* TestFiltering.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestFiltering.swift; sourceTree = "<group>"; };
 		B1384A411C1B3E8700EDF031 /* CONTRIBUTING.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = CONTRIBUTING.md; sourceTree = "<group>"; };
 		B1384A421C1B3E8700EDF031 /* LICENSE */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = LICENSE; sourceTree = "<group>"; };
 		B1384A431C1B3E8700EDF031 /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = "<group>"; };
@@ -97,6 +101,8 @@
 		C265F6671C3AEB6A00520CF9 /* XCTest */ = {
 			isa = PBXGroup;
 			children = (
+				AE7DD6071C8E81A0006FC722 /* ArgumentParser.swift */,
+				AE7DD6081C8E81A0006FC722 /* TestFiltering.swift */,
 				C265F6691C3AEB6A00520CF9 /* XCTAssert.swift */,
 				C265F66A1C3AEB6A00520CF9 /* XCTestCase.swift */,
 				C265F66C1C3AEB6A00520CF9 /* XCTestMain.swift */,
@@ -237,6 +243,8 @@
 				C265F6731C3AEB6A00520CF9 /* XCTimeUtilities.swift in Sources */,
 				C265F6701C3AEB6A00520CF9 /* XCTestCase.swift in Sources */,
 				DADB979C1C51BDA2005E68B6 /* XCTestExpectation.swift in Sources */,
+				AE7DD60A1C8E81A0006FC722 /* TestFiltering.swift in Sources */,
+				AE7DD6091C8E81A0006FC722 /* ArgumentParser.swift in Sources */,
 				C265F66F1C3AEB6A00520CF9 /* XCTAssert.swift in Sources */,
 				C265F6721C3AEB6A00520CF9 /* XCTestMain.swift in Sources */,
 			);