[SR-1541] Listing test methods

Documentation/Linux.md

@@ -4,22 +4,22 @@ When running on the Objective-C runtime, XCTest is able to find all of your test
 
 ```swift
 class TestNSURL : XCTestCase {
-    static var allTests = {
-        return [
-            ("test_URLStrings", test_URLStrings),
-            ("test_fileURLWithPath_relativeToURL", test_fileURLWithPath_relativeToURL),
-            ("test_fileURLWithPath", test_fileURLWithPath),
-            ("test_fileURLWithPath_isDirectory", test_fileURLWithPath_isDirectory),
-            // Other tests go here
-        ]
-    }()
-
-    func test_fileURLWithPath_relativeToURL() {
-        // Write your test here. Most of the XCTAssert macros you are familiar with are available.
-        XCTAssertTrue(theBestNumber == 42, "The number is wrong")
-    }
-
-    // Other tests go here
+static var allTests = {
+return [
+("test_URLStrings", test_URLStrings),
+("test_fileURLWithPath_relativeToURL", test_fileURLWithPath_relativeToURL),
+("test_fileURLWithPath", test_fileURLWithPath),
+("test_fileURLWithPath_isDirectory", test_fileURLWithPath_isDirectory),
+// Other tests go here
+]
+}()
+
+func test_fileURLWithPath_relativeToURL() {
+// Write your test here. Most of the XCTAssert macros you are familiar with are available.
+XCTAssertTrue(theBestNumber == 42, "The number is wrong")
+}
+
+// Other tests go here
 }
 ```
 
@@ -29,11 +29,27 @@ Also, this version of XCTest does not use the external test runner binary. Inste
 XCTMain([testCase(TestNSString.allTests), testCase(TestNSArray.allTests), testCase(TestNSDictionary.allTests)])
 ```
 
-The `XCTMain` function does not return, and will cause your test app to exit with either `0` for success or `1` for failure. Command line arguments given to the executable can be used to select a particular test or test case to execute. For example:
+We are currently investigating ideas on how to make these additional steps for test discovery automatic when running on the Swift runtime.
+
+# Command Line Usage
+The `XCTMain` function does not return, and will cause your test app to exit with either `0` for success or `1` for failure. Certain command line arguments can be used to modify the test runner behavior:
+
+* A particular test or test case can be selected to execute. For example:
 
 ```sh
-./FooTests FooTestCase/testFoo  # Run a single test case
-./FooTests FooTestCase          # Run all the tests in FooTestCase
+$ ./FooTests Tests.FooTestCase/testFoo  # Run a single test case
+$ ./FooTests Tests.FooTestCase          # Run all the tests in FooTestCase
 ```
+* Tests can be listed, instead of executed.
 
-We are currently investigating ideas on how to make these additional steps for test discovery automatic when running on the Swift runtime.
+```sh
+$ ./FooTests --list-tests
+Listing 4 tests in FooTests.xctest:
+
+Tests.FooTestCase/testFoo
+Tests.FooTestCase/testBar
+Tests.BarTestCase/test123
+
+$ ./FooTests --dump-tests-json
+{"tests":[{"tests":[{"tests":[{"name":"testFoo"},{"name":"testBar"}],"name":"Tests.FooTestCase"},{"tests":[{"name":"test123"}],"name":"Tests.BarTestCase"}],"name":"Tests.xctest"}],"name":"All tests"}
+```

Sources/XCTest/ArgumentParser.swift

@@ -8,17 +8,58 @@
 //
 //
 //  ArgumentParser.swift
-//  Tools for parsing test execution configuration from command line arguments
+//  Tools for parsing test execution configuration from command line arguments.
 //
 
+/// Utility for converting command line arguments into a strongly-typed
+/// representation of the passed-in options
 internal struct ArgumentParser {
+
+    /// The basic operations that can be performed by an XCTest runner executable
+    enum ExecutionMode {
+        /// Run a test or test suite, printing results to stdout and exiting with
+        /// a non-0 return code if any tests failed. The name of a test or class
+        /// may be provided to only run a subset of test cases.
+        case run(selectedTestName: String?)
+
+        /// The different ways that the tests can be represented when they are listed
+        enum ListType {
+            /// A flat list of the tests that can be run. The lines in this
+            /// output are valid test names for the `run` mode.
+            case humanReadable
+
+            /// A JSON representation of the test suite, intended for consumption
+            /// by other tools
+            case json
+        }
+
+        /// Print a list of all the tests in the suite.
+        case list(type: ListType)
+
+        var selectedTestName: String? {
+            if case .run(let name) = self {
+                return name
+            } else {
+                return nil
+            }
+        }
+    }
+
     private let arguments: [String]
 
     init(arguments: [String] = Process.arguments) {
         self.arguments = arguments
     }
 
-    var selectedTestName: String? {
-        return arguments.count > 1 ? arguments[1] : nil
+    var executionMode: ExecutionMode {
+        if arguments.count <= 1 {
+            return .run(selectedTestName: nil)
+        } else if arguments[1] == "--list-tests" || arguments[1] == "-l" {
+            return .list(type: .humanReadable)
+        } else if arguments[1] == "--dump-tests-json" {
+            return .list(type: .json)
+        } else {
+            return .run(selectedTestName: arguments[1])
+        }
     }
 }

Sources/XCTest/TestFiltering.swift

@@ -16,7 +16,7 @@ internal typealias TestFilter = (XCTestCase.Type, String) -> Bool
 internal struct TestFiltering {
     private let selectedTestName: String?
 
-    init(selectedTestName: String? = ArgumentParser().selectedTestName) {
+    init(selectedTestName: String?) {
         self.selectedTestName = selectedTestName
     }
 

Sources/XCTest/TestListing.swift

@@ -0,0 +1,107 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  TestListing.swift
+//  Implementation of the mode for printing the list of tests.
+//
+
+#if os(Linux) || os(FreeBSD)
+    import Foundation
+#else
+    import SwiftFoundation
+#endif
+
+internal struct TestListing {
+    private let testSuite: XCTestSuite
+
+    init(testSuite: XCTestSuite) {
+        self.testSuite = testSuite
+    }
+
+    /// Prints a flat list of the tests in the suite, in the format used to
+    /// specify a test by name when running tests.
+    func printTestList() {
+        let list = testSuite.list()
+        let tests = list.count == 1 ? "test" : "tests"
+        let bundleName = testSuite.findBundleTestSuite()?.name ?? "<<unknown bundle>>"
+
+        print("Listing \(list.count) \(tests) in \(bundleName):\n")
+        for entry in testSuite.list() {
+            print(entry)
+        }
+    }
+
+    /// Prints a JSON representation of the tests in the suite, mirring the internal
+    /// tree representation of test suites and test cases. This output is intended
+    /// to be consumed by other tools.
+    func printTestJSON() {
+        let json = try! NSJSONSerialization.data(withJSONObject: testSuite.dictionaryRepresentation())
+        print(NSString(data: json, encoding: NSUTF8StringEncoding)!)
+    }
+}
+
+protocol Listable {
+    func list() -> [String]
+    func dictionaryRepresentation() -> NSDictionary
+}
+
+private func moduleName(value: Any) -> String {
+    let moduleAndType = String(reflecting: value.dynamicType)
+    return String(moduleAndType.characters.split(separator: ".").first!)
+}
+
+extension XCTestSuite: Listable {
+    private var listables: [Listable] {
+        return tests
+            .flatMap({ ($0 as? Listable) })
+    }
+
+    private var listingName: String {
+        if let childTestCase = tests.first as? XCTestCase where name == String(childTestCase.dynamicType) {
+            return "\(moduleName(value: childTestCase)).\(name)"
+        } else {
+            return name
+        }
+    }
+
+    func list() -> [String] {
+        return listables.flatMap({ $0.list() })
+    }
+
+    func dictionaryRepresentation() -> NSDictionary {
+        let listedTests = tests.flatMap({ ($0 as? Listable)?.dictionaryRepresentation() })
+        return [
+                   "name": listingName.bridge(),
+                   "tests": listedTests.bridge()
+            ].bridge()
+    }
+
+    func findBundleTestSuite() -> XCTestSuite? {
+        if name.hasSuffix(".xctest") {
+            return self
+        } else {
+            return tests.flatMap({ ($0 as? XCTestSuite)?.findBundleTestSuite() }).first
+        }
+    }
+}
+
+extension XCTestCase: Listable {
+    func list() -> [String] {
+        let adjustedName = name.characters
+            .split(separator: ".")
+            .map(String.init)
+            .joined(separator: "/")
+        return ["\(moduleName(value: self)).\(adjustedName)"]
+    }
+
+    func dictionaryRepresentation() -> NSDictionary {
+        let methodName = String(name.characters.split(separator: ".").last!)
+        return ["name": methodName].bridge()
+    }
+}

Sources/XCTest/XCTestMain.swift

@@ -50,22 +50,17 @@
 /// - Parameter testCases: An array of test cases run, each produced by a call to the `testCase` function
 /// - seealso: `testCase`
 @noreturn public func XCTMain(_ testCases: [XCTestCaseEntry]) {
-    // Add a test observer that prints test progress to stdout.
-    let observationCenter = XCTestObservationCenter.shared()
-    observationCenter.addTestObserver(PrintObserver())
-
-    // Announce that the test bundle will start executing.
     let testBundle = NSBundle.mainBundle()
-    observationCenter.testBundleWillStart(testBundle)
+
+    let executionMode = ArgumentParser().executionMode
 
     // Apple XCTest behaves differently if tests have been filtered:
     // - The root `XCTestSuite` is named "Selected tests" instead of
     //   "All tests".
     // - An `XCTestSuite` representing the .xctest test bundle is not included.
-    let selectedTestName = ArgumentParser().selectedTestName
     let rootTestSuite: XCTestSuite
     let currentTestSuite: XCTestSuite
-    if selectedTestName == nil {
+    if executionMode.selectedTestName == nil {
         rootTestSuite = XCTestSuite(name: "All tests")
         currentTestSuite = XCTestSuite(name: "\(testBundle.bundlePath.lastPathComponent).xctest")
         rootTestSuite.addTest(currentTestSuite)
@@ -74,13 +69,28 @@
         currentTestSuite = rootTestSuite
     }
 
-    let filter = TestFiltering(selectedTestName: selectedTestName)
+    let filter = TestFiltering(selectedTestName: executionMode.selectedTestName)
     TestFiltering.filterTests(testCases, filter: filter.selectedTestFilter)
         .map(XCTestCaseSuite.init)
         .forEach(currentTestSuite.addTest)
 
-    rootTestSuite.run()
+    switch executionMode {
+    case .list(type: .humanReadable):
+        TestListing(testSuite: rootTestSuite).printTestList()
+        exit(0)
+    case .list(type: .json):
+        TestListing(testSuite: rootTestSuite).printTestJSON()
+        exit(0)
+    case .run(selectedTestName: _):
+        // Add a test observer that prints test progress to stdout.
+        let observationCenter = XCTestObservationCenter.shared()
+        observationCenter.addTestObserver(PrintObserver())
+
+        observationCenter.testBundleWillStart(testBundle)
+        rootTestSuite.run()
+        observationCenter.testBundleDidFinish(testBundle)
 
-    observationCenter.testBundleDidFinish(testBundle)
-    exit(rootTestSuite.testRun!.totalFailureCount == 0 ? 0 : 1)
+        exit(rootTestSuite.testRun!.totalFailureCount == 0 ? 0 : 1)
+    }
 }
+

Tests/Functional/ListTests/main.swift

@@ -0,0 +1,71 @@
+// RUN: %{swiftc} %s -o %{built_tests_dir}/ListTests
+// RUN: %{built_tests_dir}/ListTests --list-tests > %t_list || true
+// RUN: %{xctest_checker} %t_list %s
+// RUN: %{built_tests_dir}/ListTests --dump-tests-json > %t_json || true
+// RUN: %{built_tests_dir}/ListTests --verify %t_json > %t_verify
+// RUN: %{xctest_checker} %t_verify verify_json.expected
+
+#if os(Linux) || os(FreeBSD)
+    import XCTest
+    import Foundation
+#else
+    import SwiftXCTest
+    import SwiftFoundation
+#endif
+
+// The JSON output isn't a stable enough format to use FileCheck-style line
+// verification directly. Instead, verify the output by deserializing the output 
+// a stable representation of the test tree for checking.
+if Process.arguments.contains("--verify") {
+    func dump(_ value: Any, prefix: String = "") {
+        guard let object = value as? [String: Any] else { return print("<<wrong type>>") }
+        guard let name = object["name"] as? String else { return print("<<missing name>>") }
+        print(prefix + name)
+        guard let children = object["tests"] as? [Any] else { return }
+        children.forEach {
+            dump($0, prefix: prefix + " ")
+        }
+    }
+
+    let deserialized = try! NSJSONSerialization.jsonObject(with: NSData(contentsOfFile: Process.arguments[2])!)
+    dump(deserialized)
+    exit(0)
+}
+
+// CHECK: Listing 4 tests in .*\.xctest:
+// CHECK: ^$
+
+class FirstTestCase: XCTestCase {
+    static var allTests = {
+        return [
+                   ("test_foo", test_foo),
+                   ("test_bar", test_bar),
+        ]
+    }()
+
+    // CHECK: ListTests.FirstTestCase/test_foo
+    func test_foo() {}
+
+    // CHECK: ListTests.FirstTestCase/test_bar
+    func test_bar() {}
+}
+
+class SecondTestCase: XCTestCase {
+    static var allTests = {
+        return [
+                   ("test_someMore", test_someMore),
+                   ("test_allTheThings", test_allTheThings),
+        ]
+    }()
+
+    // CHECK: ListTests.SecondTestCase/test_someMore
+    func test_someMore() {}
+
+    // CHECK: ListTests.SecondTestCase/test_allTheThings
+    func test_allTheThings() {}
+}
+
+XCTMain([
+            testCase(FirstTestCase.allTests),
+            testCase(SecondTestCase.allTests),
+            ])

Tests/Functional/ListTests/verify_json.expected

@@ -0,0 +1,8 @@
+// CHECK: ^All tests
+// CHECK: ^ .*\.xctest
+// CHECK: ^  ListTests.FirstTestCase
+// CHECK: ^   test_foo
+// CHECK: ^   test_bar
+// CHECK: ^  ListTests.SecondTestCase
+// CHECK: ^   test_someMore
+// CHECK: ^   test_allTheThings