avoid deprecation warnings from test discovery (#3992)

motivation: do not prduce warning when tests are marked as deprecated, since they are designed to test deprecated functionality

changes:
* mark generated test-discovery wrappers as psuedo-deprecated so they dont produce deprecation warnings
* use @main for the test discovery entry point instead of main.swift which helps avoid deprecation warnings at the entry point
* cleanup test discovery generate  names to be easier to reason about
* add and adjust tests for throwing variants
* add test for deprecation use case

Author: tomerd

Fixtures/Miscellaneous/TestDiscovery/Async/Tests/AsyncTests/SwiftTests.swift

@@ -7,6 +7,9 @@ class AsyncTests: XCTestCase {
     func testAsync() async {
     }
 
+    func testAsyncThrows() async throws {
+    }
+
     @MainActor func testMainActor() async {
         XCTAssertTrue(Thread.isMainThread)
     }

Fixtures/Miscellaneous/TestDiscovery/Deprecation/Package.swift

@@ -0,0 +1,10 @@
+// swift-tools-version:4.2
+import PackageDescription
+
+let package = Package(
+    name: "Simple",
+    targets: [
+        .target(name: "Simple"),
+        .testTarget(name: "SimpleTests", dependencies: ["Simple"]),
+    ]
+)

Fixtures/Miscellaneous/TestDiscovery/Deprecation/Sources/Simple/Simple.swift

@@ -0,0 +1,6 @@
+struct Simple {
+    func hello() {}
+
+    @available(*, deprecated, message: "use hello instead")
+    func deprecatedHello() {}
+}

Fixtures/Miscellaneous/TestDiscovery/Deprecation/Tests/SimpleTests/SwiftTests.swift

@@ -0,0 +1,13 @@
+import XCTest
+@testable import Simple
+
+class SimpleTests: XCTestCase {
+    func testHello() {
+        Simple().hello()
+    }
+
+    @available(*, deprecated, message: "testing deprecated API")
+    func testDeprecatedHello() {
+        Simple().deprecatedHello()
+    }
+}

Fixtures/Miscellaneous/TestDiscovery/Simple/Tests/SimpleTests/SwiftTests.swift

@@ -9,7 +9,10 @@ class SimpleTests: XCTestCase {
     func test_Example2() {
     }
 
-    func testExample3(arg: String) {
+    func testThrowing() throws {
+    }
+
+    func testWithArgs(arg: String) {
     }
 
     func nontest() {

Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

@@ -73,7 +73,8 @@ final class TestDiscoveryCommand: CustomLLBuildCommand {
             let testMethods = iterator.value.flatMap{ $0.testMethods }
             stream <<< "\n"
             stream <<< "fileprivate extension " <<< className <<< " {" <<< "\n"
-            stream <<< indent(4) <<< "static let __allTests__\(className) = [" <<< "\n"
+            stream <<< indent(4) <<< "@available(*, deprecated, message: \"Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings\")" <<< "\n"
+            stream <<< indent(4) <<< "static let __allTests = [" <<< "\n"
             for method in testMethods {
                 stream <<< indent(8) <<< method.allTestsEntry <<< ",\n"
             }
@@ -82,13 +83,14 @@ final class TestDiscoveryCommand: CustomLLBuildCommand {
         }
 
         stream <<< """
-        func __allTests_\(module)() -> [XCTestCaseEntry] {
+        @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings")
+        func __\(module)__allTests() -> [XCTestCaseEntry] {
             return [\n
         """
 
         for iterator in testsByClassNames {
             let className = iterator.key
-            stream <<< indent(8) <<< "testCase(\(className).__allTests__\(className)),\n"
+            stream <<< indent(8) <<< "testCase(\(className).__allTests),\n"
         }
 
         stream <<< """
@@ -113,7 +115,7 @@ final class TestDiscoveryCommand: CustomLLBuildCommand {
         let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() })
 
         func isMainFile(_ path: AbsolutePath) -> Bool {
-            return path.basename == "main.swift"
+            return path.basename == LLBuildManifest.TestDiscoveryTool.mainFileName
         }
 
         var maybeMainFile: AbsolutePath?
@@ -140,19 +142,26 @@ final class TestDiscoveryCommand: CustomLLBuildCommand {
         }
 
         guard let mainFile = maybeMainFile else {
-            throw InternalError("unknown main file")
+            throw InternalError("main output (\(LLBuildManifest.TestDiscoveryTool.mainFileName)) not found")
         }
 
         // Write the main file.
         let stream = try LocalFileOutputByteStream(mainFile)
 
         stream <<< "import XCTest" <<< "\n\n"
-        stream <<< "var tests = [XCTestCaseEntry]()" <<< "\n"
+
+        stream <<< "@main" <<< "\n"
+        stream <<< "@available(*, deprecated, message: \"Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings\")" <<< "\n"
+        stream <<< "struct Runner" <<< " {" <<< "\n"
+        stream <<< indent(4) <<< "static func main()" <<< " {" <<< "\n"
+        stream <<< indent(8) <<< "var tests = [XCTestCaseEntry]()" <<< "\n"
         for module in testsByModule.keys {
-            stream <<< "tests += __allTests_\(module)()" <<< "\n"
+            stream <<< indent(8) <<< "tests += __\(module)__allTests()" <<< "\n"
         }
-        stream <<< "\n"
-        stream <<< "XCTMain(tests)" <<< "\n"
+        stream <<< indent(8) <<< "\n"
+        stream <<< indent(8) <<< "XCTMain(tests)" <<< "\n"
+        stream <<< indent(4) <<< "}" <<< "\n"
+        stream <<< "}" <<< "\n"
 
         stream.flush()
     }

Sources/Build/BuildPlan.swift

@@ -1478,7 +1478,7 @@ public class BuildPlan {
             } else {
                 // We'll generate sources containing the test names as part of the build process.
                 let derivedTestListDir = buildParameters.buildPath.appending(components: "\(testProduct.name).derived")
-                let mainFile = derivedTestListDir.appending(component: "main.swift")
+                let mainFile = derivedTestListDir.appending(component: "runner.swift")
 
                 var paths: [AbsolutePath] = []
                 paths.append(mainFile)

Sources/Build/LLBuildManifestBuilder.swift

@@ -832,8 +832,8 @@ extension LLBuildManifestBuilder {
             let objectFiles = testTargets.flatMap{ $0.objects }.sorted().map(Node.file)
             let outputs = testDiscoveryTarget.target.sources.paths
 
-            guard let mainOutput = (outputs.first{ $0.basename == "main.swift" }) else {
-                throw InternalError("output main.swift not found")
+            guard let mainOutput = (outputs.first{ $0.basename == TestDiscoveryTool.mainFileName }) else {
+                throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found")
             }
             let cmdName = mainOutput.pathString
             manifest.addTestDiscoveryCmd(

Sources/LLBuildManifest/Tools.swift

@@ -44,6 +44,7 @@ public struct PhonyTool: ToolProtocol {
 
 public struct TestDiscoveryTool: ToolProtocol {
     public static let name: String = "test-discovery-tool"
+    public static let mainFileName: String = "runner.swift"
 
     public var inputs: [Node]
     public var outputs: [Node]

Tests/FunctionalTests/MiscellaneousTests.swift

@@ -550,6 +550,7 @@ class MiscellaneousTestCase: XCTestCase {
                     static let __allTests__SimpleTests = [
                         ("test_Example2", test_Example2),
                         ("testExample1", testExample1),
+                        ("testThrowing", testThrowing),
                     ]
                 }
 

Tests/FunctionalTests/TestDiscoveryTests.swift

@@ -30,10 +30,10 @@ class TestDiscoveryTests: XCTestCase {
             let (stdout, stderr) = try executeSwiftTest(path)
             #if os(macOS)
             XCTAssertMatch(stdout, .contains("module Simple"))
-            XCTAssertMatch(stderr, .contains("Executed 2 tests"))
+            XCTAssertMatch(stderr, .contains("Executed 3 tests"))
             #else
             XCTAssertMatch(stdout, .contains("module Simple"))
-            XCTAssertMatch(stdout, .contains("Executed 2 tests"))
+            XCTAssertMatch(stdout, .contains("Executed 3 tests"))
             #endif
         }
     }
@@ -56,10 +56,10 @@ class TestDiscoveryTests: XCTestCase {
             let (stdout, stderr) = try executeSwiftTest(path)
             #if os(macOS)
             XCTAssertMatch(stdout, .contains("module Async"))
-            XCTAssertMatch(stderr, .contains("Executed 3 tests"))
+            XCTAssertMatch(stderr, .contains("Executed 4 tests"))
             #else
             XCTAssertMatch(stdout, .contains("module Async"))
-            XCTAssertMatch(stdout, .contains("Executed 3 tests"))
+            XCTAssertMatch(stdout, .contains("Executed 4 tests"))
             #endif
         }
     }
@@ -115,4 +115,16 @@ class TestDiscoveryTests: XCTestCase {
         }
         #endif
     }
+
+    func testDeprecatedTests() throws {
+        #if os(macOS)
+        try XCTSkipIf(true)
+        #else
+        fixture(name: "Miscellaneous/TestDiscovery/Deprecation") { path in
+            let (stdout, _) = try executeSwiftTest(path, extraArgs: ["--enable-test-discovery"])
+            XCTAssertMatch(stdout, .contains("Executed 2 tests"))
+            XCTAssertNoMatch(stdout, .contains("is deprecated"))
+        }
+        #endif
+    }
 }