Build: migrate to file lists for driver invocations

When invoking the driver use file lists rather than passing all the
sources on the command line.  This is particularly import for Windows
where the command line limit is 32k.  With deep dependency sets and
large projects, it is possible to overrun this limit and fail to invoke
the command.  The driver itself will invoke the frontend using response
files if the invocation will overflow the command line limit.

The test expectations need to be altered as we will use the build
directories to store the file list which is the temps path.

Author: compnerd

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -546,7 +546,8 @@ public final class SwiftTargetBuildDescription {
         }
 
         result.append("-c")
-        result.append(contentsOf: self.sources.map(\.pathString))
+        // FIXME: Eliminate side effect.
+        try result.append("@\(self.writeFileList(includeDerivedSources: true).pathString)")
 
         result.append("-I")
         result.append(self.buildParameters.buildPath.pathString)
@@ -578,9 +579,8 @@ public final class SwiftTargetBuildDescription {
 
         // FIXME: Handle WMO
 
-        for source in self.target.sources.paths {
-            result.append(source.pathString)
-        }
+        // FIXME: Eliminate side effect.
+        try result.append("@\(self.writeFileList().pathString)")
 
         result.append("-I")
         result.append(self.buildParameters.buildPath.pathString)
@@ -631,9 +631,8 @@ public final class SwiftTargetBuildDescription {
         // FIXME: Handle WMO
 
         result.append("-c")
-        for source in self.target.sources.paths {
-            result.append(source.pathString)
-        }
+        // FIXME: Eliminate side effect.
+        try result.append("@\(self.writeFileList().pathString)")
 
         result.append("-I")
         result.append(self.buildParameters.buildPath.pathString)
@@ -677,6 +676,13 @@ public final class SwiftTargetBuildDescription {
         self.buildParameters.triple.isDarwin() && self.target.type == .library
     }
 
+    private func writeFileList(includeDerivedSources: Bool = false) throws -> AbsolutePath {
+        let path = self.tempsPath.appending(component: "\(self.target.name)-\(includeDerivedSources ? "all-" : "")sources.filelist")
+        let contents = (includeDerivedSources ? self.sources : self.target.sources.paths).map { $0.pathString.spm_shellEscaped() }.joined(separator: "\n")
+        try self.fileSystem.writeFileContents(path, string: contents)
+        return path
+    }
+
     private func writeOutputFileMap() throws -> AbsolutePath {
         let path = self.tempsPath.appending("output-file-map.json")
         let masterDepsPath = self.tempsPath.appending("master.swiftdeps")

Tests/CommandsTests/BuildToolTests.swift

@@ -150,7 +150,7 @@ final class BuildToolTests: CommandsTestCase {
             do {
                 let result = try build(["--product", "exec1"], packagePath: fullPath)
                 XCTAssertMatch(result.binContents, ["exec1"])
-                XCTAssertNoMatch(result.binContents, ["exec2.build"])
+                XCTAssertMatch(result.binContents, ["exec2.build"])
             }
 
             do {
@@ -219,9 +219,9 @@ final class BuildToolTests: CommandsTestCase {
             do {
                 let result = try build([], packagePath: aPath)
                 XCTAssertNoMatch(result.binContents, ["bexec"])
-                XCTAssertNoMatch(result.binContents, ["BTarget2.build"])
+                XCTAssertMatch(result.binContents, ["BTarget2.build"])
                 XCTAssertNoMatch(result.binContents, ["cexec"])
-                XCTAssertNoMatch(result.binContents, ["CTarget.build"])
+                XCTAssertMatch(result.binContents, ["CTarget.build"])
             }
 
             // Dependency contains a dependent product
@@ -231,15 +231,15 @@ final class BuildToolTests: CommandsTestCase {
                 XCTAssertMatch(result.binContents, ["BTarget2.build"])
                 XCTAssertMatch(result.binContents, ["bexec"])
                 XCTAssertNoMatch(result.binContents, ["aexec"])
-                XCTAssertNoMatch(result.binContents, ["ATarget.build"])
+                XCTAssertMatch(result.binContents, ["ATarget.build"])
                 XCTAssertNoMatch(result.binContents, ["BLibrary.a"])
 
                 // FIXME: We create the modulemap during build planning, hence this uglyness.
                 let bTargetBuildDir = ((try? localFileSystem.getDirectoryContents(result.binPath.appending("BTarget1.build"))) ?? []).filter{ $0 != moduleMapFilename }
                 XCTAssertTrue(bTargetBuildDir.isEmpty, "bTargetBuildDir should be empty")
 
                 XCTAssertNoMatch(result.binContents, ["cexec"])
-                XCTAssertNoMatch(result.binContents, ["CTarget.build"])
+                XCTAssertMatch(result.binContents, ["CTarget.build"])
 
                 // Also make sure we didn't emit parseable module interfaces
                 // (do this here to avoid doing a second build in