Build: migrate to file lists for driver invocations

When invoking the driver use file lists rather than passing all the
sources on the command line.  This is particularly import for Windows
where the command line limit is 32k.  With deep dependency sets and
large projects, it is possible to overrun this limit and fail to invoke
the command.  The driver itself will invoke the frontend using response
files if the invocation will overflow the command line limit.

The test expectations need to be altered as we will use the build
directories to store the file list which is the temps path.

Author: compnerd

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -558,7 +558,8 @@ public final class SwiftTargetBuildDescription {
         }
 
         result.append("-c")
-        result.append(contentsOf: self.sources.map(\.pathString))
+        // FIXME: Eliminate side effect.
+        try result.append("@\(self.writeFileList(self.sources, "all-").pathString)")
 
         result.append("-I")
         result.append(self.buildParameters.buildPath.pathString)
@@ -590,9 +591,8 @@ public final class SwiftTargetBuildDescription {
 
         // FIXME: Handle WMO
 
-        for source in self.target.sources.paths {
-            result.append(source.pathString)
-        }
+        // FIXME: Eliminate side effect.
+        try result.append("@\(self.writeFileList(self.target.sources.paths).pathString)")
 
         result.append("-I")
         result.append(self.buildParameters.buildPath.pathString)
@@ -643,9 +643,8 @@ public final class SwiftTargetBuildDescription {
         // FIXME: Handle WMO
 
         result.append("-c")
-        for source in self.target.sources.paths {
-            result.append(source.pathString)
-        }
+        // FIXME: Eliminate side effect.
+        try result.append("@\(self.writeFileList(self.target.sources.paths).pathString)")
 
         result.append("-I")
         result.append(self.buildParameters.buildPath.pathString)
@@ -689,6 +688,13 @@ public final class SwiftTargetBuildDescription {
         self.buildParameters.triple.isDarwin() && self.target.type == .library
     }
 
+    func writeFileList(_ sources: [AbsolutePath], _ variant: String = "") throws -> AbsolutePath {
+        let path = self.tempsPath.appending(component: "\(self.target.name)-\(variant)sources.filelist")
+        let contents = sources.map { $0.pathString.spm_shellEscaped() }.joined(separator: "\n")
+        try self.fileSystem.writeFileContents(path, string: contents)
+        return path
+    }
+
     func writeOutputFileMap() throws -> AbsolutePath {
         let path = self.tempsPath.appending("output-file-map.json")
         let masterDepsPath = self.tempsPath.appending("master.swiftdeps")

Sources/Build/LLBuildManifestBuilder.swift

@@ -626,7 +626,8 @@ extension LLBuildManifestBuilder {
             tempsPath: target.tempsPath,
             objects: try target.objects,
             otherArguments: try target.compileArguments(),
-            sources: target.sources,
+            sourceCount: target.sources.count,
+            fileListPath: try target.writeFileList(target.sources),
             isLibrary: isLibrary,
             wholeModuleOptimization: self.buildParameters.configuration == .release,
             outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.

Sources/LLBuildManifest/BuildManifest.swift

@@ -171,7 +171,8 @@ public struct BuildManifest {
         tempsPath: AbsolutePath,
         objects: [AbsolutePath],
         otherArguments: [String],
-        sources: [AbsolutePath],
+        sourceCount: Int,
+        fileListPath: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
         outputFileMapPath: AbsolutePath
@@ -188,7 +189,8 @@ public struct BuildManifest {
             tempsPath: tempsPath,
             objects: objects,
             otherArguments: otherArguments,
-            sources: sources,
+            sourceCount: sourceCount,
+            fileList: fileListPath,
             isLibrary: isLibrary,
             wholeModuleOptimization: wholeModuleOptimization,
             outputFileMapPath: outputFileMapPath

Sources/LLBuildManifest/Tools.swift

@@ -241,7 +241,8 @@ public struct SwiftCompilerTool: ToolProtocol {
     public var tempsPath: AbsolutePath
     public var objects: [AbsolutePath]
     public var otherArguments: [String]
-    public var sources: [AbsolutePath]
+    public var sourceCount: Int
+    public var fileList: AbsolutePath
     public var isLibrary: Bool
     public var wholeModuleOptimization: Bool
     public var outputFileMapPath: AbsolutePath
@@ -257,7 +258,8 @@ public struct SwiftCompilerTool: ToolProtocol {
         tempsPath: AbsolutePath,
         objects: [AbsolutePath],
         otherArguments: [String],
-        sources: [AbsolutePath],
+        sourceCount: Int,
+        fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
         outputFileMapPath: AbsolutePath
@@ -272,14 +274,15 @@ public struct SwiftCompilerTool: ToolProtocol {
         self.tempsPath = tempsPath
         self.objects = objects
         self.otherArguments = otherArguments
-        self.sources = sources
+        self.sourceCount = sourceCount
+        self.fileList = fileList
         self.isLibrary = isLibrary
         self.wholeModuleOptimization = wholeModuleOptimization
         self.outputFileMapPath = outputFileMapPath
     }
 
     var description: String {
-        return "Compiling Swift Module '\(moduleName)' (\(sources.count) sources)"
+        return "Compiling Swift Module '\(moduleName)' (\(sourceCount) sources)"
     }
 
     var arguments: [String] {
@@ -305,7 +308,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         if wholeModuleOptimization {
             arguments += ["-whole-module-optimization", "-num-threads", "\(Self.numThreads)"]
         }
-        arguments += ["-c"] + sources.map { $0.pathString }
+        arguments += ["-c", "@\(fileList.pathString)"]
         arguments += ["-I", importPath.pathString]
         arguments += otherArguments
         return arguments

Tests/CommandsTests/BuildToolTests.swift

@@ -156,7 +156,7 @@ final class BuildToolTests: CommandsTestCase {
             do {
                 let result = try build(["--product", "exec1"], packagePath: fullPath)
                 XCTAssertMatch(result.binContents, ["exec1"])
-                XCTAssertNoMatch(result.binContents, ["exec2.build"])
+                XCTAssertMatch(result.binContents, ["exec2.build"])
             }
 
             do {
@@ -225,9 +225,9 @@ final class BuildToolTests: CommandsTestCase {
             do {
                 let result = try build([], packagePath: aPath)
                 XCTAssertNoMatch(result.binContents, ["bexec"])
-                XCTAssertNoMatch(result.binContents, ["BTarget2.build"])
+                XCTAssertMatch(result.binContents, ["BTarget2.build"])
                 XCTAssertNoMatch(result.binContents, ["cexec"])
-                XCTAssertNoMatch(result.binContents, ["CTarget.build"])
+                XCTAssertMatch(result.binContents, ["CTarget.build"])
             }
 
             // Dependency contains a dependent product
@@ -237,15 +237,15 @@ final class BuildToolTests: CommandsTestCase {
                 XCTAssertMatch(result.binContents, ["BTarget2.build"])
                 XCTAssertMatch(result.binContents, ["bexec"])
                 XCTAssertNoMatch(result.binContents, ["aexec"])
-                XCTAssertNoMatch(result.binContents, ["ATarget.build"])
+                XCTAssertMatch(result.binContents, ["ATarget.build"])
                 XCTAssertNoMatch(result.binContents, ["BLibrary.a"])
 
                 // FIXME: We create the modulemap during build planning, hence this uglyness.
                 let bTargetBuildDir = ((try? localFileSystem.getDirectoryContents(result.binPath.appending("BTarget1.build"))) ?? []).filter{ $0 != moduleMapFilename }
                 XCTAssertTrue(bTargetBuildDir.isEmpty, "bTargetBuildDir should be empty")
 
                 XCTAssertNoMatch(result.binContents, ["cexec"])
-                XCTAssertNoMatch(result.binContents, ["CTarget.build"])
+                XCTAssertMatch(result.binContents, ["CTarget.build"])
 
                 // Also make sure we didn't emit parseable module interfaces
                 // (do this here to avoid doing a second build in