Filelist support

Author: cltnschlosser

Sources/SwiftDriver/Driver/Driver.swift

@@ -19,6 +19,7 @@ public struct Driver {
   public enum Error: Swift.Error, Equatable, DiagnosticData {
     case invalidDriverName(String)
     case invalidInput(String)
+    case invalidArgumentValue(String, String)
     case noJobsPassedToDriverFromEmptyInputFileList
     case relativeFrontendPath(String)
     case subcommandPassedToDriver
@@ -40,6 +41,8 @@ public struct Driver {
         return "invalid driver name: \(driverName)"
       case .invalidInput(let input):
         return "invalid input: \(input)"
+      case .invalidArgumentValue(let option, let value):
+        return "invalid value '\(value)' in '\(option)'"
       case .noJobsPassedToDriverFromEmptyInputFileList:
         return "no input files"
       case .relativeFrontendPath(let path):
@@ -124,6 +127,15 @@ public struct Driver {
   /// The mapping from input files to output files for each kind.
   internal let outputFileMap: OutputFileMap?
 
+  /// The number of files required before making a file list.
+  internal let fileListThreshold: Int
+
+  /// Should use file lists for inputs (number of inputs exceeds `fileListThreshold`).
+  public let shouldUseInputFileList: Bool
+
+  /// VirtualPath for shared all sources file list. `nil` if unused.
+  public let allSourcesFileList: VirtualPath?
+
   /// The mode in which the compiler will execute.
   public let compilerMode: CompilerMode
 
@@ -263,7 +275,6 @@ public struct Driver {
       throw Error.subcommandPassedToDriver
     }
 
-
     var args = try Self.expandResponseFiles(args, fileSystem: fileSystem, diagnosticsEngine: self.diagnosticEngine)
 
     self.driverKind = try Self.determineDriverKind(args: &args)
@@ -320,6 +331,16 @@ public struct Driver {
       self.outputFileMap = outputFileMap
     }
 
+    self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)
+    self.shouldUseInputFileList = inputFiles.count > fileListThreshold
+    if shouldUseInputFileList {
+      let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation)
+      let path = RelativePath(createTemporaryFileName(prefix: "sources"))
+      self.allSourcesFileList = .fileList(path, .list(swiftInputs.map(\.file)))
+    } else {
+      self.allSourcesFileList = nil
+    }
+
     // Figure out the primary outputs from the driver.
     (self.compilerOutputType, self.linkerOutputType) = Self.determinePrimaryOutputs(&parsedOptions, driverKind: driverKind, diagnosticsEngine: diagnosticEngine)
 
@@ -802,6 +823,37 @@ extension Driver {
   }
 }
 
+extension Driver {
+  private static func computeFileListThreshold(
+    _ parsedOptions: inout ParsedOptions,
+    diagnosticsEngine: DiagnosticsEngine
+  ) throws -> Int {
+    let hasUseFileLists = parsedOptions.hasArgument(.driverUseFilelists)
+
+    if hasUseFileLists {
+      diagnosticsEngine.emit(.warn_use_filelists_deprecated)
+    }
+
+    if let threshold = parsedOptions.getLastArgument(.driverFilelistThreshold)?.asSingle {
+      if let thresholdInt = Int(threshold) {
+        return thresholdInt
+      } else {
+        throw Error.invalidArgumentValue(Option.driverFilelistThreshold.spelling, threshold)
+      }
+    } else if hasUseFileLists {
+      return 0
+    }
+
+    return 128
+  }
+}
+
+private extension Diagnostic.Message {
+  static var warn_use_filelists_deprecated: Diagnostic.Message {
+    .warning("the option '-driver-use-filelists' is deprecated; use '-driver-filelist-threshold=0' instead")
+  }
+}
+
 extension Driver {
   /// Compute the compiler mode based on the options.
   private static func computeCompilerMode(
@@ -1730,7 +1782,7 @@ extension Driver {
         return path
       }
 
-      return path.parentDirectory.appending(component: "\(moduleName).\(type.rawValue)")
+      return path.replacingExtension(with: type)
     }
 
     return try VirtualPath(path: moduleName.appendingFileTypeExtension(type))
@@ -1829,15 +1881,12 @@ extension Driver {
         _ = parsedOptions.hasArgument(isOutput)
       }
 
-      var parentPath: VirtualPath
       if let projectDirectory = projectDirectory {
         // If the build system has created a Project dir for us to include the file, use it.
-        parentPath = projectDirectory
-      } else {
-        parentPath = moduleOutputPath.parentDirectory
+        return projectDirectory.appending(component: moduleName.appendingFileTypeExtension(type))
       }
 
-      return try parentPath.appending(component: moduleName).replacingExtension(with: type)
+      return moduleOutputPath.replacingExtension(with: type)
     }
 
     // If the output option was not provided, don't produce this output at all.

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -15,7 +15,9 @@ import Foundation
 public typealias FileSystem = TSCBasic.FileSystem
 
 /// Mapping of input file paths to specific output files.
-public struct OutputFileMap: Equatable {
+public struct OutputFileMap: Hashable, Codable {
+  static let singleInputKey = VirtualPath.relative(RelativePath(""))
+
   /// The known mapping from input file to specific output files.
   public var entries: [VirtualPath : [FileType : VirtualPath]] = [:]
 
@@ -46,14 +48,14 @@ public struct OutputFileMap: Equatable {
   }
 
   public func existingOutputForSingleInput(outputType: FileType) -> VirtualPath? {
-    try! existingOutput(inputFile: VirtualPath(path: ""), outputType: outputType)
+    existingOutput(inputFile: Self.singleInputKey, outputType: outputType)
   }
 
   public func resolveRelativePaths(relativeTo absPath: AbsolutePath) -> OutputFileMap {
     let resolvedKeyValues: [(VirtualPath, [FileType : VirtualPath])] = entries.map {
       let resolvedKey: VirtualPath
       // Special case for single dependency record, leave it as is
-      if $0.key == .relative(.init("")) {
+      if $0.key == Self.singleInputKey {
         resolvedKey = $0.key
       } else {
         resolvedKey = $0.key.resolvedRelativePath(base: absPath)
@@ -78,7 +80,7 @@ public struct OutputFileMap: Equatable {
     let contents = try fileSystem.readFileContents(file)
     let result = try JSONDecoder().decode(OutputFileMapJSON.self, from: Data(contents.contents))
 
-    // Convert the loaded entries into virual output file map.
+    // Convert the loaded entries into virtual output file map.
     var outputFileMap = OutputFileMap()
     outputFileMap.entries = try result.toVirtualOutputFileMap()
 

Sources/SwiftDriver/Execution/ArgsResolver.swift

@@ -11,11 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 import TSCBasic
+@_implementationOnly import Yams
 
 /// Resolver for a job's argument template.
-public struct ArgsResolver {
+public final class ArgsResolver {
   /// The map of virtual path to the actual path.
-  public var pathMapping: [VirtualPath: AbsolutePath]
+  public var pathMapping: [VirtualPath: String]
 
   /// The file system used by the resolver.
   private let fileSystem: FileSystem
@@ -24,13 +25,16 @@ public struct ArgsResolver {
   // FIXME: We probably need a dedicated type for this...
   private let temporaryDirectory: VirtualPath
 
+  private let lock = Lock()
+
   public init(fileSystem: FileSystem, temporaryDirectory: VirtualPath? = nil) throws {
     self.pathMapping = [:]
     self.fileSystem = fileSystem
 
     if let temporaryDirectory = temporaryDirectory {
       self.temporaryDirectory = temporaryDirectory
     } else {
+      // FIXME: withTemporaryDirectory uses FileManager.default, need to create a FileSystem.temporaryDirectory api.
       let tmpDir: AbsolutePath = try withTemporaryDirectory(removeTreeOnDeinit: false) { path in
         // FIXME: TSC removes empty directories even when removeTreeOnDeinit is false. This seems like a bug.
         try fileSystem.writeFileContents(path.appending(component: ".keep-directory")) { $0 <<< "" }
@@ -60,22 +64,76 @@ public struct ArgsResolver {
       return flag
 
     case .path(let path):
-      // Return the path from the temporary directory if this is a temporary file.
-      if path.isTemporary {
-        let actualPath = temporaryDirectory.appending(component: path.name)
-        return actualPath.name
+      return try lock.withLock {
+        return try unsafeResolve(path: path)
       }
+    }
+  }
+
+  /// Needs to be done inside of `lock`. Marked unsafe to make that more obvious.
+  private func unsafeResolve(path: VirtualPath) throws -> String {
+    // If there was a path mapping, use it.
+    if let actualPath = pathMapping[path] {
+      return actualPath
+    }
+
+    // Return the path from the temporary directory if this is a temporary file.
+    if path.isTemporary {
+      let actualPath = temporaryDirectory.appending(component: path.name)
+      if case let .fileList(_, fileList) = path {
+        switch fileList {
+        case let .list(items):
+          try createFileList(path: actualPath, contents: items)
+        case let .outputFileMap(map):
+          try createFileList(path: actualPath, outputFileMap: map)
+        }
+      }
+      let result = actualPath.name
+      pathMapping[path] = result
+      return result
+    }
 
-      // If there was a path mapping, use it.
-      if let actualPath = pathMapping[path] {
-        return actualPath.pathString
+    // Otherwise, return the path.
+    let result = path.name
+    pathMapping[path] = result
+    return result
+  }
+
+  private func createFileList(path: VirtualPath, contents: [VirtualPath]) throws {
+    // FIXME: Need a way to support this for distributed build systems...
+    if let absPath = path.absolutePath {
+      try fileSystem.writeFileContents(absPath) { out in
+        for path in contents {
+          try! out <<< unsafeResolve(path: path) <<< "\n"
+        }
       }
+    }
+  }
 
-      // Otherwise, return the path.
-      return path.name
+  private func createFileList(path: VirtualPath, outputFileMap: OutputFileMap) throws {
+    // FIXME: Need a way to support this for distributed build systems...
+    if let absPath = path.absolutePath {
+      // This uses Yams to escape and quote strings, but not to output the whole yaml file because
+      // it sometimes outputs mappings in explicit block format (https://yaml.org/spec/1.2/spec.html#id2798057)
+      // and the frontend (llvm) only seems to support implicit block format.
+      try fileSystem.writeFileContents(absPath) { out in
+        for (input, map) in outputFileMap.entries {
+          out <<< quoteAndEscape(path: input) <<< ":\n"
+          for (type, output) in map {
+            out <<< "  " <<< type.name <<< ": " <<< quoteAndEscape(path: output) <<< "\n"
+          }
+        }
+      }
     }
   }
 
+  private func quoteAndEscape(path: VirtualPath) -> String {
+    let inputNode = Node.scalar(Node.Scalar(try! unsafeResolve(path: path), Tag(.str), .doubleQuoted))
+    let string = try! Yams.serialize(node: inputNode)
+    // Remove the newline from the end
+    return string.trimmingCharacters(in: .whitespacesAndNewlines)
+  }
+
   private func createResponseFileIfNeeded(for job: Job, resolvedArguments: inout [String], forceResponseFiles: Bool) throws -> Bool {
     if forceResponseFiles ||
       (job.supportsResponseFiles && !commandLineFitsWithinSystemLimits(path: resolvedArguments[0], args: resolvedArguments)) {

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -14,6 +14,8 @@ import TSCBasic
 import Foundation
 
 public protocol DriverExecutor {
+  var resolver: ArgsResolver { get }
+
   /// Execute a single job and capture the output.
   @discardableResult
   func execute(job: Job,
@@ -82,7 +84,7 @@ public final class SwiftDriverExecutor: DriverExecutor {
   let diagnosticsEngine: DiagnosticsEngine
   let processSet: ProcessSet
   let fileSystem: FileSystem
-  let resolver: ArgsResolver
+  public let resolver: ArgsResolver
   let env: [String: String]
 
   public init(diagnosticsEngine: DiagnosticsEngine,
@@ -109,13 +111,13 @@ public final class SwiftDriverExecutor: DriverExecutor {
       for (envVar, value) in job.extraEnvironment {
         try ProcessEnv.setVar(envVar, value: value)
       }
-      
+
       try exec(path: arguments[0], args: arguments)
       fatalError("unreachable, exec always throws on failure")
     } else {
       var childEnv = env
       childEnv.merge(job.extraEnvironment, uniquingKeysWith: { (_, new) in new })
-      
+
       let process = try Process.launchProcess(arguments: arguments, env: childEnv)
       return try process.waitUntilExit()
     }