Merge branch 'master' into cs_embedBitcode

Author: cltnschlosser

Sources/SwiftDriver/CMakeLists.txt

@@ -22,6 +22,7 @@ add_library(SwiftDriver
   "Incremental Compilation/IncrementalCompilation.swift"
   "Incremental Compilation/InputIInfoMap.swift"
   "Incremental Compilation/InputInfo.swift"
+  "Incremental Compilation/SourceFileDependencyGraph.swift"
 
   Jobs/AutolinkExtractJob.swift
   Jobs/CommandLineArguments.swift

Sources/SwiftDriver/Driver/Driver.swift

@@ -77,6 +77,9 @@ public struct Driver {
   /// it should be identical to the real environment.
   public let env: [String: String]
 
+  /// The file system which we should interact with.
+  public let fileSystem: FileSystem
+
   /// Diagnostic engine for emitting warnings, errors, etc.
   public let diagnosticEngine: DiagnosticsEngine
 
@@ -229,17 +232,19 @@ public struct Driver {
   public init(
     args: [String],
     env: [String: String] = ProcessEnv.vars,
-    diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler])
+    diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
+    fileSystem: FileSystem = localFileSystem
   ) throws {
     self.env = env
+    self.fileSystem = fileSystem
 
     self.diagnosticEngine = diagnosticsEngine
 
     if case .subcommand = try Self.invocationRunMode(forArgs: args).mode {
       throw Error.subcommandPassedToDriver
     }
 
-    var args = try Self.expandResponseFiles(args, diagnosticsEngine: self.diagnosticEngine)[...]
+    var args = try Self.expandResponseFiles(args, fileSystem: fileSystem, diagnosticsEngine: self.diagnosticEngine)[...]
 
     self.driverKind = try Self.determineDriverKind(args: &args)
     self.optionTable = OptionTable()
@@ -249,7 +254,7 @@ public struct Driver {
       .map {
         Triple($0, normalizing: true)
       }
-    (self.toolchain, self.targetTriple) = try Self.computeToolchain(explicitTarget, diagnosticsEngine: diagnosticEngine, env: env)
+    (self.toolchain, self.targetTriple) = try Self.computeToolchain(explicitTarget, diagnosticsEngine: diagnosticEngine, env: env, fileSystem: fileSystem)
     self.targetVariantTriple = self.parsedOptions.getLastArgument(.targetVariant).map { Triple($0.asSingle, normalizing: true) }
 
     // Find the Swift compiler executable.
@@ -269,7 +274,7 @@ public struct Driver {
 
     // Compute the working directory.
     workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
-      let cwd = localFileSystem.currentWorkingDirectory
+      let cwd = fileSystem.currentWorkingDirectory
       return try cwd.map{ AbsolutePath(workingDirectoryArg.asSingle, relativeTo: $0) } ?? AbsolutePath(validating: workingDirectoryArg.asSingle)
     }
 
@@ -283,7 +288,7 @@ public struct Driver {
     self.inputFiles = inputFiles
     self.recordedInputModificationDates = .init(uniqueKeysWithValues:
       Set(inputFiles).compactMap {
-        guard let modTime = try? localFileSystem
+        guard let modTime = try? fileSystem
           .getFileInfo($0.file).modTime else { return nil }
         return ($0, modTime)
     })
@@ -292,7 +297,7 @@ public struct Driver {
     // Initialize an empty output file map, which will be populated when we start creating jobs.
     if let outputFileMapArg = parsedOptions.getLastArgument(.outputFileMap)?.asSingle {
       let path = try VirtualPath(path: outputFileMapArg)
-      outputFileMap = try .load(file: path, diagnosticEngine: diagnosticEngine)
+        outputFileMap = try .load(fileSystem: fileSystem, file: path, diagnosticEngine: diagnosticEngine)
     } else {
       outputFileMap = nil
     }
@@ -330,13 +335,14 @@ public struct Driver {
       compilerMode: compilerMode,
       outputFileMap: self.outputFileMap,
       compilerOutputType: self.compilerOutputType,
-      moduleOutput: self.moduleOutput,
+        moduleOutput: self.moduleOutput,
+      fileSystem: fileSystem,
       inputFiles: inputFiles,
       diagnosticEngine: diagnosticEngine,
       actualSwiftVersion: try? toolchain.swiftCompilerVersion()
     )
 
-    self.sdkPath = Self.computeSDKPath(&parsedOptions, compilerMode: compilerMode, toolchain: toolchain, diagnosticsEngine: diagnosticEngine, env: env)
+    self.sdkPath = Self.computeSDKPath(&parsedOptions, compilerMode: compilerMode, toolchain: toolchain, fileSystem: fileSystem, diagnosticsEngine: diagnosticEngine, env: env)
 
     self.importedObjCHeader = try Self.computeImportedObjCHeader(&parsedOptions, compilerMode: compilerMode, diagnosticEngine: diagnosticEngine)
     self.bridgingPrecompiledHeader = try Self.computeBridgingPrecompiledHeader(&parsedOptions,
@@ -534,6 +540,7 @@ extension Driver {
   /// - Parameter visitedResponseFiles: Set containing visited response files to detect recursive parsing.
   private static func expandResponseFiles(
     _ args: [String],
+    fileSystem: FileSystem,
     diagnosticsEngine: DiagnosticsEngine,
     visitedResponseFiles: inout Set<AbsolutePath>
   ) throws -> [String] {
@@ -551,9 +558,9 @@ extension Driver {
           visitedResponseFiles.remove(responseFile)
         }
 
-        let contents = try localFileSystem.readFileContents(responseFile).cString
+        let contents = try fileSystem.readFileContents(responseFile).cString
         let lines = tokenizeResponseFile(contents)
-        result.append(contentsOf: try expandResponseFiles(lines, diagnosticsEngine: diagnosticsEngine, visitedResponseFiles: &visitedResponseFiles))
+        result.append(contentsOf: try expandResponseFiles(lines, fileSystem: fileSystem, diagnosticsEngine: diagnosticsEngine, visitedResponseFiles: &visitedResponseFiles))
       } else {
         result.append(arg)
       }
@@ -565,24 +572,23 @@ extension Driver {
   /// Expand response files in the input arguments and return a new argument list.
   public static func expandResponseFiles(
     _ args: [String],
+    fileSystem: FileSystem,
     diagnosticsEngine: DiagnosticsEngine
   ) throws -> [String] {
     var visitedResponseFiles = Set<AbsolutePath>()
-    return try expandResponseFiles(args, diagnosticsEngine: diagnosticsEngine, visitedResponseFiles: &visitedResponseFiles)
+    return try expandResponseFiles(args, fileSystem: fileSystem, diagnosticsEngine: diagnosticsEngine, visitedResponseFiles: &visitedResponseFiles)
   }
 }
 
 extension Driver {
   /// Determine the driver kind based on the command-line arguments, consuming the arguments
   /// conveying this information.
   public static func determineDriverKind(
-    args: inout ArraySlice<String>,
-    cwd: AbsolutePath? = localFileSystem.currentWorkingDirectory
+    args: inout ArraySlice<String>
   ) throws -> DriverKind {
     // Get the basename of the driver executable.
     let execRelPath = args.removeFirst()
-    let execPath = try cwd.map{ AbsolutePath(execRelPath, relativeTo: $0) } ?? AbsolutePath(validating: execRelPath)
-    var driverName = execPath.basename
+    var driverName = try VirtualPath(path: execRelPath).basenameWithoutExt
 
     // Determine driver kind based on the first argument.
     let driverModeOption = "--driver-mode="
@@ -680,7 +686,7 @@ extension Driver {
         forceResponseFiles: forceResponseFiles,
         recordedInputModificationDates: recordedInputModificationDates
     )
-    try jobExecutor.execute(env: env)
+    try jobExecutor.execute(env: env, fileSystem: fileSystem)
   }
 
   public mutating func createToolExecutionDelegate() -> ToolExecutionDelegate {
@@ -704,7 +710,7 @@ extension Driver {
       try ProcessEnv.setVar(envVar, value: value)
     }
 
-    try job.verifyInputsNotModified(since: self.recordedInputModificationDates)
+    try job.verifyInputsNotModified(since: self.recordedInputModificationDates, fileSystem: fileSystem)
 
     return try exec(path: arguments[0], args: arguments)
   }
@@ -988,6 +994,9 @@ extension Driver {
       case .interpret:
         compilerOutputType = nil
 
+      case .scanDependencies:
+        compilerOutputType = .jsonDependencies
+
       default:
         fatalError("unhandled output mode option \(outputOption)")
       }
@@ -1401,6 +1410,7 @@ extension Driver {
     _ parsedOptions: inout ParsedOptions,
     compilerMode: CompilerMode,
     toolchain: Toolchain,
+    fileSystem: FileSystem,
     diagnosticsEngine: DiagnosticsEngine,
     env: [String: String]
   ) -> String? {
@@ -1436,14 +1446,14 @@ extension Driver {
       // FIXME: TSC should provide a better utility for this.
       if let absPath = try? AbsolutePath(validating: sdkPath) {
         path = absPath
-      } else if let cwd = localFileSystem.currentWorkingDirectory {
+      } else if let cwd = fileSystem.currentWorkingDirectory {
         path = AbsolutePath(sdkPath, relativeTo: cwd)
       } else {
         diagnosticsEngine.emit(.warning_no_such_sdk(sdkPath))
         return sdkPath
       }
 
-      if !localFileSystem.exists(path) {
+      if !fileSystem.exists(path) {
         diagnosticsEngine.emit(.warning_no_such_sdk(sdkPath))
       }
       // .. else check if SDK is too old (we need target triple to diagnose that).
@@ -1549,11 +1559,12 @@ extension Driver {
   static func computeToolchain(
     _ explicitTarget: Triple?,
     diagnosticsEngine: DiagnosticsEngine,
-    env: [String: String]
+    env: [String: String],
+    fileSystem: FileSystem
   ) throws -> (Toolchain, Triple) {
     let toolchainType = try explicitTarget?.toolchainType(diagnosticsEngine) ??
           defaultToolchainType
-    let toolchain = toolchainType.init(env: env)
+    let toolchain = toolchainType.init(env: env, fileSystem: fileSystem)
     return (toolchain, try explicitTarget ?? toolchain.hostTargetTriple())
   }
 }

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -12,6 +12,8 @@
 import TSCBasic
 import Foundation
 
+public typealias FileSystem = TSCBasic.FileSystem
+
 /// Mapping of input file paths to specific output files.
 public struct OutputFileMap: Equatable {
   /// The known mapping from input file to specific output files.
@@ -68,11 +70,12 @@ public struct OutputFileMap: Equatable {
 
   /// Load the output file map at the given path.
   public static func load(
+    fileSystem: FileSystem,
     file: VirtualPath,
     diagnosticEngine: DiagnosticsEngine
   ) throws -> OutputFileMap {
     // Load and decode the file.
-    let contents = try localFileSystem.readFileContents(file)
+    let contents = try fileSystem.readFileContents(file)
     let result = try JSONDecoder().decode(OutputFileMapJSON.self, from: Data(contents.contents))
 
     // Convert the loaded entries into virual output file map.
@@ -84,6 +87,7 @@ public struct OutputFileMap: Equatable {
 
   /// Store the output file map at the given path.
   public func store(
+    fileSystem: FileSystem,
     file: AbsolutePath,
     diagnosticEngine: DiagnosticsEngine
   ) throws {
@@ -98,7 +102,7 @@ public struct OutputFileMap: Equatable {
   #endif
 
     let contents = try encoder.encode(OutputFileMapJSON.fromVirtualOutputFileMap(entries).entries)
-    try localFileSystem.writeFileContents(file, bytes: ByteString(contents))
+    try fileSystem.writeFileContents(file, bytes: ByteString(contents))
   }
 }
 

Sources/SwiftDriver/Execution/JobExecutor.swift

@@ -20,14 +20,18 @@ public struct ArgsResolver {
   /// The map of virtual path to the actual path.
   public var pathMapping: [VirtualPath: AbsolutePath]
 
+  /// The file system used by the resolver.
+  private let fileSystem: FileSystem
+
   /// Path to the directory that will contain the temporary files.
   private let temporaryDirectory: AbsolutePath
 
-  public init() throws {
+  public init(fileSystem: FileSystem = localFileSystem) throws {
     self.pathMapping = [:]
+    self.fileSystem = fileSystem
     self.temporaryDirectory = try withTemporaryDirectory(removeTreeOnDeinit: false) { path in
       // FIXME: TSC removes empty directories even when removeTreeOnDeinit is false. This seems like a bug.
-      try localFileSystem.writeFileContents(path.appending(component: ".keep-directory")) { $0 <<< "" }
+      try fileSystem.writeFileContents(path.appending(component: ".keep-directory")) { $0 <<< "" }
       return path
     }
   }
@@ -75,7 +79,7 @@ public struct ArgsResolver {
              "Platform does not support response files for job: \(job)")
       // Match the integrated driver's behavior, which uses response file names of the form "arguments-[0-9a-zA-Z].resp".
       let responseFilePath = temporaryDirectory.appending(component: "arguments-\(abs(job.hashValue)).resp")
-      try localFileSystem.writeFileContents(responseFilePath) {
+      try fileSystem.writeFileContents(responseFilePath) {
         $0 <<< resolvedArguments[1...].map{ $0.spm_shellEscaped() }.joined(separator: "\n")
       }
       resolvedArguments = [resolvedArguments[0], "@\(responseFilePath.pathString)"]
@@ -123,6 +127,9 @@ public final class JobExecutor {
     /// The environment variables.
     let env: [String: String]
 
+    /// The file system.
+    let fileSystem: FileSystem
+
     /// The job executor delegate.
     let executorDelegate: JobExecutorDelegate
 
@@ -144,6 +151,7 @@ public final class JobExecutor {
     init(
       argsResolver: ArgsResolver,
       env: [String: String],
+      fileSystem: FileSystem,
       producerMap: [VirtualPath: Job],
       executorDelegate: JobExecutorDelegate,
       jobQueue: OperationQueue,
@@ -154,6 +162,7 @@ public final class JobExecutor {
       self.producerMap = producerMap
       self.argsResolver = argsResolver
       self.env = env
+      self.fileSystem = fileSystem
       self.executorDelegate = executorDelegate
       self.jobQueue = jobQueue
       self.processSet = processSet
@@ -202,8 +211,8 @@ public final class JobExecutor {
   }
 
   /// Execute all jobs.
-  public func execute(env: [String: String]) throws {
-    let context = createContext(jobs, env: env)
+  public func execute(env: [String: String], fileSystem: FileSystem) throws {
+    let context = createContext(jobs, env: env, fileSystem: fileSystem)
 
     let delegate = JobExecutorBuildDelegate(context)
     let engine = LLBuildEngine(delegate: delegate)
@@ -217,7 +226,7 @@ public final class JobExecutor {
   }
 
   /// Create the context required during the execution.
-  func createContext(_ jobs: [Job], env: [String: String]) -> Context {
+  func createContext(_ jobs: [Job], env: [String: String], fileSystem: FileSystem) -> Context {
     var producerMap: [VirtualPath: Job] = [:]
     for job in jobs {
       for output in job.outputs {
@@ -233,6 +242,7 @@ public final class JobExecutor {
     return Context(
       argsResolver: argsResolver,
       env: env,
+      fileSystem: fileSystem,
       producerMap: producerMap,
       executorDelegate: executorDelegate,
       jobQueue: jobQueue,
@@ -254,7 +264,7 @@ struct JobExecutorBuildDelegate: LLBuildEngineDelegate {
   func lookupRule(rule: String, key: Key) -> Rule {
     switch rule {
     case ExecuteAllJobsRule.ruleName:
-      return ExecuteAllJobsRule(key)
+      return ExecuteAllJobsRule(key, fileSystem: context.fileSystem)
     case ExecuteJobRule.ruleName:
       return ExecuteJobRule(key, context: context)
     default:
@@ -295,9 +305,9 @@ class ExecuteAllJobsRule: LLBuildRule {
   /// True if any of the inputs had any error.
   private var allInputsSucceeded: Bool = true
 
-  init(_ key: Key) {
+  init(_ key: Key, fileSystem: FileSystem) {
     self.key = RuleKey(key)
-    super.init()
+    super.init(fileSystem: fileSystem)
   }
 
   override func start(_ engine: LLTaskBuildEngine) {
@@ -344,7 +354,7 @@ class ExecuteJobRule: LLBuildRule {
   init(_ key: Key, context: JobExecutor.Context) {
     self.key = RuleKey(key)
     self.context = context
-    super.init()
+    super.init(fileSystem: context.fileSystem)
   }
 
   override func start(_ engine: LLTaskBuildEngine) {
@@ -392,7 +402,7 @@ class ExecuteJobRule: LLBuildRule {
       let arguments: [String] = try resolver.resolveArgumentList(for: job,
                                                                  forceResponseFiles: context.forceResponseFiles)
 
-      try job.verifyInputsNotModified(since: context.recordedInputModificationDates)
+      try job.verifyInputsNotModified(since: context.recordedInputModificationDates, fileSystem: engine.fileSystem)
 
       let process = try context.executorDelegate.launchProcess(
         for: job, arguments: arguments, env: env