[WIP] Print commands required to explicitly build module dependencies.

This PR resurrects #47 and builds on top of it.

Adds new flag -driver-print-module-dependencies-jobs to instruct the driver to scan for dependencies and generate build jobs for each dependent Swift or C module.

At present, this WIP does the following:
- Invoke the Swift frontend Fast Dependency Scanner to prescan for dependencies.
- Construct a Module Dependency Graph from the dependency scanner's JSON output.
- Compute the order in which dependent modules should be built.
- Generate and print build jbos for each dependent module.

Next Steps:
Actually have the driver schedule and execute the dependent module build jobs ahead of time, eliminating the need to re-build those modules when it tries to import them.

Author: artemcm

Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift

@@ -0,0 +1,179 @@
+//===--------------- ModuleDependencyBuildGeneration.swift ----------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import TSCBasic
+import TSCUtility
+import Foundation
+
+extension ModuleDependencyGraph {
+  /// A topologically-ordered list of dependent modules to be built.
+  nonmutating func topologicalBuildOrder() -> [ModuleDependencyId] {
+    return topologicalBuildBatchOrder().flatMap { $0 }
+  }
+
+  /// Use Kahn's algorithm to determine the order in which the dependent
+  /// modules should be built.
+  nonmutating func topologicalBuildBatchOrder() -> [[ModuleDependencyId]] {
+    var orderedModuleBatchList : [[ModuleDependencyId]] = []
+    // A worklist of modules and their dependencies
+    var moduleDependenciesMap = modules.mapValues { value in Set(value.directDependencies) }
+    // Find modules without any dependencies and remove them from the worklist
+    var currentBatch = Array(moduleDependenciesMap.filter { $0.value.isEmpty }.keys)
+    var nextBatch : [ModuleDependencyId] = []
+    currentBatch.forEach { moduleDependenciesMap.removeValue(forKey: $0) }
+    orderedModuleBatchList.append(currentBatch)
+
+    // Proceed until there are no more independent modules to be built
+    while (!currentBatch.isEmpty) {
+      let readyModule = currentBatch.removeFirst()
+      // Remove dependency on `readyModule` from all other modules
+      for key in moduleDependenciesMap.keys {
+        var deps = moduleDependenciesMap[key]
+        deps?.remove(readyModule)
+        moduleDependenciesMap[key] = deps
+      }
+
+      // Add newly 'freed' modules to the the next batch
+      for newlyIndependentModule in moduleDependenciesMap.filter({ $0.value.isEmpty }) {
+        nextBatch.append(newlyIndependentModule.key)
+        moduleDependenciesMap.removeValue(forKey: newlyIndependentModule.key)
+      }
+
+      // This batch is done, write it to the output and move on to the next one
+      if (currentBatch.isEmpty && !nextBatch.isEmpty) {
+        currentBatch = nextBatch
+        nextBatch.removeAll()
+        orderedModuleBatchList.append(currentBatch)
+      }
+    }
+
+    // If there are still modules left to build, the module dependencies
+    // must be malformed (contain cycles).
+    assert(moduleDependenciesMap.isEmpty, "Cycles in module dependency graph detected.")
+    return orderedModuleBatchList
+  }
+}
+
+extension Driver {
+  /// For the current moduleDependencyGraph, plan the order and generate jobs
+  /// for explicitly building all dependency modules.
+  mutating func planExplicitModuleDependenciesCompile() throws -> [Job] {
+    var jobs: [Job] = []
+    guard let dependencyGraph = moduleDependencyGraph else { return jobs }
+    let moduleBuildOrder = dependencyGraph.topologicalBuildOrder()
+    for id in moduleBuildOrder {
+      switch id {
+        case .swift(let moduleName):
+          let swiftModuleBuildJob = try genSwiftModuleDependencyBuildJob(moduleId: id,
+                                                                         moduleName: moduleName)
+          jobs.append(swiftModuleBuildJob)
+        case .clang(let moduleName):
+          let clangModuleBuildJob = try genClangModuleDependencyBuildJob(moduleId: id,
+                                                                         moduleName: moduleName)
+          jobs.append(clangModuleBuildJob)
+
+      }
+    }
+    return jobs
+  }
+
+  /// For a given swift module dependency, generate a build job
+  mutating private func genSwiftModuleDependencyBuildJob(moduleId: ModuleDependencyId,
+                                                         moduleName: String) throws -> Job {
+    // FIXIT: Needs more error handling
+    guard let moduleInfo = moduleDependencyGraph!.modules[moduleId] else {
+      fatalError("Constructing build job for unknown module: \(moduleName)")
+    }
+    guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
+      fatalError("Malformed Swift Module Dependency: \(moduleName), no details object.")
+    }
+
+    var inputs: [TypedVirtualPath] = []
+    var outputs: [TypedVirtualPath] = [
+      TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule)
+    ]
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+
+    // If source files are given, compile them into a module re-using the
+    // current compile's options
+    if (!moduleInfo.sourceFiles.isEmpty) {
+      commandLine.appendFlags("-emit-module", "-module-name", moduleName)
+      commandLine.appendFlag("-emit-module-path=" + moduleInfo.modulePath)
+      commandLine.append(contentsOf: try moduleInfo.sourceFiles.map {
+                          Job.ArgTemplate.path(try VirtualPath(path: $0))})
+      inputs.append(contentsOf: try moduleInfo.sourceFiles.map {
+                      TypedVirtualPath(file: try VirtualPath(path: $0), type: .swift) })
+      try addCommonFrontendOptions(commandLine: &commandLine)
+      try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
+    } else {
+      // If no source files are provided, there must be a .swiftinterfaces file
+      // and a list of command line options specified in the `details` field.
+      guard let moduleInterfacePath = swiftModuleDetails.moduleInterfacePath else {
+        fatalError("Malformed Swift Module Dependency: \(moduleName), no moduleInterfacePath object.")
+      }
+      inputs.append(TypedVirtualPath(file: try VirtualPath(path: moduleInterfacePath),
+                                     type: .swiftInterface))
+      try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
+      swiftModuleDetails.commandLine?.forEach { commandLine.appendFlag($0) }
+    }
+
+    return Job(
+      kind: .emitModule,
+      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      commandLine: commandLine,
+      inputs: inputs,
+      outputs: outputs
+    )
+  }
+
+  /// For a given clang module dependency, generate a build job
+  mutating private func genClangModuleDependencyBuildJob(moduleId: ModuleDependencyId,
+                                                         moduleName: String) throws -> Job {
+    // For clang modules, the Fast Dependency Scanner emits a list of source
+    // files (with a .modulemap among them), and a list of compile command
+    // options.
+    // FIXIT: Needs more error handling
+    guard var moduleInfo = moduleDependencyGraph!.modules[moduleId] else {
+      fatalError("Constructing build job for unknown module: \(moduleName)")
+    }
+    guard case .clang(let clangModuleDetails) = moduleInfo.details else {
+      fatalError("Malformed Clang Module Dependency: \(moduleName), no details object.")
+    }
+    var inputs: [TypedVirtualPath] = []
+    var outputs: [TypedVirtualPath] = [
+      TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .pcm)
+    ]
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+    commandLine.appendFlags("-emit-pcm", "-module-name", moduleName)
+
+    // The .modulemap is currently duplicated in moduleMapPath and in sourceFiles.
+    // It also must be the very first input argument to the compile command.
+    if let index = moduleInfo.sourceFiles.firstIndex(of: clangModuleDetails.moduleMapPath) {
+      moduleInfo.sourceFiles.swapAt(0, index)
+    }
+    commandLine.append(contentsOf: try moduleInfo.sourceFiles.map {
+                        Job.ArgTemplate.path(try VirtualPath(path: $0))})
+    inputs.append(contentsOf: try moduleInfo.sourceFiles.map {
+                    TypedVirtualPath(file: try VirtualPath(path: $0), type: .swift) })
+    try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
+    clangModuleDetails.commandLine?.forEach { commandLine.appendFlags("-Xcc", $0) }
+
+    return Job(
+      kind: .generatePCM,
+      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      commandLine: commandLine,
+      inputs: inputs,
+      outputs: outputs
+    )
+  }
+}

Sources/SwiftDriver/Dependency Scanning/ModuleDependencyGraph.swift

@@ -0,0 +1,145 @@
+//===--------------- ModuleDependencyGraph.swift --------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+
+
+enum ModuleDependencyId: Hashable {
+  case swift(String)
+  case clang(String)
+
+  var moduleName: String {
+    switch self {
+      case .swift(let name): return name
+      case .clang(let name): return name
+    }
+  }
+}
+
+extension ModuleDependencyId: Codable {
+  enum CodingKeys: CodingKey {
+    case swift
+    case clang
+  }
+
+  init(from decoder: Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    do {
+      let moduleName =  try container.decode(String.self, forKey: .swift)
+      self = .swift(moduleName)
+    } catch {
+      let moduleName =  try container.decode(String.self, forKey: .clang)
+      self = .clang(moduleName)
+    }
+  }
+
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    switch self {
+      case .swift(let moduleName):
+        try container.encode(moduleName, forKey: .swift)
+      case .clang(let moduleName):
+        try container.encode(moduleName, forKey: .clang)
+    }
+  }
+}
+
+/// Details specific to Swift modules.
+struct SwiftModuleDetails: Codable {
+  /// The module interface from which this module was built, if any.
+  var moduleInterfacePath: String?
+
+  /// The bridging header, if any.
+  var bridgingHeaderPath: String?
+
+  /// The source files referenced by the bridging header.
+  var bridgingSourceFiles: [String]? = []
+
+  /// Options to the compile command
+  var commandLine: [String]? = []
+}
+
+/// Details specific to Clang modules.
+struct ClangModuleDetails: Codable {
+  /// The path to the module map used to build this module.
+  var moduleMapPath: String
+
+  /// clang-generated context hash
+  var contextHash: String?
+
+  /// Options to the compile command
+  var commandLine: [String]? = []
+}
+
+struct ModuleInfo: Codable {
+  /// The path for the module.
+  var modulePath: String
+
+  /// The source files used to build this module.
+  var sourceFiles: [String] = []
+
+  /// The set of direct module dependencies of this module.
+  var directDependencies: [ModuleDependencyId] = []
+
+  /// Specific details of a particular kind of module.
+  var details: Details
+
+  /// Specific details of a particular kind of module.
+  enum Details {
+    /// Swift modules may be built from a module interface, and may have
+    /// a bridging header.
+    case swift(SwiftModuleDetails)
+
+    /// Clang modules are built from a module map file.
+    case clang(ClangModuleDetails)
+  }
+}
+
+extension ModuleInfo.Details: Codable {
+  enum CodingKeys: CodingKey {
+    case swift
+    case clang
+  }
+
+  init(from decoder: Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    do {
+      let details = try container.decode(SwiftModuleDetails.self, forKey: .swift)
+      self = .swift(details)
+    } catch {
+      let details = try container.decode(ClangModuleDetails.self, forKey: .clang)
+      self = .clang(details)
+    }
+  }
+
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    switch self {
+      case .swift(let details):
+        try container.encode(details, forKey: .swift)
+      case .clang(let details):
+        try container.encode(details, forKey: .clang)
+    }
+  }
+}
+
+/// Describes the complete set of dependencies for a Swift module, including
+/// all of the Swift and C modules and source files it depends on.
+struct ModuleDependencyGraph: Codable {
+  /// The name of the main module.
+  var mainModuleName: String
+
+  /// The complete set of modules discovered
+  var modules: [ModuleDependencyId: ModuleInfo] = [:]
+
+  /// Information about the main module.
+  var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }
+}

Sources/SwiftDriver/Driver/Driver.swift

@@ -194,6 +194,11 @@ public struct Driver {
   /// This will force the driver to first emit the module and then run compile jobs.
   public var forceEmitModuleInSingleInvocation: Bool = false
 
+  /// The module dependency graph, which is populated during the planning phase
+  /// only when all modules will be prebuilt and treated as explicit by the
+  /// various compilation jobs.
+  var moduleDependencyGraph: ModuleDependencyGraph? = nil
+
   /// Handler for emitting diagnostics to stderr.
   public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in
     let stream = stderrStream

Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift

@@ -0,0 +1,50 @@
+//===--------------- ModuleDependencyScanning.swift -----------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+import TSCBasic
+
+extension Driver {
+  /// Precompute the dependencies for a given Swift compilation, producing a
+  /// complete dependency graph including all Swift and C module files and
+  /// source files.
+  // TODO: Instead of directly invoking the frontend,
+  // treat this as any other `Job`.
+  mutating func computeModuleDependencyGraph() throws
+      -> ModuleDependencyGraph? {
+    // Grab the swift compiler
+    let resolver = try ArgsResolver()
+    let compilerPath = VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler))
+    let tool = try resolver.resolve(.path(compilerPath))
+
+    // Aggregate the fast dependency scanner arguments
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+    commandLine.appendFlag("-scan-dependencies")
+    if parsedOptions.hasArgument(.parseStdlib) {
+       commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule)
+    }
+    try addCommonFrontendOptions(commandLine: &commandLine)
+    // FIXME: MSVC runtime flags
+
+    // Pass on the input files
+    commandLine.append(contentsOf: inputFiles.map { .path($0.file)})
+
+    // Execute dependency scanner and decode its output
+    let arguments = [tool] + (try commandLine.map { try resolver.resolve($0) })
+    let scanProcess = try Process.launchProcess(arguments: arguments, env: env)
+    let result = try scanProcess.waitUntilExit()
+    guard let outputData = try? Data(result.utf8Output().utf8) else {
+      return nil
+    }
+    return try JSONDecoder().decode(ModuleDependencyGraph.self, from: outputData)
+  }
+}