Print commands required to explicitly build module dependencies.

Sources/SwiftDriver/CMakeLists.txt

@@ -7,12 +7,16 @@
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_library(SwiftDriver
+  "Dependency Scanning/ModuleDependencyBuildGeneration.swift"
+  "Dependency Scanning/InterModuleDependencyGraph.swift"
+
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift
   Driver/Driver.swift
   Driver/LinkKind.swift
   Driver/OutputFileMap.swift
   Driver/ToolExecutionDelegate.swift
+  Driver/ModuleDependencyScanning.swift
 
   Execution/JobExecutor.swift
   Execution/ParsableOutput.swift

Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift

@@ -0,0 +1,145 @@
+//===--------------- ModuleDependencyGraph.swift --------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+
+
+enum ModuleDependencyId: Hashable {
+  case swift(String)
+  case clang(String)
+
+  var moduleName: String {
+    switch self {
+      case .swift(let name): return name
+      case .clang(let name): return name
+    }
+  }
+}
+
+extension ModuleDependencyId: Codable {
+  enum CodingKeys: CodingKey {
+    case swift
+    case clang
+  }
+
+  init(from decoder: Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    do {
+      let moduleName =  try container.decode(String.self, forKey: .swift)
+      self = .swift(moduleName)
+    } catch {
+      let moduleName =  try container.decode(String.self, forKey: .clang)
+      self = .clang(moduleName)
+    }
+  }
+
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    switch self {
+      case .swift(let moduleName):
+        try container.encode(moduleName, forKey: .swift)
+      case .clang(let moduleName):
+        try container.encode(moduleName, forKey: .clang)
+    }
+  }
+}
+
+/// Details specific to Swift modules.
+struct SwiftModuleDetails: Codable {
+  /// The module interface from which this module was built, if any.
+  var moduleInterfacePath: String?
+
+  /// The bridging header, if any.
+  var bridgingHeaderPath: String?
+
+  /// The source files referenced by the bridging header.
+  var bridgingSourceFiles: [String]? = []
+
+  /// Options to the compile command
+  var commandLine: [String]? = []
+}
+
+/// Details specific to Clang modules.
+struct ClangModuleDetails: Codable {
+  /// The path to the module map used to build this module.
+  var moduleMapPath: String
+
+  /// clang-generated context hash
+  var contextHash: String?
+
+  /// Options to the compile command
+  var commandLine: [String]? = []
+}
+
+struct ModuleInfo: Codable {
+  /// The path for the module.
+  var modulePath: String
+
+  /// The source files used to build this module.
+  var sourceFiles: [String] = []
+
+  /// The set of direct module dependencies of this module.
+  var directDependencies: [ModuleDependencyId] = []
+
+  /// Specific details of a particular kind of module.
+  var details: Details
+
+  /// Specific details of a particular kind of module.
+  enum Details {
+    /// Swift modules may be built from a module interface, and may have
+    /// a bridging header.
+    case swift(SwiftModuleDetails)
+
+    /// Clang modules are built from a module map file.
+    case clang(ClangModuleDetails)
+  }
+}
+
+extension ModuleInfo.Details: Codable {
+  enum CodingKeys: CodingKey {
+    case swift
+    case clang
+  }
+
+  init(from decoder: Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    do {
+      let details = try container.decode(SwiftModuleDetails.self, forKey: .swift)
+      self = .swift(details)
+    } catch {
+      let details = try container.decode(ClangModuleDetails.self, forKey: .clang)
+      self = .clang(details)
+    }
+  }
+
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    switch self {
+      case .swift(let details):
+        try container.encode(details, forKey: .swift)
+      case .clang(let details):
+        try container.encode(details, forKey: .clang)
+    }
+  }
+}
+
+/// Describes the complete set of dependencies for a Swift module, including
+/// all of the Swift and C modules and source files it depends on.
+struct InterModuleDependencyGraph: Codable {
+  /// The name of the main module.
+  var mainModuleName: String
+
+  /// The complete set of modules discovered
+  var modules: [ModuleDependencyId: ModuleInfo] = [:]
+
+  /// Information about the main module.
+  var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }
+}

Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift

@@ -0,0 +1,109 @@
+//===--------------- ModuleDependencyBuildGeneration.swift ----------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import TSCBasic
+import TSCUtility
+import Foundation
+
+extension Driver {
+  /// For the current moduleDependencyGraph, plan the order and generate jobs
+  /// for explicitly building all dependency modules.
+  mutating func planExplicitModuleDependenciesCompile(dependencyGraph: InterModuleDependencyGraph)
+  throws -> [Job] {
+    var jobs: [Job] = []
+    for (id, moduleInfo) in dependencyGraph.modules {
+      // The generation of the main module file will be handled elsewhere in the driver.
+      if (id.moduleName == dependencyGraph.mainModuleName) {
+        continue
+      }
+      switch id {
+        case .swift(let moduleName):
+          let swiftModuleBuildJob = try genSwiftModuleDependencyBuildJob(moduleInfo: moduleInfo,
+                                                                         moduleName: moduleName)
+          jobs.append(swiftModuleBuildJob)
+        case .clang(let moduleName):
+          let clangModuleBuildJob = try genClangModuleDependencyBuildJob(moduleInfo: moduleInfo,
+                                                                         moduleName: moduleName)
+          jobs.append(clangModuleBuildJob)
+
+      }
+    }
+    return jobs
+  }
+
+  /// For a given swift module dependency, generate a build job
+  mutating private func genSwiftModuleDependencyBuildJob(moduleInfo: ModuleInfo,
+                                                         moduleName: String) throws -> Job {
+    // FIXIT: Needs more error handling
+    guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
+      throw Error.malformedModuleDependency(moduleName, "no `details` object")
+    }
+
+    var inputs: [TypedVirtualPath] = []
+    var outputs: [TypedVirtualPath] = [
+      TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule)
+    ]
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+
+    // Build the .swiftinterfaces file using a list of command line options specified in the
+    // `details` field.
+    guard let moduleInterfacePath = swiftModuleDetails.moduleInterfacePath else {
+      throw Error.malformedModuleDependency(moduleName, "no `moduleInterfacePath` object")
+    }
+    inputs.append(TypedVirtualPath(file: try VirtualPath(path: moduleInterfacePath),
+                                   type: .swiftInterface))
+    try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
+    swiftModuleDetails.commandLine?.forEach { commandLine.appendFlag($0) }
+
+    return Job(
+      kind: .emitModule,
+      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      commandLine: commandLine,
+      inputs: inputs,
+      outputs: outputs
+    )
+  }
+
+  /// For a given clang module dependency, generate a build job
+  mutating private func genClangModuleDependencyBuildJob(moduleInfo: ModuleInfo,
+                                                         moduleName: String) throws -> Job {
+    // For clang modules, the Fast Dependency Scanner emits a list of source
+    // files (with a .modulemap among them), and a list of compile command
+    // options.
+    // FIXIT: Needs more error handling
+    guard case .clang(let clangModuleDetails) = moduleInfo.details else {
+      throw Error.malformedModuleDependency(moduleName, "no `details` object")
+    }
+    var inputs: [TypedVirtualPath] = []
+    var outputs: [TypedVirtualPath] = [
+      TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .pcm)
+    ]
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+    commandLine.appendFlags("-emit-pcm", "-module-name", moduleName)
+
+    // The only required input is the .modulemap for this module.
+    commandLine.append(Job.ArgTemplate.path(try VirtualPath(path: clangModuleDetails.moduleMapPath)))
+    inputs.append(TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath),
+                                   type: .clangModuleMap))
+    try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
+    clangModuleDetails.commandLine?.forEach { commandLine.appendFlags("-Xcc", $0) }
+
+    return Job(
+      kind: .generatePCM,
+      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      commandLine: commandLine,
+      inputs: inputs,
+      outputs: outputs
+    )
+  }
+}

Sources/SwiftDriver/Driver/Driver.swift

@@ -50,6 +50,7 @@ public struct Driver {
     case subcommandPassedToDriver
     case integratedReplRemoved
     case conflictingOptions(Option, Option)
+    case malformedModuleDependency(String, String)
 
     public var description: String {
       switch self {
@@ -68,6 +69,8 @@ public struct Driver {
         return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead."
       case .conflictingOptions(let one, let two):
         return "conflicting options '\(one.spelling)' and '\(two.spelling)'"
+      case .malformedModuleDependency(let moduleName, let errorDescription):
+        return "Malformed Module Dependency: \(moduleName), \(errorDescription)"
       }
     }
   }

Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift

@@ -0,0 +1,50 @@
+//===--------------- ModuleDependencyScanning.swift -----------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+import TSCBasic
+
+extension Driver {
+  /// Precompute the dependencies for a given Swift compilation, producing a
+  /// complete dependency graph including all Swift and C module files and
+  /// source files.
+  // TODO: Instead of directly invoking the frontend,
+  // treat this as any other `Job`.
+  mutating func computeModuleDependencyGraph() throws
+      -> InterModuleDependencyGraph? {
+    // Grab the swift compiler
+    let resolver = try ArgsResolver()
+    let compilerPath = VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler))
+    let tool = try resolver.resolve(.path(compilerPath))
+
+    // Aggregate the fast dependency scanner arguments
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+    commandLine.appendFlag("-scan-dependencies")
+    if parsedOptions.hasArgument(.parseStdlib) {
+       commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule)
+    }
+    try addCommonFrontendOptions(commandLine: &commandLine)
+    // FIXME: MSVC runtime flags
+
+    // Pass on the input files
+    commandLine.append(contentsOf: inputFiles.map { .path($0.file)})
+
+    // Execute dependency scanner and decode its output
+    let arguments = [tool] + (try commandLine.map { try resolver.resolve($0) })
+    let scanProcess = try Process.launchProcess(arguments: arguments, env: env)
+    let result = try scanProcess.waitUntilExit()
+    guard let outputData = try? Data(result.utf8Output().utf8) else {
+      return nil
+    }
+    return try JSONDecoder().decode(InterModuleDependencyGraph.self, from: outputData)
+  }
+}

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -69,7 +69,7 @@ extension Driver {
          .swiftDocumentation, .swiftInterface,
          .swiftSourceInfoFile, .raw_sib, .llvmBitcode, .diagnostics,
          .objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace,
-         .indexData, .optimizationRecord, .pcm, .pch, nil:
+         .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, nil:
       return false
     }
   }
@@ -253,7 +253,7 @@ extension FileType {
 
     case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
-         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile:
+         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap:
       fatalError("Output type can never be a primary output")
     }
   }

Sources/SwiftDriver/Jobs/Planning.swift

@@ -34,6 +34,23 @@ extension Driver {
   private mutating func planStandardCompile() throws -> [Job] {
     var jobs = [Job]()
 
+    // If we've been asked to prebuild module dependencies, prescan the source
+    // files to produce a module dependency graph and turn it into a set
+    // of jobs required to build all dependencies.
+    // For the time being, just prints the jobs' compile commands.
+    if parsedOptions.contains(.driverPrintModuleDependenciesJobs) {
+      let moduleDependencyGraph = try computeModuleDependencyGraph()
+      let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
+      if let dependencyGraph = moduleDependencyGraph {
+        let modulePrebuildJobs =
+              try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph)
+        for job in modulePrebuildJobs {
+          try Self.printJob(job, resolver: try ArgsResolver(),
+                            forceResponseFiles: forceResponseFiles)
+        }
+      }
+    }
+
     // Keep track of the various outputs we care about from the jobs we build.
     var linkerInputs: [TypedVirtualPath] = []
     var moduleInputs: [TypedVirtualPath] = []