[5.5][Explicit Module Builds] Cherry-pick explicit module build changes

Sources/CSwiftScan/include/swiftscan_header.h

@@ -200,17 +200,20 @@ typedef struct {
   //=== Scanner Functions ---------------------------------------------------===//  
   swiftscan_scanner_t (*swiftscan_scanner_create)(void);
   void (*swiftscan_scanner_dispose)(swiftscan_scanner_t);
-
   swiftscan_dependency_graph_t
   (*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
-
   swiftscan_batch_scan_result_t *
   (*swiftscan_batch_scan_result_create)(swiftscan_scanner_t,
                                         swiftscan_batch_scan_input_t *,
                                         swiftscan_scan_invocation_t);
-
   swiftscan_import_set_t
   (*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
+
+  //=== Scanner Cache Functions ---------------------------------------------===//
+  void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
+  bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
+  void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
+
 } swiftscan_functions_t;
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H

Sources/SwiftDriver/CMakeLists.txt

@@ -10,7 +10,6 @@ add_library(SwiftDriver
   "ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift"
   "ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift"
   "ExplicitModuleBuilds/ModuleDependencyScanning.swift"
-  "ExplicitModuleBuilds/PlaceholderDependencyResolution.swift"
   "ExplicitModuleBuilds/SerializableModuleArtifacts.swift"
   "ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift"
   "ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift"

Sources/SwiftDriver/Driver/Driver.swift

@@ -23,6 +23,7 @@ public struct Driver {
     case invalidArgumentValue(String, String)
     case relativeFrontendPath(String)
     case subcommandPassedToDriver
+    case externalTargetDetailsAPIError
     case integratedReplRemoved
     case cannotSpecify_OForMultipleOutputs
     case conflictingOptions(Option, Option)
@@ -38,6 +39,7 @@ public struct Driver {
     case malformedModuleDependency(String, String)
     case missingPCMArguments(String)
     case missingModuleDependency(String)
+    case missingContextHashOnSwiftDependency(String)
     case dependencyScanningFailure(Int, String)
     case missingExternalDependency(String)
 
@@ -56,6 +58,8 @@ public struct Driver {
         return "relative frontend path: \(path)"
       case .subcommandPassedToDriver:
         return "subcommand passed to driver"
+      case .externalTargetDetailsAPIError:
+        return "Cannot specify both: externalTargetModulePathMap and externalTargetModuleDetailsMap"
       case .integratedReplRemoved:
         return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead."
       case .cannotSpecify_OForMultipleOutputs:
@@ -91,6 +95,8 @@ public struct Driver {
         return "Missing extraPcmArgs to build Clang module: \(moduleName)"
       case .missingModuleDependency(let moduleName):
         return "Missing Module Dependency Info: \(moduleName)"
+      case .missingContextHashOnSwiftDependency(let moduleName):
+        return "Missing Context Hash for Swift dependency: \(moduleName)"
       case .dependencyScanningFailure(let code, let error):
         return "Module Dependency Scanner returned with non-zero exit status: \(code), \(error)"
       case .unableToLoadOutputFileMap(let path):
@@ -302,12 +308,9 @@ public struct Driver {
   /// is shared across many targets; otherwise, a new instance is created by the driver itself.
   @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle
 
-  // TODO: Once the clients have transitioned to using the InterModuleDependencyOracle API,
-  // this must convey information about the externally-prebuilt targets only
-  /// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit
-  /// build of the current module. Consists of a map of externally-built targets, and a map of all previously
-  /// discovered/scanned modules and their infos.
-  @_spi(Testing) public var externalBuildArtifacts: ExternalBuildArtifacts? = nil
+  /// A dictionary of external targets that are a part of the same build, mapping to filesystem paths
+  /// of their module files
+  @_spi(Testing) public var externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap? = nil
 
   /// A collection of all the flags the selected toolchain's `swift-frontend` supports
   public let supportedFrontendFlags: Set<String>
@@ -384,9 +387,17 @@ public struct Driver {
   ///   expand response files, etc. By default this is the local filesystem.
   /// - Parameter executor: Used by the driver to execute jobs. The default argument
   ///   is present to streamline testing, it shouldn't be used in production.
-  /// - Parameter externalBuildArtifacts: All external artifacts a build system may pass in as input to the explicit
-  ///   build of the current module. Consists of a map of externally-built targets, and a map of all previously
-  ///   discovered/scanned modules.
+  /// - Parameter integratedDriver: Used to distinguish whether the driver is being used as
+  ///   an executable or as a library.
+  /// - Parameter compilerExecutableDir: Directory that contains the compiler executable to be used.
+  ///   Used when in `integratedDriver` mode as a substitute for the driver knowing its executable path.
+  /// - Parameter externalTargetModulePathMap: DEPRECATED: A dictionary of external targets
+  ///   that are a part of the same build, mapping to filesystem paths of their module files.
+  /// - Parameter externalTargetModuleDetailsMap: A dictionary of external targets that are a part of
+  ///   the same build, mapping to a details value which includes a filesystem path of their
+  ///   `.swiftmodule` and a flag indicating whether the external target is a framework.
+  /// - Parameter interModuleDependencyOracle: An oracle for querying inter-module dependencies,
+  ///   shared across different module builds by a build system.
   public init(
     args: [String],
     env: [String: String] = ProcessEnv.vars,
@@ -395,10 +406,9 @@ public struct Driver {
     executor: DriverExecutor,
     integratedDriver: Bool = true,
     compilerExecutableDir: AbsolutePath? = nil,
-    // FIXME: Duplication with externalBuildArtifacts and externalTargetModulePathMap
-    // is a temporary backwards-compatibility shim to help transition SwiftPM to the new API
-    externalBuildArtifacts: ExternalBuildArtifacts? = nil,
+    // Deprecated in favour of the below `externalTargetModuleDetailsMap`
     externalTargetModulePathMap: ExternalTargetModulePathMap? = nil,
+    externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap? = nil,
     interModuleDependencyOracle: InterModuleDependencyOracle? = nil
   ) throws {
     self.env = env
@@ -408,10 +418,15 @@ public struct Driver {
     self.diagnosticEngine = diagnosticsEngine
     self.executor = executor
 
-    if let externalArtifacts = externalBuildArtifacts {
-      self.externalBuildArtifacts = externalArtifacts
-    } else if let externalTargetPaths = externalTargetModulePathMap {
-      self.externalBuildArtifacts = (externalTargetPaths, [:])
+    if externalTargetModulePathMap != nil && externalTargetModuleDetailsMap != nil {
+      throw Error.externalTargetDetailsAPIError
+    }
+    if let externalTargetPaths = externalTargetModulePathMap {
+      self.externalTargetModuleDetailsMap = externalTargetPaths.mapValues {
+        ExternalTargetModuleDetails(path: $0, isFramework: false)
+      }
+    } else if let externalTargetDetails = externalTargetModuleDetailsMap {
+      self.externalTargetModuleDetailsMap = externalTargetDetails
     }
 
     if case .subcommand = try Self.invocationRunMode(forArgs: args).mode {
@@ -504,14 +519,6 @@ public struct Driver {
       self.interModuleDependencyOracle = dependencyOracle
     } else {
       self.interModuleDependencyOracle = InterModuleDependencyOracle()
-
-      // This is a shim for backwards-compatibility with ModuleInfoMap-based API
-      // used by SwiftPM
-      if let externalArtifacts = externalBuildArtifacts {
-        if !externalArtifacts.1.isEmpty {
-          try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
-        }
-      }
     }
 
     self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -14,13 +14,21 @@ import TSCUtility
 import Foundation
 
 /// A map from a module identifier to a path to its .swiftmodule file.
+/// Deprecated in favour of the below `ExternalTargetModuleDetails`
 public typealias ExternalTargetModulePathMap = [ModuleDependencyId: AbsolutePath]
 
-// FIXME: ExternalBuildArtifacts is a temporary backwards-compatibility shim
-// to help transition SwiftPM to the new API.
-/// A tuple all external artifacts a build system may pass in as input to the explicit build of the current module
-/// Consists of a map of externally-built targets, and a map of all previously discovered/scanned modules.
-public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleInfoMap)
+/// Details about an external target, including the path to its .swiftmodule file
+/// and whether it is a framework.
+public struct ExternalTargetModuleDetails {
+  @_spi(Testing) public init(path: AbsolutePath, isFramework: Bool) {
+    self.path = path
+    self.isFramework = isFramework
+  }
+  let path: AbsolutePath
+  let isFramework: Bool
+}
+
+public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalTargetModuleDetails]
 
 /// In Explicit Module Build mode, this planner is responsible for generating and providing
 /// build jobs for all module dependencies and providing compile command options
@@ -246,7 +254,7 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
                                                           commandLine: inout [Job.ArgTemplate]) throws {
     // Prohibit the frontend from implicitly building textual modules into binary modules.
     commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc",
-                            "-fno-implicit-modules")
+                            "-fno-implicit-modules", "-Xcc", "-Xclang", "-Xcc", "-fno-implicit-module-maps")
     var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
     var clangDependencyArtifacts: [ClangModuleArtifactInfo] = []
     try addModuleDependencies(moduleId: moduleId, pcmArgs: pcmArgs,
@@ -323,16 +331,17 @@ public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleIn
                                     modulePath: TextualVirtualPath(path: clangModulePath),
                                     moduleMapPath: dependencyClangModuleDetails.moduleMapPath))
         case .swiftPrebuiltExternal:
-          let compiledModulePath = try dependencyGraph
-                                         .swiftPrebuiltDetails(of: dependencyId)
-                                         .compiledModulePath
+          let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId)
+          let compiledModulePath = prebuiltModuleDetails.compiledModulePath
+          let isFramework = prebuiltModuleDetails.isFramework
           let swiftModulePath: TypedVirtualPath =
             .init(file: compiledModulePath.path, type: .swiftModule)
           // Accumulate the requried information about this dependency
           // TODO: add .swiftdoc and .swiftsourceinfo for this module.
           swiftDependencyArtifacts.append(
             SwiftModuleArtifactInfo(name: dependencyId.moduleName,
-                                    modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle)))
+                                    modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
+                                    isFramework: isFramework))
         case .swiftPlaceholder:
           fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")
       }
@@ -444,7 +453,7 @@ extension ExplicitDependencyBuildPlanner {
 
 /// Encapsulates some of the common queries of the ExplicitDependencyBuildPlanner with error-checking
 /// on the dependency graph's structure.
-internal extension InterModuleDependencyGraph {
+@_spi(Testing) public extension InterModuleDependencyGraph {
   func moduleInfo(of moduleId: ModuleDependencyId) throws -> ModuleInfo {
     guard let moduleInfo = modules[moduleId] else {
       throw Driver.Error.missingModuleDependency(moduleId.moduleName)

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -15,11 +15,10 @@ import TSCBasic
   /// For targets that are built alongside the driver's current module, the scanning action will report them as
   /// textual targets to be built from source. Because we can rely on these targets to have been built prior
   /// to the driver's current target, we resolve such external targets as prebuilt binary modules, in the graph.
-  mutating func resolveExternalDependencies(for externalBuildArtifacts: ExternalBuildArtifacts)
+  mutating func resolveExternalDependencies(for externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap)
   throws {
-    let externalTargetModulePathMap = externalBuildArtifacts.0
-
-     for (externalModuleId, externalModulePath) in externalTargetModulePathMap {
+    for (externalModuleId, externalModuleDetails) in externalTargetModuleDetailsMap {
+      let externalModulePath = externalModuleDetails.path
       // Replace the occurence of a Swift module to-be-built from source-file
       // to an info that describes a pre-built binary module.
       let swiftModuleId: ModuleDependencyId = .swift(externalModuleId.moduleName)
@@ -30,57 +29,22 @@ import TSCBasic
         // a dependency on a target that is not actually used.
         continue
       }
-
-       let newModuleId: ModuleDependencyId = .swiftPrebuiltExternal(externalModuleId.moduleName)
+      
+      let newModuleId: ModuleDependencyId = .swiftPrebuiltExternal(externalModuleId.moduleName)
       let newExternalModuleDetails =
-        try SwiftPrebuiltExternalModuleDetails(compiledModulePath:
-                                                TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()))
+      try SwiftPrebuiltExternalModuleDetails(compiledModulePath:
+                                              TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
+                                             isFramework: externalModuleDetails.isFramework)
       let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()),
                                sourceFiles: [],
                                directDependencies: currentInfo.directDependencies,
                                details: .swiftPrebuiltExternal(newExternalModuleDetails))
-
-       Self.replaceModule(originalId: swiftModuleId, replacementId: newModuleId,
+      Self.replaceModule(originalId: swiftModuleId, replacementId: newModuleId,
                          replacementInfo: newInfo, in: &modules)
     }
   }
 }
 
-@_spi(Testing) public extension InterModuleDependencyOracle {
-  /// An API to allow clients to accumulate InterModuleDependencyGraphs across mutiple main externalModules/targets
-  /// into a single collection of discovered externalModules.
-  func mergeModules(from dependencyGraph: InterModuleDependencyGraph) throws {
-    try queue.sync {
-      for (moduleId, moduleInfo) in dependencyGraph.modules {
-        try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &externalModules)
-      }
-    }
-  }
-
-  // This is a backwards-compatibility shim to handle existing ModuleInfoMap-based API
-  // used by SwiftPM
-  func mergeModules(from moduleInfoMap: ModuleInfoMap) throws {
-    try queue.sync {
-      for (moduleId, moduleInfo) in moduleInfoMap {
-        try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &externalModules)
-      }
-    }
-  }
-}
-
-public extension InterModuleDependencyGraph {
-  // This is a shim for backwards-compatibility with existing API used by SwiftPM.
-  // TODO: After SwiftPM switches to using the oracle, this should be deleted.
-  static func mergeModules(
-    from dependencyGraph: InterModuleDependencyGraph,
-    into discoveredModules: inout ModuleInfoMap
-  ) throws {
-    for (moduleId, moduleInfo) in dependencyGraph.modules {
-      try mergeModule(moduleId, moduleInfo, into: &discoveredModules)
-    }
-  }
-}
-
 extension InterModuleDependencyGraph {
   /// Compute a set of modules that are "reachable" (form direct or transitive dependency)
   /// from each module in the graph.

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -97,6 +97,11 @@ public struct SwiftModuleDetails: Codable {
   /// Options to the compile command
   public var commandLine: [String]? = []
 
+  /// The context hash for this module that encodes the producing interface's path,
+  /// target triple, etc. This field is optional because it is absent for the ModuleInfo
+  /// corresponding to the main module being built.
+  public var contextHash: String?
+
   /// To build a PCM to be used by this Swift module, we need to append these
   /// arguments to the generic PCM build arguments reported from the dependency
   /// graph.
@@ -127,12 +132,17 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable {
   /// The path to the .swiftSourceInfo file.
   public var moduleSourceInfoPath: TextualVirtualPath?
 
+  /// A flag to indicate whether or not this module is a framework.
+  public var isFramework: Bool
+
   public init(compiledModulePath: TextualVirtualPath,
               moduleDocPath: TextualVirtualPath? = nil,
-              moduleSourceInfoPath: TextualVirtualPath? = nil) throws {
+              moduleSourceInfoPath: TextualVirtualPath? = nil,
+              isFramework: Bool = false) throws {
     self.compiledModulePath = compiledModulePath
     self.moduleDocPath = moduleDocPath
     self.moduleSourceInfoPath = moduleSourceInfoPath
+    self.isFramework = isFramework
   }
 }