Merge pull request #1386 from artemcm/ExplicitModule59CherrySeason

[5.9 🍒] Miscellaneous Dependency Scanner and Explicit Modules Improvements

Author: artemcm

Sources/CSwiftScan/include/swiftscan_header.h

@@ -122,6 +122,8 @@ typedef struct {
   (*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t);
   bool
   (*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t);
+  swiftscan_string_set_t *
+  (*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t);
 
   //=== Swift Binary Module Details query APIs ------------------------------===//
   swiftscan_string_ref_t

Sources/SwiftDriver/Driver/Driver.swift

@@ -1332,6 +1332,37 @@ extension Driver {
       return
     }
 
+    // If we're only supposed to explain a dependency on a given module, do so now.
+    if let explainModuleName = parsedOptions.getLastArgument(.explainModuleDependency) {
+      guard let dependencyPlanner = explicitDependencyBuildPlanner else {
+        fatalError("Cannot explain dependency without Explicit Build Planner")
+      }
+      guard let dependencyPaths = try dependencyPlanner.explainDependency(explainModuleName.asSingle) else {
+        diagnosticEngine.emit(.remark("No such module dependency found: '\(explainModuleName.asSingle)'"))
+        return
+      }
+      diagnosticEngine.emit(.remark("Module '\(moduleOutputInfo.name)' depends on '\(explainModuleName.asSingle)'"))
+      for path in dependencyPaths {
+        var pathString:String = ""
+        for (index, moduleId) in path.enumerated() {
+          switch moduleId {
+          case .swift(let moduleName):
+            pathString = pathString + "[" + moduleName + "]"
+          case .swiftPrebuiltExternal(let moduleName):
+            pathString = pathString + "[" + moduleName + "]"
+          case .clang(let moduleName):
+            pathString = pathString + "[" + moduleName + "](ObjC)"
+          case .swiftPlaceholder(_):
+            fatalError("Unexpected unresolved Placeholder module")
+          }
+          if index < path.count - 1 {
+            pathString = pathString + " -> "
+          }
+        }
+        diagnosticEngine.emit(.note(pathString))
+      }
+    }
+
     if parsedOptions.contains(.driverPrintOutputFileMap) {
       if let outputFileMap = self.outputFileMap {
         stderrStream <<< outputFileMap.description

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -124,7 +124,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
     }
     for moduleId in swiftDependencies {
       let moduleInfo = try dependencyGraph.moduleInfo(of: moduleId)
-      let pcmArgs = try dependencyGraph.swiftModulePCMArgs(of: moduleId)
       var inputs: [TypedVirtualPath] = []
       let outputs: [TypedVirtualPath] = [
         TypedVirtualPath(file: moduleInfo.modulePath.path, type: .swiftModule)
@@ -488,6 +487,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
   }
 }
 
+internal extension ExplicitDependencyBuildPlanner {
+  func explainDependency(_ dependencyModuleName: String) throws -> [[ModuleDependencyId]]? {
+    return try dependencyGraph.explainDependency(dependencyModuleName: dependencyModuleName)
+  }
+}
+
 // InterModuleDependencyGraph printing, useful for debugging
 internal extension InterModuleDependencyGraph {
   func prettyPrintString() throws -> String {

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -75,9 +75,22 @@ extension InterModuleDependencyGraph {
       }
     // Traverse the set of modules in reverse topological order, assimilating transitive closures
     for moduleId in topologicalIdList.reversed() {
-      for dependencyId in try moduleInfo(of: moduleId).directDependencies! {
+      let moduleInfo = try moduleInfo(of: moduleId)
+      for dependencyId in moduleInfo.directDependencies! {
         transitiveClosureMap[moduleId]!.formUnion(transitiveClosureMap[dependencyId]!)
       }
+      // For Swift dependencies, their corresponding Swift Overlay dependencies
+      // and bridging header dependencies are equivalent to direct dependencies.
+      if case .swift(let swiftModuleDetails) = moduleInfo.details {
+        let swiftOverlayDependencies = swiftModuleDetails.swiftOverlayDependencies ?? []
+        for dependencyId in swiftOverlayDependencies {
+          transitiveClosureMap[moduleId]!.formUnion(transitiveClosureMap[dependencyId]!)
+        }
+        let bridgingHeaderDependencies = swiftModuleDetails.bridgingHeaderDependencies ?? []
+        for dependencyId in bridgingHeaderDependencies {
+          transitiveClosureMap[moduleId]!.formUnion(transitiveClosureMap[dependencyId]!)
+        }
+      }
     }
     // For ease of use down-the-line, remove the node's self from its set of reachable nodes
     for (key, _) in transitiveClosureMap {
@@ -222,3 +235,48 @@ extension InterModuleDependencyGraph {
                       details: .clang(combinedModuleDetails))
   }
 }
+
+internal extension InterModuleDependencyGraph {
+  func explainDependency(dependencyModuleName: String) throws -> [[ModuleDependencyId]]? {
+    guard modules.contains(where: { $0.key.moduleName == dependencyModuleName }) else { return nil }
+    var results = [[ModuleDependencyId]]()
+    try findAllPaths(source: .swift(mainModuleName),
+                     to: dependencyModuleName,
+                     pathSoFar: [.swift(mainModuleName)],
+                     results: &results)
+    return Array(results)
+  }
+
+
+  private func findAllPaths(source: ModuleDependencyId,
+                            to moduleName: String,
+                            pathSoFar: [ModuleDependencyId],
+                            results: inout [[ModuleDependencyId]]) throws {
+    let sourceInfo = try moduleInfo(of: source)
+    // If the source is our target, we are done
+    guard source.moduleName != moduleName else {
+      // If the source is a target Swift module, also check if it
+      // depends on a corresponding Clang module with the same name.
+      // If it does, add it to the path as well.
+      var completePath = pathSoFar
+      if let dependencies = sourceInfo.directDependencies,
+         dependencies.contains(.clang(moduleName)) {
+        completePath.append(.clang(moduleName))
+      }
+      results.append(completePath)
+      return
+    }
+
+    // If no more dependencies, this is a leaf node, we are done
+    guard let dependencies = sourceInfo.directDependencies else {
+      return
+    }
+
+    for dependency in dependencies {
+      try findAllPaths(source: dependency,
+                       to: moduleName,
+                       pathSoFar: pathSoFar + [dependency],
+                       results: &results)
+    }
+  }
+}

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -79,7 +79,9 @@ extension ModuleDependencyId: Codable {
 /// Bridging header
 public struct BridgingHeader: Codable {
   var path: TextualVirtualPath
+  /// The source files referenced by the bridging header.
   var sourceFiles: [TextualVirtualPath]
+  /// Modules that the bridging header specifically depends on
   var moduleDependencies: [String]
 }
 
@@ -92,13 +94,16 @@ public struct SwiftModuleDetails: Codable {
   public var compiledModuleCandidates: [TextualVirtualPath]?
 
   /// The bridging header, if any.
-  public var bridgingHeaderPath: TextualVirtualPath?
-
-  /// The source files referenced by the bridging header.
-  public var bridgingSourceFiles: [TextualVirtualPath]? = []
-
-  /// Modules that the bridging header specifically depends on
-  public var bridgingHeaderDependencies: [ModuleDependencyId]? = []
+  public var bridgingHeader: BridgingHeader?
+  public var bridgingHeaderPath: TextualVirtualPath? {
+    bridgingHeader?.path
+  }
+  public var bridgingSourceFiles: [TextualVirtualPath]? {
+    bridgingHeader?.sourceFiles
+  }
+  public var bridgingHeaderDependencies: [ModuleDependencyId]? {
+    bridgingHeader?.moduleDependencies.map { .clang($0) }
+  }
 
   /// Options to the compile command
   public var commandLine: [String]? = []
@@ -115,6 +120,9 @@ public struct SwiftModuleDetails: Codable {
 
   /// A flag to indicate whether or not this module is a framework.
   public var isFramework: Bool?
+
+  /// A set of Swift Overlays of Clang Module Dependencies
+  var swiftOverlayDependencies: [ModuleDependencyId]?
 }
 
 /// Details specific to Swift placeholder dependencies.

Sources/SwiftDriver/Jobs/Planning.swift

@@ -140,7 +140,8 @@ extension Driver {
                                                           IncrementalCompilationState.InitialStateForPlanning?)
   throws -> InterModuleDependencyGraph? {
     let interModuleDependencyGraph: InterModuleDependencyGraph?
-    if parsedOptions.contains(.driverExplicitModuleBuild) {
+    if parsedOptions.contains(.driverExplicitModuleBuild) ||
+       parsedOptions.contains(.explainModuleDependency) {
       interModuleDependencyGraph =
         try initialIncrementalState?.maybeUpToDatePriorInterModuleDependencyGraph ??
             gatherModuleDependencies()
@@ -195,13 +196,15 @@ extension Driver {
     dependencyGraph: InterModuleDependencyGraph?,
     addJob: (Job) -> Void)
   throws {
-    // If asked, add jobs to precompile module dependencies
-    guard parsedOptions.contains(.driverExplicitModuleBuild) else { return }
     guard let resolvedDependencyGraph = dependencyGraph else {
-      fatalError("Attempting to plan explicit dependency build without a dependency graph")
+      return
     }
     let modulePrebuildJobs =
         try generateExplicitModuleDependenciesJobs(dependencyGraph: resolvedDependencyGraph)
+    // If asked, add jobs to precompile module dependencies. Otherwise exit.
+    // We may have a dependency graph but not be required to add pre-compile jobs to the build plan,
+    // for example when `-explain-dependency` is being used.
+    guard parsedOptions.contains(.driverExplicitModuleBuild) else { return }
     modulePrebuildJobs.forEach(addJob)
   }
 

Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift

@@ -123,7 +123,8 @@ private extension SwiftScan {
     guard let moduleDetailsRef = api.swiftscan_module_info_get_details(moduleInfoRef) else {
       throw DependencyScanningError.missingField("modules[\(moduleId)].details")
     }
-    let details = try constructModuleDetails(from: moduleDetailsRef)
+    let details = try constructModuleDetails(from: moduleDetailsRef,
+                                             moduleAliases: moduleAliases)
 
     return (moduleId, ModuleInfo(modulePath: modulePath, sourceFiles: sourceFiles,
                                  directDependencies: directDependencies,
@@ -133,12 +134,14 @@ private extension SwiftScan {
   /// From a reference to a binary-format module info details object info returned by libSwiftScan,
   /// construct an instance of an `ModuleInfo`.Details as used by the driver.
   /// The object returned by libSwiftScan is a union so ensure to execute dependency-specific queries.
-  func constructModuleDetails(from moduleDetailsRef: swiftscan_module_details_t)
+  func constructModuleDetails(from moduleDetailsRef: swiftscan_module_details_t,
+                              moduleAliases: [String: String]?)
   throws -> ModuleInfo.Details {
     let moduleKind = api.swiftscan_module_detail_get_kind(moduleDetailsRef)
     switch moduleKind {
       case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL:
-        return .swift(try constructSwiftTextualModuleDetails(from: moduleDetailsRef))
+        return .swift(try constructSwiftTextualModuleDetails(from: moduleDetailsRef,
+                                                             moduleAliases: moduleAliases))
       case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_BINARY:
         return .swiftPrebuiltExternal(try constructSwiftBinaryModuleDetails(from: moduleDetailsRef))
       case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER:
@@ -151,7 +154,8 @@ private extension SwiftScan {
   }
 
   /// Construct a `SwiftModuleDetails` from a `swiftscan_module_details_t` reference
-  func constructSwiftTextualModuleDetails(from moduleDetailsRef: swiftscan_module_details_t)
+  func constructSwiftTextualModuleDetails(from moduleDetailsRef: swiftscan_module_details_t,
+                                          moduleAliases: [String: String]?)
   throws -> SwiftModuleDetails {
     let moduleInterfacePath =
       try getOptionalPathDetail(from: moduleDetailsRef,
@@ -168,6 +172,15 @@ private extension SwiftScan {
     let bridgingHeaderDependencies =
       try getOptionalStringArrayDetail(from: moduleDetailsRef,
                                        using: api.swiftscan_swift_textual_detail_get_bridging_module_dependencies)
+    let bridgingHeader: BridgingHeader?
+    if let resolvedBridgingHeaderPath = bridgingHeaderPath {
+      bridgingHeader = BridgingHeader(path: resolvedBridgingHeaderPath,
+                                      sourceFiles: bridgingSourceFiles ?? [],
+                                      moduleDependencies: bridgingHeaderDependencies ?? [])
+    } else {
+      bridgingHeader = nil
+    }
+
     let commandLine =
       try getOptionalStringArrayDetail(from: moduleDetailsRef,
                                        using: api.swiftscan_swift_textual_detail_get_command_line)
@@ -180,15 +193,25 @@ private extension SwiftScan {
                           using: api.swiftscan_swift_textual_detail_get_context_hash)
     let isFramework = api.swiftscan_swift_textual_detail_get_is_framework(moduleDetailsRef)
 
+    // Decode all dependencies of this module
+    let swiftOverlayDependencies: [ModuleDependencyId]?
+    if supportsSeparateSwiftOverlayDependencies(),
+       let encodedOverlayDepsRef = api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies(moduleDetailsRef) {
+      let encodedOverlayDependencies = try toSwiftStringArray(encodedOverlayDepsRef.pointee)
+      swiftOverlayDependencies =
+        try encodedOverlayDependencies.map { try decodeModuleNameAndKind(from: $0, moduleAliases: moduleAliases) }
+    } else {
+      swiftOverlayDependencies = nil
+    }
+
     return SwiftModuleDetails(moduleInterfacePath: moduleInterfacePath,
                               compiledModuleCandidates: compiledModuleCandidates,
-                              bridgingHeaderPath: bridgingHeaderPath,
-                              bridgingSourceFiles: bridgingSourceFiles,
-                              bridgingHeaderDependencies: bridgingHeaderDependencies?.map { .clang($0) },
+                              bridgingHeader: bridgingHeader,
                               commandLine: commandLine,
                               contextHash: contextHash,
                               extraPcmArgs: extraPcmArgs,
-                              isFramework: isFramework)
+                              isFramework: isFramework,
+                              swiftOverlayDependencies: swiftOverlayDependencies)
   }
 
   /// Construct a `SwiftPrebuiltExternalModuleDetails` from a `swiftscan_module_details_t` reference

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -263,6 +263,10 @@ internal extension swiftscan_diagnostic_severity_t {
   func resetScannerCache() {
     api.swiftscan_scanner_cache_reset(scanner)
   }
+
+  @_spi(Testing) public func supportsSeparateSwiftOverlayDependencies() -> Bool {
+    return api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies != nil
+  }
 
   @_spi(Testing) public func supportsScannerDiagnostics() -> Bool {
     return api.swiftscan_scanner_diagnostics_query != nil &&
@@ -419,6 +423,10 @@ private extension swiftscan_functions_t {
     self.swiftscan_swift_binary_detail_get_is_framework =
       try loadOptional("swiftscan_swift_binary_detail_get_is_framework")
 
+    // Swift Overlay Dependencies
+    self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies =
+      try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies")
+
     // MARK: Required Methods
     func loadRequired<T>(_ symbol: String) throws -> T {
       guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else {