[Explicit Module Builds] Add option to invoke the command-line executable dependency scanner instead of using libSwiftScan

Author: artemcm

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -99,31 +99,101 @@ internal extension Driver {
     let scannerJob = try dependencyScanningJob()
     let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
     let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
-    var command = try itemizedJobCommand(of: scannerJob,
-                                         forceResponseFiles: forceResponseFiles,
-                                         using: executor.resolver)
-    // Remove the tool executable to only leave the arguments
-    command.removeFirst()
-    let dependencyGraph =
-      try interModuleDependencyOracle.getDependencies(workingDirectory: cwd,
-                                                      commandLine: command)
+
+    let dependencyGraph: InterModuleDependencyGraph
+
+    if (parsedOptions.hasArgument(.driverScanDependenciesNonLib)) {
+
+      var command = try itemizedJobCommand(of: scannerJob,
+                                           forceResponseFiles: forceResponseFiles,
+                                           using: executor.resolver)
+      // Remove the tool executable to only leave the arguments
+      command.removeFirst()
+      dependencyGraph =
+        try interModuleDependencyOracle.getDependencies(workingDirectory: cwd,
+                                                        commandLine: command)
+    } else {
+      // Fallback to legacy invocation of the dependency scanner with
+      // `swift-frontend -scan-dependencies`
+      print("Dependency Scanner invocation:")
+      print(try executor.description(of: scannerJob, forceResponseFiles: false))
+      dependencyGraph =
+        try self.executor.execute(job: scannerJob,
+                                  capturingJSONOutputAs: InterModuleDependencyGraph.self,
+                                  forceResponseFiles: forceResponseFiles,
+                                  recordedInputModificationDates: recordedInputModificationDates)
+    }
     return dependencyGraph
   }
 
   mutating func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo])
   throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
     let batchScanningJob = try batchDependencyScanningJob(for: moduleInfos)
     let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
-    let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
-    var command = try itemizedJobCommand(of: batchScanningJob,
-                                         forceResponseFiles: forceResponseFiles,
-                                         using: executor.resolver)
-    // Remove the tool executable to only leave the arguments
-    command.removeFirst()
+
+    let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]]
+    if (!parsedOptions.hasArgument(.driverScanDependenciesNonLib)) {
+      let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
+      var command = try itemizedJobCommand(of: batchScanningJob,
+                                           forceResponseFiles: forceResponseFiles,
+                                           using: executor.resolver)
+      // Remove the tool executable to only leave the arguments
+      command.removeFirst()
+      moduleVersionedGraphMap =
+        try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd,
+                                                             commandLine: command,
+                                                             batchInfos: moduleInfos)
+    } else {
+      // Fallback to legacy invocation of the dependency scanner with
+      // `swift-frontend -scan-dependencies`
+      print("Dependency Scanner (batch) invocation:")
+      print(try executor.description(of: batchScanningJob, forceResponseFiles: false))
+      moduleVersionedGraphMap = try executeLegacyBatchScan(moduleInfos: moduleInfos,
+                                                           batchScanningJob: batchScanningJob,
+                                                           forceResponseFiles: forceResponseFiles)
+    }
+    return moduleVersionedGraphMap
+  }
+
+  // Perform a batch scan by invoking the command-line dependency scanner and decoding the resulting
+  // JSON.
+  fileprivate func executeLegacyBatchScan(moduleInfos: [BatchScanModuleInfo],
+                                          batchScanningJob: Job,
+                                          forceResponseFiles: Bool)
+  throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
+    let batchScanResult =
+      try self.executor.execute(job: batchScanningJob,
+                                forceResponseFiles: forceResponseFiles,
+                                recordedInputModificationDates: recordedInputModificationDates)
+    let success = batchScanResult.exitStatus == .terminated(code: EXIT_SUCCESS)
+    guard success else {
+      throw JobExecutionError.jobFailedWithNonzeroExitCode(
+        type(of: executor).computeReturnCode(exitStatus: batchScanResult.exitStatus),
+        try batchScanResult.utf8stderrOutput())
+    }
+    // Decode the resulting dependency graphs and build a dictionary from a moduleId to
+    // a set of dependency graphs that were built for it
     let moduleVersionedGraphMap =
-      try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd,
-                                                           commandLine: command,
-                                                           batchInfos: moduleInfos)
+      try moduleInfos.reduce(into: [ModuleDependencyId: [InterModuleDependencyGraph]]()) {
+        let moduleId: ModuleDependencyId
+        let dependencyGraphPath: VirtualPath
+        switch $1 {
+          case .swift(let swiftModuleBatchScanInfo):
+            moduleId = .swift(swiftModuleBatchScanInfo.swiftModuleName)
+            dependencyGraphPath = try VirtualPath(path: swiftModuleBatchScanInfo.output)
+          case .clang(let clangModuleBatchScanInfo):
+            moduleId = .clang(clangModuleBatchScanInfo.clangModuleName)
+            dependencyGraphPath = try VirtualPath(path: clangModuleBatchScanInfo.output)
+        }
+        let contents = try fileSystem.readFileContents(dependencyGraphPath)
+        let decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self,
+                                                    from: Data(contents.contents))
+        if $0[moduleId] != nil {
+          $0[moduleId]!.append(decodedGraph)
+        } else {
+          $0[moduleId] = [decodedGraph]
+        }
+      }
     return moduleVersionedGraphMap
   }
 

Sources/SwiftOptions/ExtraOptions.swift

@@ -12,13 +12,15 @@
 extension Option {
   public static let driverPrintGraphviz: Option = Option("-driver-print-graphviz", .flag, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Write the job graph as a graphviz file", group: .internalDebug)
   public static let driverExplicitModuleBuild: Option = Option("-experimental-explicit-module-build", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit")
+  public static let driverScanDependenciesNonLib: Option = Option("-nonlib-dependency-scanner", .flag, attributes: [.helpHidden], helpText: "Use calls to `swift-frontend -scan-dependencies` instead of dedicated dependency scanning library")
   public static let driverWarnUnusedOptions: Option = Option("-driver-warn-unused-options", .flag, attributes: [.helpHidden], helpText: "Emit warnings for any provided options which are unused by the driver.")
   public static let emitModuleSeparately: Option = Option("-experimental-emit-module-separately", .flag, attributes: [.helpHidden], helpText: "Emit module files as a distinct job")
 
   public static var extraOptions: [Option] {
     return [
       Option.driverPrintGraphviz,
       Option.driverExplicitModuleBuild,
+      Option.driverScanDependenciesNonLib,
       Option.driverWarnUnusedOptions,
       Option.emitModuleSeparately
     ]