[Dependency Scanning] Add ability to use the scanner's import-prescan mode

artemcm · artemcm · commit e5d6e9f8ca77 · 2021-06-16T14:42:09.000-07:00
An ability to do a simple import pre-scan was added to the dependency scanner in swiftlang/swift#34028, with a corresponding libSwiftScan API added later. This PR adds the ability to invoke this import pre-scan from the driver.
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift
@@ -266,3 +266,7 @@ public struct InterModuleDependencyGraph: Codable {
   /// Information about the main module.
   public var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }
 }
+
+public struct InterModuleDependencyImports: Codable {
+  public var imports: [String]
+}
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift
@@ -52,6 +52,16 @@ public class InterModuleDependencyOracle {
     }
   }
 
+  @_spi(Testing) public func getImports(workingDirectory: AbsolutePath,
+                                             commandLine: [String])
+  throws -> InterModuleDependencyImports {
+    precondition(hasScannerInstance)
+    return try queue.sync {
+      return try swiftScanLibInstance!.preScanImports(workingDirectory: workingDirectory,
+                                                      invocationCommand: commandLine)
+    }
+  }
+
   /// Given a specified toolchain path, locate and instantiate an instance of the SwiftScan library
   /// Returns True if a library instance exists (either verified or newly-created).
   @_spi(Testing) public func verifyOrCreateScannerInstance(fileSystem: FileSystem,
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift
@@ -19,11 +19,11 @@ extension Diagnostic.Message {
   }
 }
 
-internal extension Driver {
+public extension Driver {
   /// Precompute the dependencies for a given Swift compilation, producing a
   /// dependency graph including all Swift and C module files and
   /// source files.
-  mutating func dependencyScanningJob() throws -> Job {
+  mutating private func dependencyScanningJob() throws -> Job {
     let (inputs, commandLine) = try dependencyScannerInvocationCommand()
 
     // Construct the scanning job.
@@ -40,7 +40,7 @@ internal extension Driver {
 
   /// Generate a full command-line invocation to be used for the dependency scanning action
   /// on the target module.
-  mutating func dependencyScannerInvocationCommand()
+  mutating private func dependencyScannerInvocationCommand()
   throws -> ([TypedVirtualPath],[Job.ArgTemplate]) {
     // Aggregate the fast dependency scanner arguments
     var inputs: [TypedVirtualPath] = []
@@ -66,7 +66,7 @@ internal extension Driver {
   }
 
   /// Serialize a map of placeholder (external) dependencies for the dependency scanner.
-  func serializeExternalDependencyArtifacts(externalBuildArtifacts: ExternalBuildArtifacts)
+  private func serializeExternalDependencyArtifacts(externalBuildArtifacts: ExternalBuildArtifacts)
   throws -> VirtualPath {
     let (externalTargetModulePathMap, externalModuleInfoMap)  = externalBuildArtifacts
     var placeholderArtifacts: [SwiftModuleArtifactInfo] = []
@@ -94,11 +94,8 @@ internal extension Driver {
                                                                   contents)
   }
 
-  mutating func performDependencyScan() throws -> InterModuleDependencyGraph {
-    let scannerJob = try dependencyScanningJob()
-    let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
-    let dependencyGraph: InterModuleDependencyGraph
-
+  /// Returns false if the lib is available and ready to use
+  private func initSwiftScanLib() throws -> Bool {
     // If `-nonlib-dependency-scanner` was specified or the libSwiftScan library cannot be found,
     // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency
     // scanning.
@@ -110,20 +107,62 @@ internal extension Driver {
       fallbackToFrontend = true
       diagnosticEngine.emit(.warn_scanner_frontend_fallback())
     }
+    return fallbackToFrontend
+  }
+
+  private func sanitizeCommandForLibScanInvocation(_ command: inout [String]) {
+    // Remove the tool executable to only leave the arguments. When passing the
+    // command line into libSwiftScan, the library is itself the tool and only
+    // needs to parse the remaining arguments.
+    command.removeFirst()
+    // We generate full swiftc -frontend -scan-dependencies invocations in order to also be
+    // able to launch them as standalone jobs. Frontend's argument parser won't recognize
+    // -frontend when passed directly.
+    if command.first == "-frontend" {
+      command.removeFirst()
+    }
+  }
 
-    if (!fallbackToFrontend) {
+  mutating func performImportPrescan() throws -> InterModuleDependencyImports {
+    let preScanJob = try importPreScanningJob()
+    let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
+    let imports: InterModuleDependencyImports
+
+    let isSwiftScanLibAvailable = !(try initSwiftScanLib())
+    if isSwiftScanLibAvailable {
+      let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
+      var command = try itemizedJobCommand(of: preScanJob,
+                                           forceResponseFiles: forceResponseFiles,
+                                           using: executor.resolver)
+      sanitizeCommandForLibScanInvocation(&command)
+      imports =
+        try interModuleDependencyOracle.getImports(workingDirectory: cwd,
+                                                   commandLine: command)
+
+    } else {
+      // Fallback to legacy invocation of the dependency scanner with
+      // `swift-frontend -scan-dependencies -import-prescan`
+      imports =
+        try self.executor.execute(job: preScanJob,
+                                  capturingJSONOutputAs: InterModuleDependencyImports.self,
+                                  forceResponseFiles: forceResponseFiles,
+                                  recordedInputModificationDates: recordedInputModificationDates)
+    }
+    return imports
+  }
+
+  mutating internal func performDependencyScan() throws -> InterModuleDependencyGraph {
+    let scannerJob = try dependencyScanningJob()
+    let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
+    let dependencyGraph: InterModuleDependencyGraph
+
+    let isSwiftScanLibAvailable = !(try initSwiftScanLib())
+    if isSwiftScanLibAvailable {
       let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
       var command = try itemizedJobCommand(of: scannerJob,
                                            forceResponseFiles: forceResponseFiles,
                                            using: executor.resolver)
-      // Remove the tool executable to only leave the arguments
-      command.removeFirst()
-      // We generate full swiftc -frontend -scan-dependencies invocations in order to also be
-      // able to launch them as standalone jobs. Frontend's argument parser won't recognize
-      // -frontend when passed directly.
-      if command.first == "-frontend" {
-        command.removeFirst()
-      }
+      sanitizeCommandForLibScanInvocation(&command)
       dependencyGraph =
         try interModuleDependencyOracle.getDependencies(workingDirectory: cwd,
                                                         commandLine: command)
@@ -139,37 +178,19 @@ internal extension Driver {
     return dependencyGraph
   }
 
-  mutating func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo])
+  mutating internal func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo])
   throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
     let batchScanningJob = try batchDependencyScanningJob(for: moduleInfos)
     let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
-
-    // If `-nonlib-dependency-scanner` was specified or the libSwiftScan library cannot be found,
-    // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency
-    // scanning.
-    var fallbackToFrontend = parsedOptions.hasArgument(.driverScanDependenciesNonLib)
-    let scanLibPath = try Self.getScanLibPath(of: toolchain, hostTriple: hostTriple, env: env)
-    if try interModuleDependencyOracle
-        .verifyOrCreateScannerInstance(fileSystem: fileSystem,
-                                       swiftScanLibPath: scanLibPath) == false {
-      fallbackToFrontend = true
-      diagnosticEngine.emit(.warn_scanner_frontend_fallback())
-    }
-
     let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]]
-    if (!fallbackToFrontend) {
+
+    let isSwiftScanLibAvailable = !(try initSwiftScanLib())
+    if isSwiftScanLibAvailable {
       let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
       var command = try itemizedJobCommand(of: batchScanningJob,
                                            forceResponseFiles: forceResponseFiles,
                                            using: executor.resolver)
-      // Remove the tool executable to only leave the arguments
-      command.removeFirst()
-      // We generate full swiftc -frontend -scan-dependencies invocations in order to also be
-      // able to launch them as standalone jobs. Frontend's argument parser won't recognize
-      // -frontend when passed directly.
-      if command.first == "-frontend" {
-        command.removeFirst()
-      }
+      sanitizeCommandForLibScanInvocation(&command)
       moduleVersionedGraphMap =
         try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd,
                                                              commandLine: command,
@@ -226,8 +247,36 @@ internal extension Driver {
     return moduleVersionedGraphMap
   }
 
+  /// Precompute the set of module names as imported by the current module
+  mutating private func importPreScanningJob() throws -> Job {
+    // Aggregate the fast dependency scanner arguments
+    var inputs: [TypedVirtualPath] = []
+    var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
+    commandLine.appendFlag("-frontend")
+    commandLine.appendFlag("-scan-dependencies")
+    commandLine.appendFlag("-import-prescan")
+    try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs,
+                                 bridgingHeaderHandling: .precompiled,
+                                 moduleDependencyGraphUse: .dependencyScan)
+    // FIXME: MSVC runtime flags
+
+    // Pass on the input files
+    commandLine.append(contentsOf: inputFiles.map { .path($0.file) })
+
+    // Construct the scanning job.
+    return Job(moduleName: moduleOutputInfo.name,
+               kind: .scanDependencies,
+               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               commandLine: commandLine,
+               displayInputs: inputs,
+               inputs: inputs,
+               primaryInputs: [],
+               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
+               supportsResponseFiles: true)
+  }
+
   /// Precompute the dependencies for a given collection of modules using swift frontend's batch scanning mode
-  mutating func batchDependencyScanningJob(for moduleInfos: [BatchScanModuleInfo]) throws -> Job {
+  mutating private func batchDependencyScanningJob(for moduleInfos: [BatchScanModuleInfo]) throws -> Job {
     var inputs: [TypedVirtualPath] = []
 
     // Aggregate the fast dependency scanner arguments
diff --git a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift
@@ -42,6 +42,16 @@ internal extension SwiftScan {
     return resultGraph
   }
 
+  /// From a reference to a binary-format set of module imports return by libSwiftScan pre-scan query,
+  /// construct an instance of an `InterModuleDependencyImports` set
+  func constructImportSet(from importSetRef: swiftscan_import_set_t) throws
+  -> InterModuleDependencyImports {
+    guard let importsRef = api.swiftscan_import_set_get_imports(importSetRef) else {
+      throw DependencyScanningError.missingField("import_set.imports")
+    }
+    return InterModuleDependencyImports(imports: try toSwiftStringArray(importsRef.pointee))
+  }
+
   /// From a reference to a binary-format dependency graph collection returned by libSwiftScan batch scan query,
   /// corresponding to the specified batch scan input (`BatchScanModuleInfo`), construct instances of
   /// `InterModuleDependencyGraph` for each result.
diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift
@@ -92,6 +92,33 @@ internal final class SwiftScan {
     dylib.leak()
   }
 
+  func preScanImports(workingDirectory: AbsolutePath,
+                      invocationCommand: [String]) throws -> InterModuleDependencyImports {
+    // Create and configure the scanner invocation
+    let invocation = api.swiftscan_scan_invocation_create()
+    defer { api.swiftscan_scan_invocation_dispose(invocation) }
+    api.swiftscan_scan_invocation_set_working_directory(invocation,
+                                                        workingDirectory
+                                                          .description
+                                                          .cString(using: String.Encoding.utf8))
+    withArrayOfCStrings(invocationCommand) { invocationStringArray in
+      api.swiftscan_scan_invocation_set_argv(invocation,
+                                             Int32(invocationCommand.count),
+                                             invocationStringArray)
+    }
+
+    let importSetRefOrNull = api.swiftscan_import_set_create(scanner, invocation)
+    guard let importSetRef = importSetRefOrNull else {
+      throw DependencyScanningError.dependencyScanFailed
+    }
+
+    let importSet = try constructImportSet(from: importSetRef)
+    // Free the memory allocated for the in-memory representation of the import set
+    // returned by the scanner, now that we have translated it.
+    api.swiftscan_import_set_dispose(importSetRef)
+    return importSet
+  }
+
   func scanDependencies(workingDirectory: AbsolutePath,
                         invocationCommand: [String]) throws -> InterModuleDependencyGraph {
     // Create and configure the scanner invocation
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
@@ -658,6 +658,54 @@ final class ExplicitModuleBuildTests: XCTestCase {
     return (stdLibPath, shimsPath, driver.toolchain, driver.hostTriple)
   }
 
+  /// Test the libSwiftScan dependency scanning (import-prescan).
+  func testDependencyImportPrescan() throws {
+    let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+
+    // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
+    // queries.
+    let dependencyOracle = InterModuleDependencyOracle()
+    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
+                                                hostTriple: hostTriple,
+                                                env: ProcessEnv.vars)
+    guard try dependencyOracle
+            .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                           swiftScanLibPath: scanLibPath) else {
+      XCTFail("Dependency scanner library not found")
+      return
+    }
+
+    // Create a simple test case.
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testDependencyScanning.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import C;"
+        $0 <<< "import E;"
+        $0 <<< "import G;"
+      }
+      let packageRootPath = URL(fileURLWithPath: #file).pathComponents
+        .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
+      let testInputsPath = packageRootPath + "/TestInputs"
+      let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
+      let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
+      let scannerCommand = ["-scan-dependencies",
+                            "-import-prescan",
+                            "-I", cHeadersPath,
+                            "-I", swiftModuleInterfacesPath,
+                            "-I", stdLibPath.description,
+                            "-I", shimsPath.description,
+                            main.pathString]
+
+      let imports =
+        try! dependencyOracle.getImports(workingDirectory: path,
+                                         commandLine: scannerCommand)
+      let expectedImports = ["C", "E", "G", "Swift", "SwiftOnoneSupport"]
+      // Dependnig on how recent the platform we are running on, the Concurrency module may or may not be present.
+      let expectedImports2 = ["C", "E", "G", "Swift", "SwiftOnoneSupport", "_Concurrency"]
+      XCTAssertTrue(Set(imports.imports) == Set(expectedImports) || Set(imports.imports) == Set(expectedImports2))
+    }
+  }
+
   /// Test the libSwiftScan dependency scanning.
   func testDependencyScanning() throws {
     let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()

Original file line number	Diff line number	Diff line change
`@@ -266,3 +266,7 @@ public struct InterModuleDependencyGraph: Codable {`
`266`	`266`	`/// Information about the main module.`
`267`	`267`	`public var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }`
`268`	`268`	`}`
	`269`	`+`
	`270`	`+public struct InterModuleDependencyImports: Codable {`
	`271`	`+ public var imports: [String]`
	`272`	`+}`