Merge pull request #592 from nkcsgexi/handle-scoped-prebuilt

PrebuiltModuleGen: support a core mode where only Foundation and below are handled.

Author: nkcsgexi

Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift

@@ -197,8 +197,9 @@ extension Driver {
     )
   }
 
-  public mutating func generatePrebuitModuleGenerationJobs(_ inputMap: [String: [PrebuiltModuleInput]],
-                                                           _ prebuiltModuleDir: AbsolutePath) throws -> ([Job], [Job]) {
+  public mutating func generatePrebuitModuleGenerationJobs(with inputMap: [String: [PrebuiltModuleInput]],
+                                                           into prebuiltModuleDir: AbsolutePath,
+                                                           exhaustive: Bool) throws -> ([Job], [Job]) {
     assert(sdkPath != nil)
     // Run the dependency scanner and update the dependency oracle with the results
     let dependencyGraph = try gatherModuleDependencies()
@@ -225,22 +226,32 @@ extension Driver {
       return (key, outputPaths)
     })
 
-    func getDependenciesPaths(_ module: String, _ arch: Triple.Arch) throws -> [TypedVirtualPath] {
-      var results: [TypedVirtualPath] = []
+    func collectSwiftModuleNames(_ ids: [ModuleDependencyId]) -> [String] {
+      return ids.compactMap { id in
+        if case .swift(let module) = id {
+          return module
+        }
+        return nil
+      }
+    }
+
+    func getSwiftDependencies(for module: String) -> [String] {
       let info = dependencyGraph.modules[.swift(module)]!
       guard let dependencies = info.directDependencies else {
-        return results
+        return []
       }
+      return collectSwiftModuleNames(dependencies)
+    }
 
-      for dep in dependencies {
-        if case let .swift(moduleName) = dep {
-          if let outputs = outputMap[moduleName] {
-            // Depending only those .swiftmodule files with the same arch kind.
-            // FIXME: handling arm64 and arm64e specifically.
-            let selectOutputs = outputs.filter ({ $0.arch == arch }).map { $0.path }
-            results.append(contentsOf: selectOutputs)
-          }
+    func getOutputPaths(for modules: [String], with arch: Triple.Arch) throws -> [TypedVirtualPath] {
+      var results: [TypedVirtualPath] = []
+      modules.forEach { module in
+        guard let allOutputs = outputMap[module] else {
+          diagnosticEngine.emit(error: "cannot find output paths for \(module)")
+          return
         }
+        let allPaths = allOutputs.filter { $0.arch == arch }.map { $0.path }
+        results.append(contentsOf: allPaths)
       }
       return results
     }
@@ -252,30 +263,43 @@ extension Driver {
         assert(inputPaths.count == outputPaths.count)
         assert(!inputPaths.isEmpty)
         for i in 0..<inputPaths.count {
-          try action(inputPaths[i], outputPaths[i])
+          let (input, output) = (inputPaths[i], outputPaths[i])
+          assert(input.path.file.basenameWithoutExt == output.path.file.basenameWithoutExt)
+          try action(input, output)
         }
       }
     }
-    // Keep track of modules that are not handled.
+    // Keep track of modules we haven't handled.
     var unhandledModules = Set<String>(inputMap.keys)
-    let moduleInfo = dependencyGraph.mainModule
-    if let dependencies = moduleInfo.directDependencies {
-      for dep in dependencies {
-        switch dep {
-        case .swift(let moduleName):
-          // Removed moduleName from the list.
-          unhandledModules.remove(moduleName)
-          try forEachInputOutputPair(moduleName) {
-            jobs.append(try generateSingleModuleBuildingJob(moduleName,
-              prebuiltModuleDir, $0, $1,
-              try getDependenciesPaths(moduleName, $0.arch)))
-          }
-        default:
-          continue
+    if let importedModules = dependencyGraph.mainModule.directDependencies {
+      // Start from those modules explicitly imported into the file under scanning
+      var openModules = collectSwiftModuleNames(importedModules)
+      var idx = 0
+      while idx != openModules.count {
+        let module = openModules[idx]
+        let dependencies = getSwiftDependencies(for: module)
+        try forEachInputOutputPair(module) { input, output in
+          jobs.append(try generateSingleModuleBuildingJob(module,
+            prebuiltModuleDir, input, output,
+            try getOutputPaths(for: dependencies, with: input.arch)))
         }
+        // For each dependency, add to the list to handle if the list doesn't
+        // contain this dependency.
+        dependencies.forEach({ newModule in
+          if !openModules.contains(newModule) {
+            diagnosticEngine.emit(note: "\(newModule) is discovered.")
+            openModules.append(newModule)
+          }
+        })
+        unhandledModules.remove(module)
+        idx += 1
       }
     }
 
+    // We are done if we don't need to handle all inputs exhaustively.
+    if !exhaustive {
+      return (jobs, [])
+    }
     // For each unhandled module, generate dangling jobs for each associated
     // interfaces.
     // The only known usage of this so for is in macosx SDK where some collected

Sources/swift-build-sdk-interfaces/main.swift

@@ -34,6 +34,10 @@ if rawOutputDir.isEmpty {
   exit(1)
 }
 
+/// When -core is specified, only most significant modules are handled. Currently,
+/// they are Foundation and anything below.
+let coreMode = CommandLine.arguments.contains("-core")
+
 class PrebuitGenDelegate: JobExecutionDelegate {
   var failingModules = Set<String>()
   var commandMap: [Int: String] = [:]
@@ -55,8 +59,10 @@ class PrebuitGenDelegate: JobExecutionDelegate {
           stderrStream.flush()
         }
       }
+#if !os(Windows)
     case .signalled:
       diagnosticsEngine.emit(.remark("\(job.moduleName) interrupted"))
+#endif
     }
   }
 
@@ -110,7 +116,7 @@ do {
                            to: sysVersionFile)
   let processSet = ProcessSet()
   let inputMap = try collector.collectSwiftInterfaceMap()
-  let allModules = inputMap.keys
+  let allModules = coreMode ? ["Foundation"] : Array(inputMap.keys)
   try withTemporaryFile(suffix: ".swift") {
     let tempPath = $0.path
     try localFileSystem.writeFileContents(tempPath, body: {
@@ -129,7 +135,7 @@ do {
                             diagnosticsEngine: diagnosticsEngine,
                             executor: executor,
                             compilerExecutableDir: swiftcPath.parentDirectory)
-    let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(inputMap, outputDir)
+    let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap, into: outputDir, exhaustive: !coreMode)
     let delegate = PrebuitGenDelegate()
     do {
       try executor.execute(workload: DriverExecutorWorkload.init(jobs, nil, continueBuildingAfterErrors: true),

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -771,6 +771,22 @@ final class ExplicitModuleBuildTests: XCTestCase {
         XCTAssertTrue(input.file.basename == name)
       }
     }
+    let packageRootPath = URL(fileURLWithPath: #file).pathComponents
+      .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
+    let testInputsPath = packageRootPath + "/TestInputs"
+    let mockSDKPath : String = testInputsPath + "/mock-sdk.sdk"
+    let diagnosticEnging = DiagnosticsEngine()
+    let collector = try SDKPrebuiltModuleInputsCollector(VirtualPath(path: mockSDKPath).absolutePath!, diagnosticEnging)
+    let interfaceMap = try collector.collectSwiftInterfaceMap()
+
+    // Check interface map always contain everything
+    XCTAssertTrue(interfaceMap["Swift"]!.count == 2)
+    XCTAssertTrue(interfaceMap["A"]!.count == 2)
+    XCTAssertTrue(interfaceMap["E"]!.count == 2)
+    XCTAssertTrue(interfaceMap["F"]!.count == 2)
+    XCTAssertTrue(interfaceMap["G"]!.count == 2)
+    XCTAssertTrue(interfaceMap["H"]!.count == 2)
+
     try withTemporaryDirectory { path in
       let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
       try localFileSystem.writeFileContents(main) {
@@ -781,26 +797,13 @@ final class ExplicitModuleBuildTests: XCTestCase {
         $0 <<< "import H\n"
         $0 <<< "import Swift\n"
       }
-      let packageRootPath = URL(fileURLWithPath: #file).pathComponents
-        .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
-      let testInputsPath = packageRootPath + "/TestInputs"
-      let mockSDKPath : String = testInputsPath + "/mock-sdk.sdk"
       var driver = try Driver(args: ["swiftc", main.pathString,
                                      "-sdk", mockSDKPath,
                                     ])
 
-      let diagnosticEnging = DiagnosticsEngine()
-      let collector = try SDKPrebuiltModuleInputsCollector(VirtualPath(path: mockSDKPath).absolutePath!, diagnosticEnging)
-      let interfaceMap = try collector.collectSwiftInterfaceMap()
-      XCTAssertTrue(interfaceMap["Swift"]!.count == 2)
-      XCTAssertTrue(interfaceMap["A"]!.count == 2)
-      XCTAssertTrue(interfaceMap["E"]!.count == 2)
-      XCTAssertTrue(interfaceMap["F"]!.count == 2)
-      XCTAssertTrue(interfaceMap["G"]!.count == 2)
-      XCTAssertTrue(interfaceMap["H"]!.count == 2)
-
-      let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(interfaceMap,
-                                                                VirtualPath(path: "/tmp/").absolutePath!)
+      let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
+                                                                                into: VirtualPath(path: "/tmp/").absolutePath!,
+                                                                                exhaustive: true)
 
       XCTAssertTrue(danglingJobs.count == 2)
       XCTAssertTrue(danglingJobs.allSatisfy { job in
@@ -825,6 +828,77 @@ final class ExplicitModuleBuildTests: XCTestCase {
       checkInputOutputIntegrity(GJobs[0])
       checkInputOutputIntegrity(GJobs[1])
     }
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import H\n"
+      }
+      var driver = try Driver(args: ["swiftc", main.pathString,
+                                     "-sdk", mockSDKPath,
+                                    ])
+
+      let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
+                                                                                into: VirtualPath(path: "/tmp/").absolutePath!,
+                                                                                exhaustive: false)
+
+      XCTAssertTrue(danglingJobs.isEmpty)
+      XCTAssertTrue(jobs.count == 12)
+      XCTAssertTrue(jobs.allSatisfy {$0.outputs.count == 1})
+      XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile})
+      XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))})
+      let HJobs = jobs.filter { $0.moduleName == "H"}
+      XCTAssertTrue(HJobs.count == 2)
+      XCTAssertTrue(getInputModules(HJobs[0]) == ["A", "E", "F", "G", "Swift"])
+      XCTAssertTrue(getInputModules(HJobs[1]) == ["A", "E", "F", "G", "Swift"])
+      XCTAssertTrue(getOutputName(HJobs[0]) != getOutputName(HJobs[1]))
+      checkInputOutputIntegrity(HJobs[0])
+      checkInputOutputIntegrity(HJobs[1])
+      let GJobs = jobs.filter { $0.moduleName == "G"}
+      XCTAssertTrue(GJobs.count == 2)
+      XCTAssertTrue(getInputModules(GJobs[0]) == ["E", "Swift"])
+      XCTAssertTrue(getInputModules(GJobs[1]) == ["E", "Swift"])
+      XCTAssertTrue(getOutputName(GJobs[0]) != getOutputName(GJobs[1]))
+      checkInputOutputIntegrity(GJobs[0])
+      checkInputOutputIntegrity(GJobs[1])
+    }
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import Swift\n"
+      }
+      var driver = try Driver(args: ["swiftc", main.pathString,
+                                     "-sdk", mockSDKPath,
+                                    ])
+
+      let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
+                                                                                into: VirtualPath(path: "/tmp/").absolutePath!,
+                                                                                exhaustive: false)
+
+      XCTAssertTrue(danglingJobs.isEmpty)
+      XCTAssert(jobs.count == 2)
+      XCTAssert(jobs.allSatisfy { $0.moduleName == "Swift" })
+    }
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import F\n"
+      }
+      var driver = try Driver(args: ["swiftc", main.pathString,
+                                     "-sdk", mockSDKPath,
+                                    ])
+
+      let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
+                                                                                into: VirtualPath(path: "/tmp/").absolutePath!,
+                                                                                exhaustive: false)
+
+      XCTAssertTrue(danglingJobs.isEmpty)
+      XCTAssertTrue(jobs.count == 6)
+      jobs.forEach({ job in
+        // Check we don't pull in other modules than A, F and Swift
+        XCTAssertTrue(["A", "F", "Swift"].contains(job.moduleName))
+        checkInputOutputIntegrity(job)
+      })
+    }
   }
 #endif
 }