Merge pull request #597 from nkcsgexi/verbose

PrebuiltModuleGen: Allow arm64 interfaces to depend on arm64e interfaces

Author: nkcsgexi

Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift

@@ -12,6 +12,69 @@
 import TSCBasic
 import SwiftOptions
 
+public class PrebuitModuleGenerationDelegate: JobExecutionDelegate {
+  var failingModules = Set<String>()
+  var commandMap: [Int: String] = [:]
+  let diagnosticsEngine: DiagnosticsEngine
+  let verbose: Bool
+  public init(_ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool) {
+    self.diagnosticsEngine = diagnosticsEngine
+    self.verbose = verbose
+  }
+
+  func printJobInfo(_ job: Job, _ start: Bool) {
+    guard verbose else {
+      return
+    }
+    for arg in job.commandLine {
+      if case .path(let p) = arg {
+        if p.extension == "swiftinterface" {
+          Driver.stdErrQueue.sync {
+            stderrStream <<< (start ? "started: " : "finished: ")
+            stderrStream <<< p.absolutePath!.pathString <<< "\n"
+            stderrStream.flush()
+          }
+          return
+        }
+      }
+    }
+  }
+
+  public func jobStarted(job: Job, arguments: [String], pid: Int) {
+    commandMap[pid] = arguments.reduce("") { return $0 + " " + $1 }
+    printJobInfo(job, true)
+  }
+
+  public var hasStdlibFailure: Bool {
+    return failingModules.contains("Swift") || failingModules.contains("_Concurrency")
+  }
+
+  public func jobFinished(job: Job, result: ProcessResult, pid: Int) {
+    switch result.exitStatus {
+    case .terminated(code: let code):
+      if code == 0 {
+        printJobInfo(job, false)
+      } else {
+        failingModules.insert(job.moduleName)
+        let result: String = try! result.utf8stderrOutput()
+        Driver.stdErrQueue.sync {
+          stderrStream <<< "failed: " <<< commandMap[pid]! <<< "\n"
+          stderrStream <<< result <<< "\n"
+          stderrStream.flush()
+        }
+      }
+#if !os(Windows)
+    case .signalled:
+      diagnosticsEngine.emit(.remark("\(job.moduleName) interrupted"))
+#endif
+    }
+  }
+
+  public func jobSkipped(job: Job) {
+    diagnosticsEngine.emit(.error("\(job.moduleName) skipped"))
+  }
+}
+
 public struct PrebuiltModuleInput {
   // The path to the input/output of the a module building task.
   let path: TypedVirtualPath
@@ -245,14 +308,24 @@ extension Driver {
       return collectSwiftModuleNames(dependencies)
     }
 
-    func getOutputPaths(for modules: [String], with arch: Triple.Arch) throws -> [TypedVirtualPath] {
+    func getOutputPaths(withName modules: [String], loadableFor arch: Triple.Arch) throws -> [TypedVirtualPath] {
       var results: [TypedVirtualPath] = []
       modules.forEach { module in
         guard let allOutputs = outputMap[module] else {
           diagnosticEngine.emit(error: "cannot find output paths for \(module)")
           return
         }
-        let allPaths = allOutputs.filter { $0.arch == arch }.map { $0.path }
+        let allPaths = allOutputs.filter { output in
+          if output.arch == arch {
+            return true
+          }
+          // arm64e interfaces can be loded from an arm64 interface but not vice
+          // versa.
+          if arch == .aarch64 && output.arch == .aarch64e {
+            return true
+          }
+          return false
+        }.map { $0.path }
         results.append(contentsOf: allPaths)
       }
       return results
@@ -283,7 +356,7 @@ extension Driver {
         try forEachInputOutputPair(module) { input, output in
           jobs.append(try generateSingleModuleBuildingJob(module,
             prebuiltModuleDir, input, output,
-            try getOutputPaths(for: dependencies, with: input.arch)))
+            try getOutputPaths(withName: dependencies, loadableFor: input.arch)))
         }
         // For each dependency, add to the list to handle if the list doesn't
         // contain this dependency.

Sources/swift-build-sdk-interfaces/main.swift

@@ -38,38 +38,9 @@ if rawOutputDir.isEmpty {
 /// they are Foundation and anything below.
 let coreMode = CommandLine.arguments.contains("-core")
 
-class PrebuitGenDelegate: JobExecutionDelegate {
-  var failingModules = Set<String>()
-  var commandMap: [Int: String] = [:]
-  func jobStarted(job: Job, arguments: [String], pid: Int) {
-    commandMap[pid] = arguments.reduce("") { return $0 + " " + $1 }
-  }
-
-  var hasStdlibFailure: Bool {
-    return failingModules.contains("Swift") || failingModules.contains("_Concurrency")
-  }
+/// Verbose to print more info
+let verbose = CommandLine.arguments.contains("-v")
 
-  func jobFinished(job: Job, result: ProcessResult, pid: Int) {
-    switch result.exitStatus {
-    case .terminated(code: let code):
-      if code != 0 {
-        failingModules.insert(job.moduleName)
-        Driver.stdErrQueue.sync {
-          stderrStream <<< "failed: " <<< commandMap[pid]! <<< "\n"
-          stderrStream.flush()
-        }
-      }
-#if !os(Windows)
-    case .signalled:
-      diagnosticsEngine.emit(.remark("\(job.moduleName) interrupted"))
-#endif
-    }
-  }
-
-  func jobSkipped(job: Job) {
-    diagnosticsEngine.emit(.error("\(job.moduleName) skipped"))
-  }
-}
 do {
   let sdkPath = try VirtualPath(path: sdkPathRaw).absolutePath!
   if !localFileSystem.exists(sdkPath) {
@@ -136,7 +107,7 @@ do {
                             executor: executor,
                             compilerExecutableDir: swiftcPath.parentDirectory)
     let (jobs, danglingJobs) = try driver.generatePrebuitModuleGenerationJobs(with: inputMap, into: outputDir, exhaustive: !coreMode)
-    let delegate = PrebuitGenDelegate()
+    let delegate = PrebuitModuleGenerationDelegate(diagnosticsEngine, verbose)
     do {
       try executor.execute(workload: DriverExecutorWorkload.init(jobs, nil, continueBuildingAfterErrors: true),
                            delegate: delegate, numParallelJobs: 128)

TestInputs/mock-sdk.sdk/usr/lib/swift/F.swiftmodule/arm64-apple-macos.swiftinterface

@@ -0,0 +1,6 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name F
+import Swift
+import A
+public func FuncF() { }
+

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -766,11 +766,26 @@ final class ExplicitModuleBuildTests: XCTestCase {
     }
 
     func checkInputOutputIntegrity(_ job: Job) {
-      let name = job.outputs[0].file.basename
+      let name = job.outputs[0].file.basenameWithoutExt
+      XCTAssertTrue(job.outputs[0].file.extension == "swiftmodule")
       job.inputs.forEach { input in
-        XCTAssertTrue(input.file.basename == name)
+        XCTAssertTrue(input.file.extension == "swiftmodule")
+        let inputName = input.file.basenameWithoutExt
+        // arm64 interface can depend on ar64e interface
+        if inputName.starts(with: "arm64e-") && name.starts(with: "arm64-") {
+          return
+        }
+        XCTAssertTrue(inputName == name)
+      }
+    }
+
+    func findJob(_ jobs: [Job],_ module: String, _ basenameWithoutExt: String) -> Job? {
+      return jobs.first { job in
+        return job.moduleName == module &&
+          job.outputs[0].file.basenameWithoutExt == basenameWithoutExt
       }
     }
+
     let packageRootPath = URL(fileURLWithPath: #file).pathComponents
       .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
     let testInputsPath = packageRootPath + "/TestInputs"
@@ -783,7 +798,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     XCTAssertTrue(interfaceMap["Swift"]!.count == 2)
     XCTAssertTrue(interfaceMap["A"]!.count == 2)
     XCTAssertTrue(interfaceMap["E"]!.count == 2)
-    XCTAssertTrue(interfaceMap["F"]!.count == 2)
+    XCTAssertTrue(interfaceMap["F"]!.count == 3)
     XCTAssertTrue(interfaceMap["G"]!.count == 2)
     XCTAssertTrue(interfaceMap["H"]!.count == 2)
 
@@ -809,7 +824,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
       XCTAssertTrue(danglingJobs.allSatisfy { job in
         job.moduleName == "MissingKit"
       })
-      XCTAssertTrue(jobs.count == 12)
+      XCTAssertTrue(jobs.count == 13)
       XCTAssertTrue(jobs.allSatisfy {$0.outputs.count == 1})
       XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile})
       XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))})
@@ -842,7 +857,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
                                                                                 exhaustive: false)
 
       XCTAssertTrue(danglingJobs.isEmpty)
-      XCTAssertTrue(jobs.count == 12)
+      XCTAssertTrue(jobs.count == 13)
       XCTAssertTrue(jobs.allSatisfy {$0.outputs.count == 1})
       XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile})
       XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))})
@@ -892,13 +907,36 @@ final class ExplicitModuleBuildTests: XCTestCase {
                                                                                 exhaustive: false)
 
       XCTAssertTrue(danglingJobs.isEmpty)
-      XCTAssertTrue(jobs.count == 6)
+      XCTAssertTrue(jobs.count == 7)
       jobs.forEach({ job in
         // Check we don't pull in other modules than A, F and Swift
         XCTAssertTrue(["A", "F", "Swift"].contains(job.moduleName))
         checkInputOutputIntegrity(job)
       })
     }
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import H\n"
+      }
+      var driver = try Driver(args: ["swiftc", main.pathString,
+                                     "-sdk", mockSDKPath,
+                                    ])
+
+      let (jobs, _) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
+                                                                                into: VirtualPath(path: "/tmp/").absolutePath!,
+                                                                                exhaustive: false)
+      let F = findJob(jobs, "F", "arm64-apple-macos")!
+      let H = findJob(jobs, "H", "arm64e-apple-macos")!
+      // Test arm64 interface requires arm64e interfaces as inputs
+      XCTAssertTrue(F.inputs.contains { input in
+        input.file.basenameWithoutExt == "arm64e-apple-macos"
+      })
+      // Test arm64e interface doesn't require arm64 interfaces as inputs
+      XCTAssertTrue(!H.inputs.contains { input in
+        input.file.basenameWithoutExt == "arm64-apple-macos"
+      })
+    }
   }
 #endif
 }