Improving caching build handling to support a range of compiler

Improve the logics for when to use bridging header pch command from
dependency scanner, when to use explicit module verify interface, so
swift driver can work with more swift-frontend.

Author: cachemeifyoucan

Sources/SwiftDriver/Driver/Driver.swift

@@ -74,6 +74,8 @@ public struct Driver {
     case missingContextHashOnSwiftDependency(String)
     case dependencyScanningFailure(Int, String)
     case missingExternalDependency(String)
+    // Compiler Caching Failures
+    case unsupportedConfigurationForCaching(String)
 
     public var description: String {
       switch self {
@@ -135,6 +137,8 @@ public struct Driver {
         return "unable to load output file map '\(path)': \(error)"
       case .missingExternalDependency(let moduleName):
         return "Missing External dependency info for module: \(moduleName)"
+      case .unsupportedConfigurationForCaching(let reason):
+        return "unsupported configuration for -cache-compile-job: \(reason)"
       case .baselineGenerationRequiresTopLevelModule(let arg):
         return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'"
       case .optionRequiresAnother(let first, let second):

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -71,6 +71,11 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
     self.enableCAS = enableCAS
   }
 
+  /// Supports resolving bridging header pch command from swiftScan.
+  public func supportsBridgingHeaderPCHCommand() throws -> Bool {
+    return try swiftScanOracle.supportsBridgingHeaderPCHCommand()
+  }
+
   /// Generate build jobs for all dependencies of the main module.
   /// The main module itself is handled elsewhere in the driver.
   ///
@@ -241,7 +246,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path,
                                      type: .swiftModule))
 
-      for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] {
+      let prebuiltHeaderDependencyPaths = dependencyModule.prebuiltHeaderDependencyPaths ?? []
+      if enableCAS && !prebuiltHeaderDependencyPaths.isEmpty {
+        throw Driver.Error.unsupportedConfigurationForCaching("module \(dependencyModule.moduleName) has prebuilt header dependency")
+      }
+
+      for headerDep in prebuiltHeaderDependencyPaths {
         commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"])
         commandLine.appendPath(VirtualPath.lookup(headerDep.path))
         inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch))

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -152,6 +152,13 @@ public class InterModuleDependencyOracle {
     return swiftScan.supportsCaching
   }
 
+  @_spi(Testing) public func supportsBridgingHeaderPCHCommand() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.supportsBridgingHeaderPCHCommand
+  }
+
   @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -687,6 +687,14 @@ extension Driver {
     try dependencyPlanner.resolveBridgingHeaderDependencies(inputs: &inputs, commandLine: &commandLine)
   }
 
+  /// If explicit dependency planner supports creating bridging header pch command.
+  public func supportsBridgingHeaderPCHCommand() throws -> Bool {
+    guard let dependencyPlanner = explicitDependencyBuildPlanner else {
+      return false
+    }
+    return try dependencyPlanner.supportsBridgingHeaderPCHCommand()
+  }
+
   /// In Explicit Module Build mode, distinguish between main module jobs and intermediate dependency build jobs,
   /// such as Swift modules built from .swiftmodule files and Clang PCMs.
   public func isExplicitMainModuleJob(job: Job) -> Bool {

Sources/SwiftDriver/Jobs/GeneratePCHJob.swift

@@ -30,8 +30,7 @@ extension Driver {
 
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
 
-    // If using CAS, scanner should return the full build command.
-    if enableCaching {
+    if try supportsBridgingHeaderPCHCommand() {
       try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine)
     } else {
       try addGeneratePCHFlags(commandLine: &commandLine, inputs: &inputs)

Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift

@@ -25,6 +25,12 @@ extension Driver {
                                                                     input: mainInput.fileHandle)
   }
 
+  @_spi(Testing)
+  public func supportExplicitModuleVerifyInterface() -> Bool {
+    // swift-frontend that has -input-file-key option can support explicit module build for verify interface.
+    return isFrontendArgSupported(.inputFileKey)
+  }
+
   mutating func verifyModuleInterfaceJob(interfaceInput: TypedVirtualPath, emitModuleJob: Job, optIn: Bool) throws -> Job {
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
     var inputs: [TypedVirtualPath] = [interfaceInput]
@@ -40,15 +46,12 @@ extension Driver {
       outputs.append(TypedVirtualPath(file: outputPath, type: .diagnostics))
     }
 
-    if parsedOptions.contains(.driverExplicitModuleBuild) {
+    if parsedOptions.contains(.driverExplicitModuleBuild) && supportExplicitModuleVerifyInterface() {
       commandLine.appendFlag("-explicit-interface-module-build")
       if let key = try computeCacheKeyForInterface(emitModuleJob: emitModuleJob, interfaceKind: interfaceInput.type) {
         commandLine.appendFlag("-input-file-key")
         commandLine.appendFlag(key)
       }
-      // Need to create an output file for swiftmodule output. Currently put it next to the swift interface.
-      let moduleOutPath = try interfaceInput.file.appendingToBaseName(".verified.swiftmodule")
-      commandLine.appendFlags("-o", moduleOutPath.name)
     }
 
     // TODO: remove this because we'd like module interface errors to fail the build.

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -290,6 +290,10 @@ internal extension swiftscan_diagnostic_severity_t {
            api.swiftscan_clang_detail_get_module_cache_key != nil
   }
 
+  @_spi(Testing) public var supportsBridgingHeaderPCHCommand : Bool {
+    return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil
+  }
+
   func serializeScannerCache(to path: AbsolutePath) {
     api.swiftscan_scanner_cache_serialize(scanner,
                                           path.description.cString(using: String.Encoding.utf8))

Sources/SwiftOptions/Options.swift

@@ -62,7 +62,6 @@ extension Option {
   public static let bypassBatchModeChecks: Option = Option("-bypass-batch-mode-checks", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Bypass checks for batch-mode errors.")
   public static let cacheCompileJob: Option = Option("-cache-compile-job", .flag, attributes: [.frontend], helpText: "Enable compiler caching")
   public static let cacheDisableReplay: Option = Option("-cache-disable-replay", .flag, attributes: [.frontend], helpText: "Skip loading the compilation result from cache")
-  public static let cacheRemarks: Option = Option("-cache-remarks", .flag, attributes: [.frontend], helpText: "Show remarks for compiler caching")
   public static let candidateModuleFile: Option = Option("-candidate-module-file", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<path>", helpText: "Specify Swift module may be ready to use for an interface")
   public static let casFs: Option = Option("-cas-fs", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<cas-id>", helpText: "Root CASID for CAS FileSystem")
   public static let casPath: Option = Option("-cas-path", .separate, attributes: [.frontend], metaVar: "<path>", helpText: "Path to CAS")
@@ -518,6 +517,7 @@ extension Option {
   public static let indexSystemModules: Option = Option("-index-system-modules", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Emit index data for imported serialized swift system modules")
   public static let indexUnitOutputPathFilelist: Option = Option("-index-unit-output-path-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Specify index unit output paths in a file rather than on the command line")
   public static let indexUnitOutputPath: Option = Option("-index-unit-output-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "<path>", helpText: "Use <path> as the output path in the produced index data.")
+  public static let inputFileKey: Option = Option("-input-file-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for input file")
   public static let inputPaths: Option = Option("-input-paths", .separate, attributes: [.noDriver, .argumentIsPath], metaVar: "<path>", helpText: "The SDK contents under comparison")
   public static let inputPaths_: Option = Option("--input-paths", .separate, alias: Option.inputPaths, attributes: [.noDriver, .argumentIsPath], metaVar: "<path>", helpText: "The SDK contents under comparison")
   public static let internalizeAtLink: Option = Option("-internalize-at-link", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Allow internalizing public symbols and vtables at link time (assume all client code of public types is part of the same link unit, or that external symbols are explicitly requested via -exported_symbols_list)")
@@ -638,6 +638,7 @@ extension Option {
   public static let publicAutolinkLibrary: Option = Option("-public-autolink-library", .separate, attributes: [.frontend, .noDriver], helpText: "Add public dependent library")
   public static let RaccessNoteEQ: Option = Option("-Raccess-note=", .joined, alias: Option.RaccessNote, attributes: [.frontend, .noDriver])
   public static let RaccessNote: Option = Option("-Raccess-note", .separate, attributes: [.frontend, .noDriver], metaVar: "none|failures|all|all-validate", helpText: "Control access note remarks (default: all)")
+  public static let cacheRemarks: Option = Option("-Rcache-compile-job", .flag, attributes: [.frontend], helpText: "Show remarks for compiler caching")
   public static let emitCrossImportRemarks: Option = Option("-Rcross-import", .flag, attributes: [.frontend, .doesNotAffectIncrementalBuild], helpText: "Emit a remark if a cross-import of a module is triggered.")
   public static let dependencyScanCacheRemarks: Option = Option("-Rdependency-scan-cache", .flag, attributes: [.frontend, .noDriver], helpText: "Emit remarks indicating use of the serialized module dependency scanning cache.")
   public static let readLegacyTypeInfoPathEQ: Option = Option("-read-legacy-type-info-path=", .joined, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Read legacy type layout from the given path instead of default path")
@@ -854,7 +855,6 @@ extension Option {
       Option.bypassBatchModeChecks,
       Option.cacheCompileJob,
       Option.cacheDisableReplay,
-      Option.cacheRemarks,
       Option.candidateModuleFile,
       Option.casFs,
       Option.casPath,
@@ -1310,6 +1310,7 @@ extension Option {
       Option.indexSystemModules,
       Option.indexUnitOutputPathFilelist,
       Option.indexUnitOutputPath,
+      Option.inputFileKey,
       Option.inputPaths,
       Option.inputPaths_,
       Option.internalizeAtLink,
@@ -1430,6 +1431,7 @@ extension Option {
       Option.publicAutolinkLibrary,
       Option.RaccessNoteEQ,
       Option.RaccessNote,
+      Option.cacheRemarks,
       Option.emitCrossImportRemarks,
       Option.dependencyScanCacheRemarks,
       Option.readLegacyTypeInfoPathEQ,

Tests/SwiftDriverTests/CachingBuildTests.swift

@@ -302,6 +302,21 @@ final class CachingBuildTests: XCTestCase {
                                      "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
                                      main.nativePathString(escaped: true)] + sdkArgumentsForTesting)
 
+      guard driver.supportExplicitModuleVerifyInterface() else {
+        throw XCTSkip("-typecheck-module-from-interface doesn't support explicit build.")
+      }
+      let dependencyOracle = InterModuleDependencyOracle()
+      let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
+      guard try dependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+      guard try dependencyOracle.supportsCaching() else {
+        throw XCTSkip("libSwiftScan does not support caching.")
+      }
+
       let jobs = try driver.planBuild()
       // Figure out which Triples to use.
       let dependencyGraph = try driver.gatherModuleDependencies()
@@ -425,7 +440,7 @@ final class CachingBuildTests: XCTestCase {
       var driver = try Driver(args: ["swiftc",
                                      "-I", cHeadersPath.nativePathString(escaped: true),
                                      "-I", swiftModuleInterfacesPath.nativePathString(escaped: true),
-                                     "-explicit-module-build", "-v", "-cache-remarks",
+                                     "-explicit-module-build", "-v", "-Rcache-compile-job",
                                      "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
                                      "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
                                      "-working-directory", path.nativePathString(escaped: true),
@@ -448,6 +463,82 @@ final class CachingBuildTests: XCTestCase {
     }
   }
 
+  func testCacheBuildEndToEndWithBinaryHeaderDeps() throws {
+    try withTemporaryDirectory { path in
+      try localFileSystem.changeCurrentWorkingDirectory(to: path)
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let PCHPath = path.appending(component: "PCH")
+      try localFileSystem.createDirectory(PCHPath)
+      let FooInstallPath = path.appending(component: "Foo")
+      try localFileSystem.createDirectory(FooInstallPath)
+      let foo = path.appending(component: "foo.swift")
+      let casPath = path.appending(component: "cas")
+      try localFileSystem.writeFileContents(foo) {
+        $0 <<< "extension Profiler {"
+        $0 <<< "    public static let count: Int = 42"
+        $0 <<< "}"
+      }
+      let fooHeader = path.appending(component: "foo.h")
+      try localFileSystem.writeFileContents(fooHeader) {
+        $0 <<< "struct Profiler { void* ptr; };"
+      }
+      let main = path.appending(component: "main.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import Foo"
+      }
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+
+      var fooBuildDriver = try Driver(args: ["swiftc",
+                                             "-explicit-module-build",
+                                             "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                             "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
+                                             "-working-directory", path.nativePathString(escaped: true),
+                                             foo.nativePathString(escaped: true),
+                                             "-emit-module", "-wmo", "-module-name", "Foo",
+                                             "-emit-module-path", FooInstallPath.nativePathString(escaped: true),
+                                             "-import-objc-header", fooHeader.nativePathString(escaped: true),
+                                             "-pch-output-dir", PCHPath.nativePathString(escaped: true),
+                                             FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)]
+                                      + sdkArgumentsForTesting,
+                                      env: ProcessEnv.vars)
+
+      // Ensure this tooling supports this functionality
+      let dependencyOracle = InterModuleDependencyOracle()
+      let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib())
+      guard try dependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+      guard try dependencyOracle.supportsBinaryModuleHeaderDependencies() else {
+        throw XCTSkip("libSwiftScan does not support binary module header dependencies.")
+      }
+      guard try dependencyOracle.supportsCaching() else {
+        throw XCTSkip("libSwiftScan does not support caching.")
+      }
+
+      let fooJobs = try fooBuildDriver.planBuild()
+      try fooBuildDriver.run(jobs: fooJobs)
+      XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors)
+
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", FooInstallPath.nativePathString(escaped: true),
+                                     "-explicit-module-build", "-emit-module", "-emit-module-path",
+                                     path.appending(component: "testEMBETEWBHD.swiftmodule").nativePathString(escaped: true),
+                                     "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                     "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              env: ProcessEnv.vars)
+      // This is currently not supported.
+      XCTAssertThrowsError(try driver.planBuild()) {
+        XCTAssertEqual($0 as? Driver.Error, .unsupportedConfigurationForCaching("module Foo has prebuilt header dependency"))
+      }
+    }
+  }
+
   func testDependencyScanning() throws {
     // Create a simple test case.
     try withTemporaryDirectory { path in

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -405,6 +405,9 @@ final class ExplicitModuleBuildTests: XCTestCase {
                                      "-enable-library-evolution",
                                      main.nativePathString(escaped: true)] + sdkArgumentsForTesting)
 
+      guard driver.supportExplicitModuleVerifyInterface() else {
+        throw XCTSkip("-typecheck-module-from-interface doesn't support explicit build.")
+      }
       let jobs = try driver.planBuild()
       // Figure out which Triples to use.
       let dependencyGraph = try driver.gatherModuleDependencies()