[Caching] Update swift-driver to reflect the frontend caching configuration

Re-unify caching availability check behind swift feature query and
adjust the default caching build configuration changes.

Author: cachemeifyoucan

Sources/SwiftDriver/Driver/Driver.swift

@@ -274,7 +274,13 @@ public struct Driver {
   let enableCaching: Bool
   let useClangIncludeTree: Bool
 
-  var cas: SwiftScanCAS?
+  /// CAS instance used for compilation.
+  public var cas: SwiftScanCAS? = nil
+
+  /// Is swift caching enabled.
+  lazy var isCachingEnabled: Bool = {
+    return enableCaching && isFeatureSupported(.cache_compile_job)
+  }()
 
   /// Scanner prefix mapping.
   let scannerPrefixMap: [AbsolutePath: AbsolutePath]
@@ -418,6 +424,7 @@ public struct Driver {
   @_spi(Testing)
   public enum KnownCompilerFeature: String {
     case emit_abi_descriptor = "emit-abi-descriptor"
+    case cache_compile_job = "cache-compile-job"
   }
 
   lazy var sdkPath: VirtualPath? = {
@@ -477,14 +484,6 @@ public struct Driver {
     return supportedFrontendFeatures.contains(feature.rawValue)
   }
 
-  @_spi(Testing)
-  public func getCAS() throws -> SwiftScanCAS {
-    guard let cas = self.cas else {
-      throw DependencyScanningError.casError("CAS is not initialized but requested")
-    }
-    return cas
-  }
-
   @_spi(Testing)
   public static func findBlocklists(RelativeTo execDir: AbsolutePath) throws ->  [AbsolutePath] {
     // Expect to find all blocklists in such dir:
@@ -638,7 +637,7 @@ public struct Driver {
 
     let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
     self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
-    self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
+    self.useClangIncludeTree = !parsedOptions.hasArgument(.noClangIncludeTree) && !env.keys.contains("SWIFT_CACHING_USE_CLANG_CAS_FS")
     self.scannerPrefixMap = try Self.computeScanningPrefixMapper(&parsedOptions)
     if let sdkMapping =  parsedOptions.getLastArgument(.scannerPrefixMapSdk)?.asSingle {
       self.scannerPrefixMapSDK = try AbsolutePath(validating: sdkMapping)

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -146,20 +146,6 @@ public class InterModuleDependencyOracle {
     return swiftScan.supportsBinaryModuleHeaderDependencies
   }
 
-  @_spi(Testing) public func supportsCaching() throws -> Bool {
-    guard let swiftScan = swiftScanLibInstance else {
-      fatalError("Attempting to query supported scanner API with no scanner instance.")
-    }
-    return swiftScan.supportsCaching
-  }
-
-  @_spi(Testing) public func supportsCacheReplay() throws -> Bool {
-    guard let swiftScan = swiftScanLibInstance else {
-      fatalError("Attempting to query supported scanner API with no scanner instance.")
-    }
-    return swiftScan.supportsCacheReplay
-  }
-
   @_spi(Testing) public func supportsBridgingHeaderPCHCommand() throws -> Bool {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to query supported scanner API with no scanner instance.")

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -119,7 +119,6 @@ public extension Driver {
       commandLine.appendPath(dependencyPlaceholderMapFile)
     }
 
-    try commandLine.appendLast(.clangIncludeTree, from: &parsedOptions)
     if isFrontendArgSupported(.clangScannerModuleCachePath) {
       try commandLine.appendLast(.clangScannerModuleCachePath, from: &parsedOptions)
     }
@@ -173,7 +172,7 @@ public extension Driver {
         diagnosticEngine.emit(.warn_scanner_frontend_fallback())
       }
     }
-    if !fallbackToFrontend && enableCaching {
+    if !fallbackToFrontend && isCachingEnabled {
       self.cas = try interModuleDependencyOracle.createCAS(pluginPath: try getCASPluginPath(),
                                                            onDiskPath: try getOnDiskCASPath(),
                                                            pluginOptions: try getCASPluginOptions())

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -318,7 +318,7 @@ extension Driver {
       try commandLine.appendLast(.enableBuiltinModule, from: &parsedOptions)
     }
 
-    if !useClangIncludeTree, let workingDirectory = workingDirectory {
+    if !(isCachingEnabled && useClangIncludeTree), let workingDirectory = workingDirectory {
       // Add -Xcc -working-directory before any other -Xcc options to ensure it is
       // overridden by an explicit -Xcc -working-directory, although having a
       // different working directory is probably incorrect.
@@ -359,7 +359,7 @@ extension Driver {
     }
 
     // CAS related options.
-    if enableCaching {
+    if isCachingEnabled {
       commandLine.appendFlag(.cacheCompileJob)
       if let casPath = try getOnDiskCASPath() {
         commandLine.appendFlag(.casPath)
@@ -371,16 +371,16 @@ extension Driver {
       }
       try commandLine.appendAll(.casPluginOption, from: &parsedOptions)
       try commandLine.appendLast(.cacheRemarks, from: &parsedOptions)
+      if !useClangIncludeTree {
+        commandLine.appendFlag(.noClangIncludeTree)
+      }
     }
     addCacheReplayMapping(to: &commandLine)
-    if useClangIncludeTree {
-      commandLine.appendFlag(.clangIncludeTree)
-    }
 
     // Pass through any subsystem flags.
     try commandLine.appendAll(.Xllvm, from: &parsedOptions)
 
-    if !useClangIncludeTree {
+    if !(isCachingEnabled && useClangIncludeTree) {
       try commandLine.appendAll(.Xcc, from: &parsedOptions)
     }
 
@@ -446,7 +446,7 @@ extension Driver {
 
   mutating func addBridgingHeaderPCHCacheKeyArguments(commandLine: inout [Job.ArgTemplate],
                                                       pchCompileJob: Job?) throws {
-    guard let pchJob = pchCompileJob, enableCaching else { return }
+    guard let pchJob = pchCompileJob, isCachingEnabled else { return }
 
     // The pch input file (the bridging header) is added as last inputs to the job.
     guard let inputFile = pchJob.inputs.last else { assertionFailure("no input files from pch job"); return }
@@ -701,7 +701,7 @@ extension Driver {
     } else {
       entryInput = inputFiles[0].fileHandle
     }
-    let inputEntry = enableCaching ? remapPath(VirtualPath.lookup(entryInput)).intern() : entryInput
+    let inputEntry = isCachingEnabled ? remapPath(VirtualPath.lookup(entryInput)).intern() : entryInput
     entries[inputEntry, default: [:]][output.type] = output.fileHandle
   }
 
@@ -771,7 +771,7 @@ extension Driver {
 
   /// Helper function to add path to commandLine. Function will validate the path, and remap the path if needed.
   public mutating func addPathArgument(_ path: VirtualPath, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws {
-    guard remap && enableCaching else {
+    guard remap && isCachingEnabled else {
       commandLine.appendPath(path)
       return
     }
@@ -813,7 +813,7 @@ extension Driver {
   }
 
   public mutating func addCacheReplayMapping(to commandLine: inout [Job.ArgTemplate]) {
-    if enableCaching && isFrontendArgSupported(.scannerPrefixMap) {
+    if isCachingEnabled && isFrontendArgSupported(.scannerPrefixMap) {
       for (key, value) in prefixMapping {
         commandLine.appendFlag("-cache-replay-prefix-map")
         commandLine.appendFlag(value.pathString + "=" + key.pathString)

Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift

@@ -16,7 +16,7 @@ extension Driver {
     assert(interfaceKind == .swiftInterface || interfaceKind == .privateSwiftInterface,
            "only expect interface output kind")
     let isNeeded = emitModuleJob.outputs.contains { $0.type == interfaceKind }
-    guard enableCaching && isNeeded else { return nil }
+    guard isCachingEnabled && isNeeded else { return nil }
 
     // Assume swiftinterface file is always the supplementary output for first input file.
     let key =  try computeOutputCacheKey(commandLine: emitModuleJob.commandLine,

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -294,36 +294,6 @@ internal extension swiftscan_diagnostic_severity_t {
 #endif
   }
 
-  @_spi(Testing) public var supportsCacheReplay : Bool {
-#if os(Windows)
-    // Caching is currently not supported on Windows hosts.
-    return false
-#else
-    return supportsCaching &&
-           api.swiftscan_cache_query != nil &&
-           api.swiftscan_cache_query_async != nil &&
-           api.swiftscan_cache_action_cancel != nil &&
-           api.swiftscan_cache_cancellation_token_dispose != nil &&
-           api.swiftscan_cached_compilation_get_num_outputs != nil &&
-           api.swiftscan_cached_compilation_get_output != nil &&
-           api.swiftscan_cached_compilation_make_global_async != nil &&
-           api.swiftscan_cached_compilation_is_uncacheable != nil &&
-           api.swiftscan_cached_compilation_dispose != nil &&
-           api.swiftscan_cached_output_load != nil &&
-           api.swiftscan_cached_output_load_async != nil &&
-           api.swiftscan_cached_output_is_materialized != nil &&
-           api.swiftscan_cached_output_get_casid != nil &&
-           api.swiftscan_cached_output_get_name != nil &&
-           api.swiftscan_cached_output_dispose != nil &&
-           api.swiftscan_cache_replay_instance_create != nil &&
-           api.swiftscan_cache_replay_instance_dispose != nil &&
-           api.swiftscan_cache_replay_result_get_stdout != nil &&
-           api.swiftscan_cache_replay_result_get_stderr != nil &&
-           api.swiftscan_cache_replay_result_dispose != nil &&
-           api.swiftscan_cache_replay_compilation != nil
-#endif
-  }
-
   @_spi(Testing) public var supportsBridgingHeaderPCHCommand : Bool {
     return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil
   }

Sources/SwiftOptions/Options.swift

@@ -33,6 +33,7 @@ extension Option {
   public static let analyzeRequirementMachine: Option = Option("-analyze-requirement-machine", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Print out requirement machine statistics at the end of the compilation job")
   public static let apiDiffDataDir: Option = Option("-api-diff-data-dir", .separate, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .argumentIsPath], metaVar: "<path>", helpText: "Load platform and version specific API migration data files from <path>. Ignored if -api-diff-data-file is specified.")
   public static let apiDiffDataFile: Option = Option("-api-diff-data-file", .separate, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .argumentIsPath], metaVar: "<path>", helpText: "API migration data is from <path>")
+  public static let enableAppExtensionLibrary: Option = Option("-application-extension-library", .flag, attributes: [.frontend, .noInteractive], helpText: "Restrict code to those available for App Extension Libraries")
   public static let enableAppExtension: Option = Option("-application-extension", .flag, attributes: [.frontend, .noInteractive], helpText: "Restrict code to those available for App Extensions")
   public static let AssertConfig: Option = Option("-assert-config", .separate, attributes: [.frontend, .moduleInterface], helpText: "Specify the assert_configuration replacement. Possible values are Debug, Release, Unchecked, DisableReplacement.")
   public static let AssumeSingleThreaded: Option = Option("-assume-single-threaded", .flag, attributes: [.helpHidden, .frontend], helpText: "Assume that code will be executed in a single-threaded environment")
@@ -79,7 +80,6 @@ extension Option {
   public static let clangHeaderExposeDecls: Option = Option("-clang-header-expose-decls=", .joined, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "all-public|has-expose-attr", helpText: "Which declarations should be exposed in the generated clang header.")
   public static let clangHeaderExposeModule: Option = Option("-clang-header-expose-module", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<imported-module-name>=<generated-header-name>", helpText: "Allow the compiler to assume that APIs from the specified module are exposed to C/C++/Objective-C in another generated header, so that APIs in the current module that depend on declarations from the specified module can be exposed in the generated header.")
   public static let clangIncludeTreeRoot: Option = Option("-clang-include-tree-root", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<cas-id>", helpText: "Clang Include Tree CASID")
-  public static let clangIncludeTree: Option = Option("-clang-include-tree", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Use clang include tree")
   public static let clangScannerModuleCachePath: Option = Option("-clang-scanner-module-cache-path", .separate, attributes: [.frontend, .doesNotAffectIncrementalBuild, .argumentIsPath], helpText: "Specifies the Clang dependency scanner module cache path")
   public static let clangTarget: Option = Option("-clang-target", .separate, attributes: [.frontend], helpText: "Separately set the target we should use for internal Clang instance")
   public static let codeCompleteCallPatternHeuristics: Option = Option("-code-complete-call-pattern-heuristics", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Use heuristics to guess whether we want call pattern completions")
@@ -114,7 +114,6 @@ extension Option {
   public static let debugForbidTypecheckPrefix: Option = Option("-debug-forbid-typecheck-prefix", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Triggers llvm fatal_error if typechecker tries to typecheck a decl with the provided prefix name")
   public static let debugGenericSignatures: Option = Option("-debug-generic-signatures", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Debug generic signatures")
   public static let debugInfoFormat: Option = Option("-debug-info-format=", .joined, attributes: [.frontend], helpText: "Specify the debug info format type to either 'dwarf' or 'codeview'")
-  public static let dwarfVersion: Option = Option("-dwarf-version=", .joined, attributes: [.frontend], helpText: "DWARF debug info version to produce if requested")
   public static let debugInfoStoreInvocation: Option = Option("-debug-info-store-invocation", .flag, attributes: [.frontend], helpText: "Emit the compiler invocation in the debug info.")
   public static let debugMapping: Option = Option("-debug-mapping", .flag, attributes: [.noDriver], helpText: "Dumping information for debug purposes")
   public static let debugMapping_: Option = Option("--debug-mapping", .flag, alias: Option.debugMapping, attributes: [.noDriver], helpText: "Dumping information for debug purposes")
@@ -278,6 +277,7 @@ extension Option {
   public static let dumpTypeRefinementContexts: Option = Option("-dump-type-refinement-contexts", .flag, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild], helpText: "Type-check input file(s) and dump type refinement contexts(s)", group: .modes)
   public static let dumpTypeWitnessSystems: Option = Option("-dump-type-witness-systems", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Enables dumping type witness systems from associated type inference")
   public static let dumpUsr: Option = Option("-dump-usr", .flag, attributes: [.frontend, .noInteractive], helpText: "Dump USR for each declaration reference")
+  public static let dwarfVersion: Option = Option("-dwarf-version=", .joined, attributes: [.frontend], metaVar: "<version>", helpText: "DWARF debug info version to produce if requested")
   public static let D: Option = Option("-D", .joinedOrSeparate, attributes: [.frontend], helpText: "Marks a conditional compilation flag as true")
   public static let embedBitcodeMarker: Option = Option("-embed-bitcode-marker", .flag, attributes: [.frontend, .noInteractive], helpText: "Embed placeholder LLVM IR data as a marker")
   public static let embedBitcode: Option = Option("-embed-bitcode", .flag, attributes: [.frontend, .noInteractive], helpText: "Embed LLVM IR bitcode as data")
@@ -584,6 +584,7 @@ extension Option {
   public static let module: Option = Option("-module", .separate, attributes: [.noDriver], metaVar: "<name>", helpText: "Names of modules")
   public static let module_: Option = Option("--module", .separate, alias: Option.module, attributes: [.noDriver], metaVar: "<name>", helpText: "Names of modules")
   public static let newDriverPath: Option = Option("-new-driver-path", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<path>", helpText: "Path of the new driver to be used")
+  public static let noClangIncludeTree: Option = Option("-no-clang-include-tree", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Do not use clang include tree, fallback to use CAS filesystem to build clang modules")
   public static let noClangModuleBreadcrumbs: Option = Option("-no-clang-module-breadcrumbs", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Don't emit DWARF skeleton CUs for imported Clang modules. Use this when building a redistributable static archive.")
   public static let noColorDiagnostics: Option = Option("-no-color-diagnostics", .flag, attributes: [.frontend, .doesNotAffectIncrementalBuild], helpText: "Do not print diagnostics in color")
   public static let noEmitModuleSeparatelyWMO: Option = Option("-no-emit-module-separately-wmo", .flag, attributes: [.helpHidden], helpText: "Force emitting the swiftmodule in the same job in wmo builds")
@@ -615,7 +616,7 @@ extension Option {
   public static let O: Option = Option("-O", .flag, attributes: [.frontend, .moduleInterface], helpText: "Compile with optimizations", group: .O)
   public static let o: Option = Option("-o", .joinedOrSeparate, attributes: [.frontend, .noInteractive, .autolinkExtract, .moduleWrap, .indent, .argumentIsPath, .cacheInvariant], metaVar: "<file>", helpText: "Write output to <file>")
   public static let packageDescriptionVersion: Option = Option("-package-description-version", .separate, attributes: [.helpHidden, .frontend, .moduleInterface], metaVar: "<vers>", helpText: "The version number to be applied on the input for the PackageDescription availability kind")
-  public static let packageName: Option = Option("-package-name", .separate, attributes: [.frontend], helpText: "Name of the package the module belongs to")
+  public static let packageName: Option = Option("-package-name", .separate, attributes: [.frontend, .moduleInterface], helpText: "Name of the package the module belongs to")
   public static let parallelScan: Option = Option("-parallel-scan", .flag, attributes: [.frontend, .noDriver], helpText: "Perform dependency scanning in-parallel.")
   public static let parseAsLibrary: Option = Option("-parse-as-library", .flag, attributes: [.frontend, .noInteractive], helpText: "Parse the input file(s) as libraries, not scripts")
   public static let parseSil: Option = Option("-parse-sil", .flag, attributes: [.frontend, .noInteractive], helpText: "Parse the input file as SIL code, not Swift source")
@@ -854,6 +855,7 @@ extension Option {
       Option.analyzeRequirementMachine,
       Option.apiDiffDataDir,
       Option.apiDiffDataFile,
+      Option.enableAppExtensionLibrary,
       Option.enableAppExtension,
       Option.AssertConfig,
       Option.AssumeSingleThreaded,
@@ -900,7 +902,6 @@ extension Option {
       Option.clangHeaderExposeDecls,
       Option.clangHeaderExposeModule,
       Option.clangIncludeTreeRoot,
-      Option.clangIncludeTree,
       Option.clangScannerModuleCachePath,
       Option.clangTarget,
       Option.codeCompleteCallPatternHeuristics,
@@ -935,7 +936,6 @@ extension Option {
       Option.debugForbidTypecheckPrefix,
       Option.debugGenericSignatures,
       Option.debugInfoFormat,
-      Option.dwarfVersion,
       Option.debugInfoStoreInvocation,
       Option.debugMapping,
       Option.debugMapping_,
@@ -1099,6 +1099,7 @@ extension Option {
       Option.dumpTypeRefinementContexts,
       Option.dumpTypeWitnessSystems,
       Option.dumpUsr,
+      Option.dwarfVersion,
       Option.D,
       Option.embedBitcodeMarker,
       Option.embedBitcode,
@@ -1405,6 +1406,7 @@ extension Option {
       Option.module,
       Option.module_,
       Option.newDriverPath,
+      Option.noClangIncludeTree,
       Option.noClangModuleBreadcrumbs,
       Option.noColorDiagnostics,
       Option.noEmitModuleSeparatelyWMO,