[BridgingHeader] Auto bridging header chaining

Add ability to automatically chaining the bridging headers discovered from all
dependencies module when doing swift caching build. This will eliminate all
implicit bridging header imports from the build and make the bridging header
importing behavior much more reliable, while keep the compatibility at maximum.

For example, if the current module A depends on module B and C, and both B and
C are binary modules that uses bridging header, when building module A,
dependency scanner will construct a new header that chains three bridging
headers together with the option to build a PCH from it. This will make all
importing errors more obvious while improving the performance.

This is currently only enabled for swift caching since the new chained header
is written into CAS directly without the need to serialization.

Author: cachemeifyoucan

include/swift/Basic/LangOptions.h

@@ -201,7 +201,7 @@ namespace swift {
     /// Maximum number of typo corrections we are allowed to perform.
     /// This is disabled by default until we can get typo-correction working within acceptable performance bounds.
     unsigned TypoCorrectionLimit = 0;
-    
+
     /// Should access control be respected?
     bool EnableAccessControl = true;
 
@@ -260,7 +260,7 @@ namespace swift {
 
     /// Emit a remark when indexing a system module.
     bool EnableIndexingSystemModuleRemarks = false;
-    
+
     /// Emit a remark on early exit in explicit interface build
     bool EnableSkipExplicitInterfaceModuleBuildRemarks = false;
 
@@ -691,7 +691,7 @@ namespace swift {
     void clearAllPlatformConditionValues() {
       PlatformConditionValues.clear();
     }
-    
+
     /// Returns the value for the given platform condition or an empty string.
     StringRef getPlatformConditionValue(PlatformConditionKind Kind) const;
 
@@ -871,7 +871,7 @@ namespace swift {
     /// 4.2 GHz Intel Core i7.
     /// (It's arbitrary, but will keep the compiler from taking too much time.)
     unsigned SwitchCheckingInvocationThreshold = 200000;
-    
+
     /// If true, the time it takes to type-check each function will be dumped
     /// to llvm::errs().
     bool DebugTimeFunctionBodies = false;
@@ -922,7 +922,7 @@ namespace swift {
 
     /// Enable experimental operator designated types feature.
     bool EnableOperatorDesignatedTypes = false;
-    
+
     /// Disable constraint system performance hacks.
     bool DisableConstraintSolverPerformanceHacks = false;
 
@@ -967,6 +967,9 @@ namespace swift {
     /// The bridging header or PCH that will be imported.
     std::string BridgingHeader;
 
+    /// The bridging header PCH file.
+    std::string BridgingHeaderPCH;
+
     /// When automatically generating a precompiled header from the bridging
     /// header, place it in this directory.
     std::string PrecompiledHeaderOutputDir;
@@ -1088,6 +1091,9 @@ namespace swift {
     /// compilation source targets.
     std::vector<std::string>
     getReducedExtraArgsForSwiftModuleDependency() const;
+
+    /// Get PCH input path. Return empty string if there is no PCH input.
+    std::string getPCHInputPath() const;
   };
 
 } // end namespace swift

include/swift/ClangImporter/ClangImporter.h

@@ -506,6 +506,9 @@ class ClangImporter final : public ClangModuleLoader {
   /// information will be augmented with information about the given
   /// textual header inputs.
   ///
+  /// \param bridgingHeaderContent an optional buffer that pass the briding
+  /// heaer content to the scanner instead of relying on the filename.
+  ///
   /// \param clangScanningTool The clang dependency scanner.
   ///
   /// \param cache The module dependencies cache to update, with information
@@ -514,6 +517,7 @@ class ClangImporter final : public ClangModuleLoader {
   /// \returns \c true if an error occurred, \c false otherwise
   bool getHeaderDependencies(
       ModuleDependencyID moduleID,
+      std::optional<std::string> bridgingHeaderContent,
       clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       ModuleDependenciesCache &cache,
       ModuleDependencyIDSetVector &headerClangModuleDependencies,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -116,10 +116,10 @@ class ModuleDependencyScanner {
   resolveAllClangModuleDependencies(ArrayRef<ModuleDependencyID> swiftModules,
                                     ModuleDependenciesCache &cache,
                                     ModuleDependencyIDSetVector &discoveredClangModules);
-  void
-  resolveHeaderDependencies(ArrayRef<ModuleDependencyID> swiftModules,
-                            ModuleDependenciesCache &cache,
-                            ModuleDependencyIDSetVector &discoveredHeaderDependencyClangModules);
+  void resolveHeaderDependencies(
+      const ModuleDependencyID &rootModuleID,
+      ArrayRef<ModuleDependencyID> swiftModules, ModuleDependenciesCache &cache,
+      ModuleDependencyIDSetVector &discoveredHeaderDependencyClangModules);
   void
   resolveSwiftOverlayDependencies(ArrayRef<ModuleDependencyID> swiftModules,
                                   ModuleDependenciesCache &cache,
@@ -171,6 +171,8 @@ class ModuleDependencyScanner {
   std::mutex WorkersLock;
   /// Count of filesystem queries performed
   std::atomic<unsigned> NumLookups = 0;
+  /// If using auto bridging header chaining.
+  bool BridgingHeaderChaining = false;
 };
 
 } // namespace swift

include/swift/Frontend/FrontendOptions.h

@@ -54,6 +54,9 @@ class FrontendOptions {
   /// An Objective-C header to import and make implicitly visible.
   std::string ImplicitObjCHeaderPath;
 
+  /// An Objective-C pch to import and make implicitly visible.
+  std::string ImplicitObjCPCHPath;
+
   /// The map of aliases and real names of imported or referenced modules.
   llvm::StringMap<StringRef> ModuleAliasMap;
 

include/swift/Option/Options.td

@@ -173,12 +173,12 @@ def verify_incremental_dependencies :
   Flag<["-"], "verify-incremental-dependencies">,
   Flags<[FrontendOption, HelpHidden]>,
   HelpText<"Enable the dependency verifier for each frontend job">;
-  
+
 def strict_implicit_module_context :
   Flag<["-"], "strict-implicit-module-context">,
   Flags<[FrontendOption, HelpHidden]>,
   HelpText<"Enable the strict forwarding of compilation context to downstream implicit module dependencies">;
-  
+
 def no_strict_implicit_module_context :
   Flag<["-"], "no-strict-implicit-module-context">,
   Flags<[FrontendOption, HelpHidden]>,
@@ -315,7 +315,7 @@ def tools_directory : Separate<["-"], "tools-directory">,
 
 def D : JoinedOrSeparate<["-"], "D">, Flags<[FrontendOption]>,
   HelpText<"Marks a conditional compilation flag as true">;
-  
+
 def e : Separate<["-"], "e">, Flags<[NewDriverOnlyOption]>,
   HelpText<"Executes a line of code provided on the command line">;
 
@@ -353,6 +353,9 @@ def import_objc_header : Separate<["-"], "import-objc-header">,
 def import_bridging_header : Separate<["-"], "import-bridging-header">,
   Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
   Alias<import_objc_header>;
+def import_pch : Separate<["-"], "import-pch">,
+  Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
+  HelpText<"Import bridging header PCH file">;
 
 def pch_output_dir: Separate<["-"], "pch-output-dir">,
   Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
@@ -2117,7 +2120,7 @@ def external_plugin_path : Separate<["-"], "external-plugin-path">, Group<plugin
   Flags<[FrontendOption, ArgumentIsPath, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>,
   HelpText<"Add directory to the plugin search path with a plugin server executable">,
   MetaVarName<"<path>#<plugin-server-path>">;
-  
+
 def cas_backend: Flag<["-"], "cas-backend">,
   Flags<[FrontendOption, NoDriverOption]>,
   HelpText<"Enable using CASBackend for object file output">;

include/swift/Serialization/SerializedModuleLoader.h

@@ -430,7 +430,7 @@ class SerializedASTFile final : public LoadedFile {
   getFilenameForPrivateDecl(const Decl *decl) const override;
 
   virtual TypeDecl *lookupLocalType(StringRef MangledName) const override;
-  
+
   virtual OpaqueTypeDecl *
   lookupOpaqueResultType(StringRef MangledName) override;
 
@@ -577,6 +577,11 @@ bool extractCompilerFlagsFromInterface(
 
 /// Extract the user module version number from an interface file.
 llvm::VersionTuple extractUserModuleVersionFromInterface(StringRef moduleInterfacePath);
+
+/// Extract embedded bridging header from binary module.
+std::string
+extractEmbeddedBridgingHeaderContent(std::unique_ptr<llvm::MemoryBuffer> file,
+                                     ASTContext &Context);
 } // end namespace swift
 
 #endif

lib/Basic/LangOptions.cpp

@@ -18,6 +18,7 @@
 #include "swift/Basic/LangOptions.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/Basic/Feature.h"
+#include "swift/Basic/FileTypes.h"
 #include "swift/Basic/Platform.h"
 #include "swift/Basic/PlaygroundOption.h"
 #include "swift/Basic/Range.h"
@@ -814,3 +815,14 @@ ClangImporterOptions::getReducedExtraArgsForSwiftModuleDependency() const {
 
   return filtered_args;
 }
+
+std::string ClangImporterOptions::getPCHInputPath() const {
+  if (!BridgingHeaderPCH.empty())
+    return BridgingHeaderPCH;
+
+  if (llvm::sys::path::extension(BridgingHeader)
+          .ends_with(file_types::getExtension(file_types::TY_PCH)))
+    return BridgingHeader;
+
+  return {};
+}

lib/ClangImporter/ClangImporter.cpp

@@ -476,11 +476,10 @@ void importer::getNormalInvocationArguments(
   ClangImporterOptions &importerOpts = ctx.ClangImporterOpts;
   auto languageVersion = ctx.LangOpts.EffectiveLanguageVersion;
 
-  if (isPCHFilenameExtension(importerOpts.BridgingHeader)) {
-    invocationArgStrs.insert(invocationArgStrs.end(), {
-        "-include-pch", importerOpts.BridgingHeader
-    });
-  }
+  auto bridgingPCH = importerOpts.getPCHInputPath();
+  if (!bridgingPCH.empty())
+    invocationArgStrs.insert(invocationArgStrs.end(),
+                             {"-include-pch", bridgingPCH});
 
   // If there are no shims in the resource dir, add a search path in the SDK.
   SmallString<128> shimsPath(searchPathOpts.RuntimeResourcePath);
@@ -1066,9 +1065,10 @@ void ClangImporter::addClangInvovcationDependencies(
 std::optional<std::string>
 ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions,
                               StringRef SwiftPCHHash, bool &isExplicit) {
-  if (isPCHFilenameExtension(ImporterOptions.BridgingHeader)) {
+  auto bridgingPCH = ImporterOptions.getPCHInputPath();
+  if (!bridgingPCH.empty()) {
     isExplicit = true;
-    return ImporterOptions.BridgingHeader;
+    return bridgingPCH;
   }
   isExplicit = false;
 
@@ -1302,16 +1302,17 @@ ClangImporter::create(ASTContext &ctx,
       new ClangImporter(ctx, tracker, dwarfImporterDelegate)};
   auto &importerOpts = ctx.ClangImporterOpts;
 
-  if (isPCHFilenameExtension(importerOpts.BridgingHeader)) {
-    importer->Impl.setSinglePCHImport(importerOpts.BridgingHeader);
+  auto bridgingPCH = importerOpts.getPCHInputPath();
+  if (!bridgingPCH.empty()) {
+    importer->Impl.setSinglePCHImport(bridgingPCH);
     importer->Impl.IsReadingBridgingPCH = true;
     if (tracker) {
       // Currently ignoring dependency on bridging .pch files because they are
       // temporaries; if and when they are no longer temporaries, this condition
       // should be removed.
       auto &coll = static_cast<ClangImporterDependencyCollector &>(
         *tracker->getClangCollector());
-      coll.excludePath(importerOpts.BridgingHeader);
+      coll.excludePath(bridgingPCH);
     }
   }
 
@@ -6328,7 +6329,7 @@ static bool isImplValid(ExtensionDecl *ext) {
   // This is diagnosed in AttributeChecker::visitObjCImplementationAttr().
   if (!attr->isEarlyAdopter() && !attr->CategoryName.empty())
     return false;
-  
+
   return !attr->isCategoryNameInvalid();
 }
 
@@ -7928,7 +7929,7 @@ CxxRecordSemantics::evaluate(Evaluator &evaluator,
   if (hasIteratorAPIAttr(cxxDecl) || isIterator(cxxDecl)) {
     return CxxRecordSemanticsKind::Iterator;
   }
-  
+
   if (hasCopyTypeOperations(cxxDecl)) {
     return CxxRecordSemanticsKind::Owned;
   }
@@ -7985,7 +7986,7 @@ bool anySubobjectsSelfContained(const clang::CXXRecordDecl *decl) {
 
   if (hasCustomCopyOrMoveConstructor(decl) || hasOwnedValueAttr(decl))
     return true;
-  
+
   auto checkType = [](clang::QualType t) {
     if (auto recordType = dyn_cast<clang::RecordType>(t.getCanonicalType())) {
       if (auto cxxRecord =
@@ -8006,7 +8007,7 @@ bool anySubobjectsSelfContained(const clang::CXXRecordDecl *decl) {
     if (checkType(base.getType()))
       return true;
   }
-  
+
   return false;
 }
 

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -456,6 +456,7 @@ ClangImporter::getModuleDependencies(Identifier moduleName,
 
 bool ClangImporter::getHeaderDependencies(
     ModuleDependencyID moduleID,
+    std::optional<std::string> bridgingHeaderContent,
     clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
     ModuleDependenciesCache &cache,
     ModuleDependencyIDSetVector &headerClangModuleDependencies,
@@ -469,8 +470,9 @@ bool ClangImporter::getHeaderDependencies(
     return false;
 
   // Scan the specified textual header file and collect its dependencies
-  auto scanHeaderDependencies =
-      [&](StringRef headerPath) -> llvm::Expected<TranslationUnitDeps> {
+  auto scanHeaderDependencies = [&](StringRef headerPath,
+                                    std::optional<std::string> content)
+      -> llvm::Expected<TranslationUnitDeps> {
     auto &ctx = Impl.SwiftContext;
     std::vector<std::string> commandLineArgs =
         getClangDepScanningInvocationArguments(ctx, StringRef(headerPath));
@@ -489,9 +491,15 @@ bool ClangImporter::getHeaderDependencies(
                            ModuleOutputKind MOK) -> std::string {
       return moduleCacheRelativeLookupModuleOutput(MID, MOK, moduleOutputPath);
     };
+    std::unique_ptr<llvm::MemoryBuffer> maybeBuffer;
+    std::optional<llvm::MemoryBufferRef> maybeBufferRef;
+    if (content) {
+      maybeBuffer = llvm::MemoryBuffer::getMemBufferCopy(*content);
+      maybeBufferRef = maybeBuffer->getMemBufferRef();
+    }
     auto dependencies = clangScanningTool.getTranslationUnitDependencies(
         commandLineArgs, workingDir, cache.getAlreadySeenClangModules(),
-        lookupModuleOutput);
+        lookupModuleOutput, /*AdditionalModules=*/{}, maybeBufferRef);
     if (!dependencies)
       return dependencies.takeError();
 
@@ -514,12 +522,28 @@ bool ClangImporter::getHeaderDependencies(
     return dependencies;
   };
 
+  // - If a generated header is provided, scan the generated header.
   // - Textual module dependencies require us to process their bridging header.
   // - Binary module dependnecies may have arbitrary header inputs.
-  if (targetModuleInfo.isTextualSwiftModule() &&
-      !targetModuleInfo.getBridgingHeader()->empty()) {
+  if (bridgingHeaderContent) {
     auto clangModuleDependencies =
-        scanHeaderDependencies(*targetModuleInfo.getBridgingHeader());
+        scanHeaderDependencies("", *bridgingHeaderContent);
+    if (!clangModuleDependencies) {
+      // FIXME: Route this to a normal diagnostic.
+      llvm::logAllUnhandledErrors(clangModuleDependencies.takeError(),
+                                  llvm::errs());
+      Impl.SwiftContext.Diags.diagnose(
+          SourceLoc(), diag::clang_dependency_scan_error,
+          "failed to scan bridging header dependencies");
+      return true;
+    }
+    if (auto TreeID = clangModuleDependencies->IncludeTreeID)
+      includeTreeID = TreeID;
+    getBridgingHeaderOptions(*clangModuleDependencies, bridgingHeaderCommandLine);
+  } else if (targetModuleInfo.isTextualSwiftModule() &&
+      !targetModuleInfo.getBridgingHeader()->empty()) {
+    auto clangModuleDependencies = scanHeaderDependencies(
+        *targetModuleInfo.getBridgingHeader(), /*content=*/std::nullopt);
     if (!clangModuleDependencies) {
       // FIXME: Route this to a normal diagnostic.
       llvm::logAllUnhandledErrors(clangModuleDependencies.takeError(),
@@ -536,7 +560,8 @@ bool ClangImporter::getHeaderDependencies(
   } else if (targetModuleInfo.isSwiftBinaryModule()) {
     auto swiftBinaryDeps = targetModuleInfo.getAsSwiftBinaryModule();
     if (!swiftBinaryDeps->headerImport.empty()) {
-      auto clangModuleDependencies = scanHeaderDependencies(swiftBinaryDeps->headerImport);
+      auto clangModuleDependencies = scanHeaderDependencies(
+          swiftBinaryDeps->headerImport, /*content=*/std::nullopt);
       if (!clangModuleDependencies)
         return true;
     }