[BrdigingHeader] Auto bridging header chaining

Add ability to automatically chaining the bridging headers discovered from all
dependencies module when doing swift caching build. This will eliminate all
implicit bridging header imports from the build and make the bridging header
importing behavior much more reliable, while keep the compatibility at maximum.

For example, if the current module A depends on module B and C, and both B and
C are binary modules that uses bridging header, when building module A,
dependency scanner will construct a new header that chains three bridging
headers together with the option to build a PCH from it. This will make all
importing errors more obvious while improving the performance.

Author: cachemeifyoucan

include/swift-c/DependencyScan/DependencyScan.h

@@ -25,7 +25,7 @@
 /// SWIFTSCAN_VERSION_MINOR should increase when there are API additions.
 /// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
 #define SWIFTSCAN_VERSION_MAJOR 2
-#define SWIFTSCAN_VERSION_MINOR 0
+#define SWIFTSCAN_VERSION_MINOR 1
 
 SWIFTSCAN_BEGIN_DECLS
 
@@ -222,6 +222,14 @@ SWIFTSCAN_PUBLIC swiftscan_string_ref_t
 swiftscan_swift_textual_detail_get_user_module_version(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_swift_textual_detail_get_chained_bridging_header_path(
+    swiftscan_module_details_t details);
+
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_swift_textual_detail_get_chained_bridging_header_content(
+    swiftscan_module_details_t details);
+
 //=== Swift Binary Module Details query APIs ------------------------------===//
 
 SWIFTSCAN_PUBLIC swiftscan_string_ref_t

include/swift/AST/ModuleDependencies.h

@@ -30,11 +30,13 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/PrefixMapper.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -338,6 +340,12 @@ class SwiftSourceModuleDependenciesStorage
   /// The Swift frontend invocation arguments to build bridging header.
   std::vector<std::string> bridgingHeaderBuildCommandLine;
 
+  /// The chained bridging header path if used.
+  std::string chainedBridgingHeaderPath;
+
+  /// The chained bridging header source buffer if used.
+  std::string chainedBridgingHeaderContent;
+
   SwiftSourceModuleDependenciesStorage(
       StringRef RootID, ArrayRef<StringRef> buildCommandLine,
       ArrayRef<ScannerImportStatementInfo> moduleImports,
@@ -354,6 +362,7 @@ class SwiftSourceModuleDependenciesStorage
     return new SwiftSourceModuleDependenciesStorage(*this);
   }
 
+
   static bool classof(const ModuleDependencyInfoStorageBase *base) {
     return base->dependencyKind == ModuleDependencyKind::SwiftSource;
   }
@@ -370,6 +379,11 @@ class SwiftSourceModuleDependenciesStorage
   void addTestableImport(ImportPath::Module module) {
     testableImports.insert(module.front().Item.str());
   }
+
+  void setChainedBridgingHeaderBuffer(StringRef path, StringRef buffer) {
+    chainedBridgingHeaderPath = path.str();
+    chainedBridgingHeaderContent = buffer.str();
+  }
 };
 
 /// Describes the dependencies of a pre-built Swift module (with no
@@ -769,11 +783,11 @@ class ModuleDependencyInfo {
   setLinkLibraries(const ArrayRef<LinkLibrary> linkLibraries) {
     storage->linkLibraries.assign(linkLibraries.begin(), linkLibraries.end());
   }
-  
+
   const ArrayRef<std::string> getAuxiliaryFiles() const {
     return storage->auxiliaryFiles;
   }
-  
+
   void
   setAuxiliaryFiles(const ArrayRef<std::string> auxiliaryFiles) {
     storage->auxiliaryFiles.assign(auxiliaryFiles.begin(), auxiliaryFiles.end());
@@ -961,11 +975,14 @@ class ModuleDependencyInfo {
   void addSourceFile(StringRef sourceFile);
 
   /// Add source files that the header input depends on.
-  void addHeaderSourceFile(StringRef bridgingSourceFile);
+  void setHeaderSourceFiles(const std::vector<std::string> &sourceFiles);
 
   /// Add bridging header include tree.
   void addBridgingHeaderIncludeTree(StringRef ID);
 
+  /// Set the chained bridging header buffer.
+  void setChainedBridgingHeaderBuffer(StringRef path, StringRef buffer);
+
   /// Collect a map from a secondary module name to a list of cross-import
   /// overlays, when this current module serves as the primary module.
   llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
@@ -1025,8 +1042,9 @@ class SwiftDependencyScanningService {
   /// If use clang include tree.
   bool UseClangIncludeTree = false;
 
-  /// CAS ObjectStore Instance.
+  /// CAS Instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> ActionCache;
 
   /// File prefix mapper.
   std::unique_ptr<llvm::PrefixMapper> Mapper;
@@ -1179,11 +1197,11 @@ class ModuleDependenciesCache {
   /// Query all dependencies
   ModuleDependencyIDSetVector
   getAllDependencies(const ModuleDependencyID &moduleID) const;
-  
+
   /// Query all Clang module dependencies.
   ModuleDependencyIDSetVector
   getClangDependencies(const ModuleDependencyID &moduleID) const;
-  
+
   /// Query all directly-imported Swift dependencies
   llvm::ArrayRef<ModuleDependencyID>
   getImportedSwiftDependencies(const ModuleDependencyID &moduleID) const;

include/swift/AST/SearchPathOptions.h

@@ -577,6 +577,9 @@ class SearchPathOptions {
   /// "in-package", must not require package-only module dependencies.
   bool ResolveInPackageModuleDependencies = false;
 
+  /// Enable auto bridging header chaining.
+  bool BridgingHeaderChaining = false;
+
   /// Return all module search paths that (non-recursively) contain a file whose
   /// name is in \p Filenames.
   SmallVector<const ModuleSearchPath *, 4>

include/swift/Basic/LangOptions.h

@@ -201,7 +201,7 @@ namespace swift {
     /// Maximum number of typo corrections we are allowed to perform.
     /// This is disabled by default until we can get typo-correction working within acceptable performance bounds.
     unsigned TypoCorrectionLimit = 0;
-    
+
     /// Should access control be respected?
     bool EnableAccessControl = true;
 
@@ -260,7 +260,7 @@ namespace swift {
 
     /// Emit a remark when indexing a system module.
     bool EnableIndexingSystemModuleRemarks = false;
-    
+
     /// Emit a remark on early exit in explicit interface build
     bool EnableSkipExplicitInterfaceModuleBuildRemarks = false;
 
@@ -684,7 +684,7 @@ namespace swift {
     void clearAllPlatformConditionValues() {
       PlatformConditionValues.clear();
     }
-    
+
     /// Returns the value for the given platform condition or an empty string.
     StringRef getPlatformConditionValue(PlatformConditionKind Kind) const;
 
@@ -864,7 +864,7 @@ namespace swift {
     /// 4.2 GHz Intel Core i7.
     /// (It's arbitrary, but will keep the compiler from taking too much time.)
     unsigned SwitchCheckingInvocationThreshold = 200000;
-    
+
     /// If true, the time it takes to type-check each function will be dumped
     /// to llvm::errs().
     bool DebugTimeFunctionBodies = false;
@@ -908,7 +908,7 @@ namespace swift {
 
     /// Enable experimental operator designated types feature.
     bool EnableOperatorDesignatedTypes = false;
-    
+
     /// Disable constraint system performance hacks.
     bool DisableConstraintSolverPerformanceHacks = false;
 
@@ -956,6 +956,9 @@ namespace swift {
     /// The bridging header or PCH that will be imported.
     std::string BridgingHeader;
 
+    /// The bridging header PCH file.
+    std::string BridgingHeaderPCH;
+
     /// When automatically generating a precompiled header from the bridging
     /// header, place it in this directory.
     std::string PrecompiledHeaderOutputDir;
@@ -1077,6 +1080,9 @@ namespace swift {
     /// compilation source targets.
     std::vector<std::string>
     getReducedExtraArgsForSwiftModuleDependency() const;
+
+    /// Get PCH input path. Return empty string if there is no PCH input.
+    std::string getPCHInputPath() const;
   };
 
 } // end namespace swift

include/swift/ClangImporter/ClangImporter.h

@@ -506,14 +506,17 @@ class ClangImporter final : public ClangModuleLoader {
   /// information will be augmented with information about the given
   /// textual header inputs.
   ///
+  /// \param headerPath the path to the header to be scanned.
+  ///
   /// \param clangScanningTool The clang dependency scanner.
   ///
   /// \param cache The module dependencies cache to update, with information
   /// about new Clang modules discovered along the way.
   ///
   /// \returns \c true if an error occurred, \c false otherwise
   bool getHeaderDependencies(
-      ModuleDependencyID moduleID,
+      ModuleDependencyID moduleID, std::optional<StringRef> headerPath,
+      std::optional<llvm::MemoryBufferRef> sourceBuffer,
       clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       ModuleDependenciesCache &cache,
       ModuleDependencyIDSetVector &headerClangModuleDependencies,
@@ -675,6 +678,11 @@ class ClangImporter final : public ClangModuleLoader {
   const clang::TypedefType *getTypeDefForCXXCFOptionsDefinition(
       const clang::Decl *candidateDecl) override;
 
+  /// Create cache key for embedded bridging header.
+  static llvm::Expected<llvm::cas::ObjectRef>
+  createEmbeddedBridgingHeaderCacheKey(
+      llvm::cas::ObjectStore &CAS, llvm::cas::ObjectRef ChainedPCHIncludeTree);
+
   SourceLoc importSourceLocation(clang::SourceLocation loc) override;
 };
 

include/swift/DependencyScan/DependencyScanImpl.h

@@ -130,6 +130,13 @@ typedef struct {
 
   /// User module version
   swiftscan_string_ref_t user_module_version;
+
+  /// Chained bridging header path.
+  swiftscan_string_ref_t chained_bridging_header_path;
+
+  /// Chained bridging header content.
+  swiftscan_string_ref_t chained_bridging_header_content;
+
 } swiftscan_swift_textual_details_t;
 
 /// Swift modules with only a binary module file.

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -16,6 +16,7 @@
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
+#include "llvm/CAS/CASReference.h"
 #include "llvm/Support/ThreadPool.h"
 
 namespace swift {
@@ -48,6 +49,11 @@ class ModuleDependencyScanningWorker {
                                          llvm::PrefixMapper *prefixMapper,
                                          bool isTestableImport = false);
 
+  /// Store cache entry for include tree.
+  llvm::Error
+  createCacheKeyForEmbeddedHeader(std::string embeddedHeaderIncludeTree,
+                                  std::string chainedHeaderIncludeTree);
+
   // Worker-specific instance of CompilerInvocation
   std::unique_ptr<CompilerInvocation> workerCompilerInvocation;
   // Worker-specific instance of ASTContext
@@ -59,6 +65,9 @@ class ModuleDependencyScanningWorker {
   // Swift and Clang module loaders acting as scanners.
   std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
   std::unique_ptr<ClangImporter> clangScannerModuleLoader;
+  // CAS instance.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> ActionCache;
   // Restrict access to the parent scanner class.
   friend class ModuleDependencyScanner;
 };
@@ -116,10 +125,9 @@ class ModuleDependencyScanner {
   resolveAllClangModuleDependencies(ArrayRef<ModuleDependencyID> swiftModules,
                                     ModuleDependenciesCache &cache,
                                     ModuleDependencyIDSetVector &discoveredClangModules);
-  void
-  resolveHeaderDependencies(ArrayRef<ModuleDependencyID> swiftModules,
-                            ModuleDependenciesCache &cache,
-                            ModuleDependencyIDSetVector &discoveredHeaderDependencyClangModules);
+  void resolveHeaderDependencies(
+      ArrayRef<ModuleDependencyID> swiftModules, ModuleDependenciesCache &cache,
+      ModuleDependencyIDSetVector &discoveredHeaderDependencyClangModules);
   void
   resolveSwiftOverlayDependencies(ArrayRef<ModuleDependencyID> swiftModules,
                                   ModuleDependenciesCache &cache,
@@ -151,6 +159,12 @@ class ModuleDependencyScanner {
       ModuleDependenciesCache &cache,
       llvm::function_ref<void(ModuleDependencyID)> action);
 
+  /// Performance BridgingHeader Chaining.
+  llvm::Error
+  performBridgingHeaderChaining(const ModuleDependencyID &rootModuleID,
+                                ModuleDependenciesCache &cache,
+                                ModuleDependencyIDSetVector &allModules);
+
   /// Perform an operation utilizing one of the Scanning workers
   /// available to this scanner.
   template <typename Function, typename... Args>

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -225,7 +225,9 @@ using SwiftSourceModuleDetailsLayout =
                    IdentifierIDField,                // CASFileSystemRootID
                    IdentifierIDField,                // bridgingHeaderIncludeTree
                    FlagIDArrayIDField,               // buildCommandLine
-                   FlagIDArrayIDField                // bridgingHeaderBuildCommandLine
+                   FlagIDArrayIDField,               // bridgingHeaderBuildCommandLine
+                   IdentifierIDField,                // chainedBridgingHeaderPath
+                   IdentifierIDField                 // chainedBridgingHeaderContent
                    >;
 
 using SwiftBinaryModuleDetailsLayout =

include/swift/Frontend/CompileJobCacheKey.h

@@ -21,6 +21,7 @@
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/Basic/FileTypes.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
@@ -49,6 +50,7 @@ createCompileJobCacheKeyForOutput(llvm::cas::ObjectStore &CAS,
 llvm::Error printCompileJobCacheKey(llvm::cas::ObjectStore &CAS,
                                     llvm::cas::ObjectRef Key,
                                     llvm::raw_ostream &os);
+
 } // namespace swift
 
 #endif

include/swift/Frontend/FrontendOptions.h

@@ -54,6 +54,9 @@ class FrontendOptions {
   /// An Objective-C header to import and make implicitly visible.
   std::string ImplicitObjCHeaderPath;
 
+  /// An Objective-C pch to import and make implicitly visible.
+  std::string ImplicitObjCPCHPath;
+
   /// The map of aliases and real names of imported or referenced modules.
   llvm::StringMap<StringRef> ModuleAliasMap;
 
@@ -369,6 +372,12 @@ class FrontendOptions {
   /// Emit remarks indicating use of the serialized module dependency scanning cache.
   bool EmitDependencyScannerCacheRemarks = false;
 
+  /// The path at which the dependency scanner can write generated files.
+  std::string ScannerOutputDir;
+
+  /// If the scanner output is written directly to the disk for debugging.
+  bool WriteScannerOutput = false;
+
   /// Whether the dependency scanner invocation should resolve imports
   /// to filesystem modules in parallel.
   bool ParallelDependencyScan = true;