[clang][DependencyScanning] Track dependencies from prebuilt modules to determine IsInStableDir

clang/include/clang/Serialization/ASTReader.h

@@ -237,6 +237,18 @@ class ASTReaderListener {
     return true;
   }
 
+  /// Overloaded member function of \c visitInputFile that should
+  /// be defined when there is a distinction between
+  /// the file name and name-as-requested. For example, when deserializing input
+  /// files from precompiled AST files.
+  ///
+  /// \returns true to continue receiving the next input file, false to stop.
+  virtual bool visitInputFile(StringRef FilenameAsRequested, StringRef Filename,
+                              bool isSystem, bool isOverridden,
+                              bool isExplicitModule) {
+    return true;
+  }
+
   /// Returns true if this \c ASTReaderListener wants to receive the
   /// imports of the AST file via \c visitImport, false otherwise.
   virtual bool needsImportVisitation() const { return false; }

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -33,6 +33,7 @@ namespace dependencies {
 
 class DependencyActionController;
 class DependencyConsumer;
+class PrebuiltModuleASTAttrs;
 
 /// Modular dependency that has already been built prior to the dependency scan.
 struct PrebuiltModuleDep {
@@ -46,6 +47,47 @@ struct PrebuiltModuleDep {
         ModuleMapFile(M->PresumedModuleMapFile) {}
 };
 
+/// Attributes loaded from AST files of prebuilt modules collected prior to
+/// ModuleDepCollector creation.
+using PrebuiltModulesAttrsMap = llvm::StringMap<PrebuiltModuleASTAttrs>;
+class PrebuiltModuleASTAttrs {
+public:
+  /// When a module is discovered to not be in stable directories, traverse &
+  /// update all modules that depend on it.
+  void
+  updateDependentsNotInStableDirs(PrebuiltModulesAttrsMap &PrebuiltModulesMap);
+
+  /// Read-only access to whether the module is made up of dependencies in
+  /// stable directories.
+  bool isInStableDir() const { return IsInStableDirs; }
+
+  /// Read-only access to vfs map files.
+  const llvm::StringSet<> &getVFS() const { return VFSMap; }
+
+  /// Update the VFSMap to the one discovered from serializing the AST file.
+  void setVFS(llvm::StringSet<> &&VFS) { VFSMap = std::move(VFS); }
+
+  /// Add a direct dependent module file, so it can be updated if the current
+  /// module is from stable directores.
+  void addDependent(StringRef ModuleFile) {
+    ModuleFileDependents.insert(ModuleFile);
+  }
+
+  /// Update whether the prebuilt module resolves entirely in a stable
+  /// directories.
+  void setInStableDir(bool V = false) {
+    // Cannot reset attribute once it's false.
+    if (!IsInStableDirs)
+      return;
+    IsInStableDirs = V;
+  }
+
+private:
+  llvm::StringSet<> VFSMap;
+  bool IsInStableDirs = true;
+  std::set<StringRef> ModuleFileDependents;
+};
+
 /// This is used to identify a specific module.
 struct ModuleID {
   /// The name of the module. This may include `:` for C++20 module partitions,
@@ -171,8 +213,6 @@ struct ModuleDeps {
       BuildInfo;
 };
 
-using PrebuiltModuleVFSMapT = llvm::StringMap<llvm::StringSet<>>;
-
 class ModuleDepCollector;
 
 /// Callback that records textual includes and direct modular includes/imports
@@ -242,7 +282,7 @@ class ModuleDepCollector final : public DependencyCollector {
                      CompilerInstance &ScanInstance, DependencyConsumer &C,
                      DependencyActionController &Controller,
                      CompilerInvocation OriginalCI,
-                     PrebuiltModuleVFSMapT PrebuiltModuleVFSMap);
+                     const PrebuiltModulesAttrsMap PrebuiltModulesASTMap);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
@@ -262,8 +302,9 @@ class ModuleDepCollector final : public DependencyCollector {
   DependencyConsumer &Consumer;
   /// Callbacks for computing dependency information.
   DependencyActionController &Controller;
-  /// Mapping from prebuilt AST files to their sorted list of VFS overlay files.
-  PrebuiltModuleVFSMapT PrebuiltModuleVFSMap;
+  /// Mapping from prebuilt AST filepaths to their attributes referenced during
+  /// dependency collecting.
+  const PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
   /// Path to the main source file.
   std::string MainFile;
   /// Hash identifying the compilation conditions of the current TU.
@@ -339,6 +380,14 @@ void resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
 bool isPathInStableDir(const ArrayRef<StringRef> Directories,
                        const StringRef Input);
 
+/// Determine if options collected from a module's
+/// compilation can safely be considered as stable.
+///
+/// \param Directories Paths known to be in a stable location. e.g. Sysroot.
+/// \param HSOpts Header search options derived from the compiler invocation.
+bool areOptionsInStableDir(const ArrayRef<StringRef> Directories,
+                           const HeaderSearchOptions &HSOpts);
+
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/lib/Frontend/FrontendActions.cpp

@@ -777,10 +777,11 @@ namespace {
     /// Indicates that the AST file contains particular input file.
     ///
     /// \returns true to continue receiving the next input file, false to stop.
-    bool visitInputFile(StringRef Filename, bool isSystem,
-                        bool isOverridden, bool isExplicitModule) override {
+    bool visitInputFile(StringRef FilenameAsRequested, StringRef Filename,
+                        bool isSystem, bool isOverridden,
+                        bool isExplicitModule) override {
 
-      Out.indent(2) << "Input file: " << Filename;
+      Out.indent(2) << "Input file: " << FilenameAsRequested;
 
       if (isSystem || isOverridden || isExplicitModule) {
         Out << " [";

clang/lib/Serialization/ASTReader.cpp

@@ -2616,6 +2616,14 @@ bool ASTReader::shouldDisableValidationForFile(
   return false;
 }
 
+static std::pair<StringRef, StringRef>
+getUnresolvedInputFilenames(const ASTReader::RecordData &Record,
+                            const StringRef InputBlob) {
+  uint16_t AsRequestedLength = Record[7];
+  return {InputBlob.substr(0, AsRequestedLength),
+          InputBlob.substr(AsRequestedLength)};
+}
+
 InputFileInfo ASTReader::getInputFileInfo(ModuleFile &F, unsigned ID) {
   // If this ID is bogus, just return an empty input file.
   if (ID == 0 || ID > F.InputFileInfosLoaded.size())
@@ -2659,11 +2667,12 @@ InputFileInfo ASTReader::getInputFileInfo(ModuleFile &F, unsigned ID) {
   R.Transient = static_cast<bool>(Record[4]);
   R.TopLevel = static_cast<bool>(Record[5]);
   R.ModuleMap = static_cast<bool>(Record[6]);
-  uint16_t AsRequestedLength = Record[7];
-  R.UnresolvedImportedFilenameAsRequested = Blob.substr(0, AsRequestedLength);
-  R.UnresolvedImportedFilename = Blob.substr(AsRequestedLength);
-  if (R.UnresolvedImportedFilename.empty())
-    R.UnresolvedImportedFilename = R.UnresolvedImportedFilenameAsRequested;
+  auto [UnresolvedFilenameAsRequested, UnresolvedFilename] =
+      getUnresolvedInputFilenames(Record, Blob);
+  R.UnresolvedImportedFilenameAsRequested = UnresolvedFilenameAsRequested;
+  R.UnresolvedImportedFilename = UnresolvedFilename.empty()
+                                     ? UnresolvedFilenameAsRequested
+                                     : UnresolvedFilename;
 
   Expected<llvm::BitstreamEntry> MaybeEntry = Cursor.advance();
   if (!MaybeEntry) // FIXME this drops errors on the floor.
@@ -5704,6 +5713,11 @@ bool ASTReader::readASTFileControlBlock(
   bool DoneWithControlBlock = false;
   SmallString<0> PathBuf;
   PathBuf.reserve(256);
+  // Additional path buffer to use when multiple paths need to be resolved.
+  // For example, when deserializing input files that contains a path that was
+  // resolved from a vfs overlay and an external location.
+  SmallString<0> AdditionalPathBuf;
+  AdditionalPathBuf.reserve(256);
   while (!DoneWithControlBlock) {
     Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
     if (!MaybeEntry) {
@@ -5835,9 +5849,21 @@ bool ASTReader::readASTFileControlBlock(
           break;
         case INPUT_FILE:
           bool Overridden = static_cast<bool>(Record[3]);
-          auto Filename = ResolveImportedPath(PathBuf, Blob, ModuleDir);
+          auto [UnresolvedFilenameAsRequested, UnresolvedFilename] =
+              getUnresolvedInputFilenames(Record, Blob);
+          auto FilenameAsRequestedBuf = ResolveImportedPath(
+              PathBuf, UnresolvedFilenameAsRequested, ModuleDir);
+          StringRef Filename;
+          if (UnresolvedFilename.empty())
+            Filename = *FilenameAsRequestedBuf;
+          else {
+            auto FilenameBuf = ResolveImportedPath(
+                AdditionalPathBuf, UnresolvedFilename, ModuleDir);
+            Filename = *FilenameBuf;
+          }
           shouldContinue = Listener.visitInputFile(
-              *Filename, isSystemFile, Overridden, /*IsExplicitModule=*/false);
+              *FilenameAsRequestedBuf, Filename, isSystemFile, Overridden,
+              /*IsExplicitModule=*/false);
           break;
         }
         if (!shouldContinue)

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -90,63 +90,140 @@ static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
 
 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
 
-/// A listener that collects the imported modules and optionally the input
-/// files.
+/// A listener that collects the imported modules and the input
+/// files. While visiting, collect vfsoverlays and file inputs that determine
+/// whether prebuilt modules fully resolve in stable directories.
 class PrebuiltModuleListener : public ASTReaderListener {
 public:
   PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
                          llvm::SmallVector<std::string> &NewModuleFiles,
-                         PrebuiltModuleVFSMapT &PrebuiltModuleVFSMap,
+                         PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
                          const HeaderSearchOptions &HSOpts,
-                         const LangOptions &LangOpts, DiagnosticsEngine &Diags)
+                         const LangOptions &LangOpts, DiagnosticsEngine &Diags,
+                         const llvm::SmallVector<StringRef> &StableDirs)
       : PrebuiltModuleFiles(PrebuiltModuleFiles),
         NewModuleFiles(NewModuleFiles),
-        PrebuiltModuleVFSMap(PrebuiltModuleVFSMap), ExistingHSOpts(HSOpts),
-        ExistingLangOpts(LangOpts), Diags(Diags) {}
+        PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
+        ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
 
   bool needsImportVisitation() const override { return true; }
+  bool needsInputFileVisitation() override { return true; }
+  bool needsSystemInputFileVisitation() override { return true; }
 
+  /// Accumulate the modules are transitively depended on by the initial
+  /// prebuilt module.
   void visitImport(StringRef ModuleName, StringRef Filename) override {
     if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
       NewModuleFiles.push_back(Filename.str());
+
+    auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
+    PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
+    if (PrebuiltMapEntry.second)
+      PrebuiltModule.setInStableDir(!StableDirs.empty());
+
+    if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
+        It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
+      PrebuiltModule.addDependent(It->getKey());
+  }
+
+  /// For each input file discovered, check whether it's external path is in a
+  /// stable directory. Traversal is stopped if the current module is not
+  /// considered stable.
+  bool visitInputFile(StringRef FilenameAsRequested, StringRef Filename,
+                      bool isSystem, bool isOverridden,
+                      bool isExplicitModule) override {
+    if (StableDirs.empty())
+      return false;
+    auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
+    if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
+        (!PrebuiltEntryIt->second.isInStableDir()))
+      return false;
+
+    PrebuiltEntryIt->second.setInStableDir(
+        isPathInStableDir(StableDirs, Filename));
+    return PrebuiltEntryIt->second.isInStableDir();
   }
 
+  /// Update which module that is being actively traversed.
   void visitModuleFile(StringRef Filename,
                        serialization::ModuleKind Kind) override {
+    // If the CurrentFile is not
+    // considered stable, update any of it's transitive dependents.
+    auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
+    if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
+        !PrebuiltEntryIt->second.isInStableDir())
+      PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
+          PrebuiltModulesASTMap);
     CurrentFile = Filename;
   }
 
+  /// Check the header search options for a given module when considering
+  /// if the module comes from stable directories.
+  bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
+                               StringRef ModuleFilename,
+                               StringRef SpecificModuleCachePath,
+                               bool Complain) override {
+
+    auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
+    PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
+    if (PrebuiltMapEntry.second)
+      PrebuiltModule.setInStableDir(!StableDirs.empty());
+
+    if (PrebuiltModule.isInStableDir())
+      PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
+
+    return false;
+  }
+
+  /// Accumulate vfsoverlays used to build these prebuilt modules.
   bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
                              bool Complain) override {
-    std::vector<std::string> VFSOverlayFiles = HSOpts.VFSOverlayFiles;
-    PrebuiltModuleVFSMap.try_emplace(CurrentFile, llvm::from_range,
-                                     VFSOverlayFiles);
+
+    auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
+    PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
+    if (PrebuiltMapEntry.second)
+      PrebuiltModule.setInStableDir(!StableDirs.empty());
+
+    PrebuiltModule.setVFS(
+        llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
+
     return checkHeaderSearchPaths(
         HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
   }
 
 private:
   PrebuiltModuleFilesT &PrebuiltModuleFiles;
   llvm::SmallVector<std::string> &NewModuleFiles;
-  PrebuiltModuleVFSMapT &PrebuiltModuleVFSMap;
+  PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
   const HeaderSearchOptions &ExistingHSOpts;
   const LangOptions &ExistingLangOpts;
   DiagnosticsEngine &Diags;
   std::string CurrentFile;
+  const llvm::SmallVector<StringRef> &StableDirs;
 };
 
 /// Visit the given prebuilt module and collect all of the modules it
 /// transitively imports and contributing input files.
 static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
                                 CompilerInstance &CI,
                                 PrebuiltModuleFilesT &ModuleFiles,
-                                PrebuiltModuleVFSMapT &PrebuiltModuleVFSMap,
+                                PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
                                 DiagnosticsEngine &Diags) {
+
+  // Gather the set of stable directories to use as transitive dependencies are
+  // discovered.
+  llvm::SmallVector<StringRef> StableDirs;
+  std::string SysrootToUse(CI.getHeaderSearchOpts().Sysroot);
+  if (!SysrootToUse.empty() &&
+      (llvm::sys::path::root_directory(SysrootToUse) != SysrootToUse))
+    StableDirs = {SysrootToUse, CI.getHeaderSearchOpts().ResourceDir};
+
   // List of module files to be processed.
   llvm::SmallVector<std::string> Worklist;
-  PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModuleVFSMap,
+
+  PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
                                   CI.getHeaderSearchOpts(), CI.getLangOpts(),
-                                  Diags);
+                                  Diags, StableDirs);
 
   Listener.visitModuleFile(PrebuiltModuleFilename,
                            serialization::MK_ExplicitModule);
@@ -371,16 +448,18 @@ class DependencyScanningAction : public tooling::ToolAction {
     auto *FileMgr = ScanInstance.createFileManager(FS);
     ScanInstance.createSourceManager(*FileMgr);
 
-    // Store the list of prebuilt module files into header search options. This
-    // will prevent the implicit build to create duplicate modules and will
-    // force reuse of the existing prebuilt module files instead.
-    PrebuiltModuleVFSMapT PrebuiltModuleVFSMap;
+    // Store a mapping of prebuilt module files and their properties like header
+    // search options. This will prevent the implicit build to create duplicate
+    // modules and will force reuse of the existing prebuilt module files
+    // instead.
+    PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
+
     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
       if (visitPrebuiltModule(
               ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
               ScanInstance,
               ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
-              PrebuiltModuleVFSMap, ScanInstance.getDiagnostics()))
+              PrebuiltModulesASTMap, ScanInstance.getDiagnostics()))
         return false;
 
     // Create the dependency collector that will collect the produced
@@ -410,7 +489,7 @@ class DependencyScanningAction : public tooling::ToolAction {
     case ScanningOutputFormat::Full:
       MDC = std::make_shared<ModuleDepCollector>(
           Service, std::move(Opts), ScanInstance, Consumer, Controller,
-          OriginalInvocation, std::move(PrebuiltModuleVFSMap));
+          OriginalInvocation, std::move(PrebuiltModulesASTMap));
       ScanInstance.addDependencyCollector(MDC);
       break;
     }