[clang][deps] Cherry-pick some dependency-scanner related commits

clang/include/clang/Basic/DiagnosticGroups.td

@@ -744,6 +744,7 @@ def UnusedLocalTypedef : DiagGroup<"unused-local-typedef">;
 def UnusedPropertyIvar :  DiagGroup<"unused-property-ivar">;
 def UnusedGetterReturnValue : DiagGroup<"unused-getter-return-value">;
 def UsedButMarkedUnused : DiagGroup<"used-but-marked-unused">;
+def UsedSearchPath : DiagGroup<"search-path-usage">;
 def UserDefinedLiterals : DiagGroup<"user-defined-literals">;
 def UserDefinedWarnings : DiagGroup<"user-defined-warnings">;
 def ReorderCtor : DiagGroup<"reorder-ctor">;

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -430,6 +430,9 @@ def warn_pp_hdrstop_filename_ignored : Warning<
   "#pragma hdrstop filename not supported, "
   "/Fp can be used to specify precompiled header filename">,
   InGroup<ClangClPch>;
+def remark_pp_search_path_usage : Remark<
+  "search path used: '%0'">,
+  InGroup<UsedSearchPath>;
 def err_pp_file_not_found_angled_include_not_fatal : Error<
   "'%0' file not found with <angled> %select{include|import}1; "
   "use \"quotes\" instead">;

clang/include/clang/Lex/HeaderMap.h

@@ -77,13 +77,6 @@ class HeaderMap : private HeaderMapImpl {
   static std::unique_ptr<HeaderMap> Create(const FileEntry *FE,
                                            FileManager &FM);
 
-  /// Check to see if the specified relative filename is located in this
-  /// HeaderMap.  If so, open it and return its FileEntry.  If RawPath is not
-  /// NULL and the file is found, RawPath will be set to the raw path at which
-  /// the file was found in the file system. For example, for a search path
-  /// ".." and a filename "../file.h" this would be "../../file.h".
-  Optional<FileEntryRef> LookupFile(StringRef Filename, FileManager &FM) const;
-
   using HeaderMapImpl::dump;
   using HeaderMapImpl::getFileName;
   using HeaderMapImpl::lookupFilename;

clang/include/clang/Lex/HeaderSearch.h

@@ -172,6 +172,9 @@ class HeaderSearch {
   /// Header-search options used to initialize this header search.
   std::shared_ptr<HeaderSearchOptions> HSOpts;
 
+  /// Mapping from SearchDir to HeaderSearchOptions::UserEntries indices.
+  llvm::DenseMap<unsigned, unsigned> SearchDirToHSEntry;
+
   DiagnosticsEngine &Diags;
   FileManager &FileMgr;
 
@@ -182,6 +185,9 @@ class HeaderSearch {
   /// NoCurDirSearch is true, then the check for the file in the current
   /// directory is suppressed.
   std::vector<DirectoryLookup> SearchDirs;
+  /// Whether the DirectoryLookup at the corresponding index in SearchDirs has
+  /// been successfully used to lookup a file.
+  std::vector<bool> SearchDirsUsage;
   unsigned AngledDirIdx = 0;
   unsigned SystemDirIdx = 0;
   bool NoCurDirSearch = false;
@@ -280,22 +286,25 @@ class HeaderSearch {
   DiagnosticsEngine &getDiags() const { return Diags; }
 
   /// Interface for setting the file search paths.
-  void SetSearchPaths(const std::vector<DirectoryLookup> &dirs,
-                      unsigned angledDirIdx, unsigned systemDirIdx,
-                      bool noCurDirSearch) {
+  void SetSearchPaths(std::vector<DirectoryLookup> dirs, unsigned angledDirIdx,
+                      unsigned systemDirIdx, bool noCurDirSearch,
+                      llvm::DenseMap<unsigned, unsigned> searchDirToHSEntry) {
     assert(angledDirIdx <= systemDirIdx && systemDirIdx <= dirs.size() &&
         "Directory indices are unordered");
-    SearchDirs = dirs;
+    SearchDirs = std::move(dirs);
+    SearchDirsUsage.assign(SearchDirs.size(), false);
     AngledDirIdx = angledDirIdx;
     SystemDirIdx = systemDirIdx;
     NoCurDirSearch = noCurDirSearch;
+    SearchDirToHSEntry = std::move(searchDirToHSEntry);
     //LookupFileCache.clear();
   }
 
   /// Add an additional search path.
   void AddSearchPath(const DirectoryLookup &dir, bool isAngled) {
     unsigned idx = isAngled ? SystemDirIdx : AngledDirIdx;
     SearchDirs.insert(SearchDirs.begin() + idx, dir);
+    SearchDirsUsage.insert(SearchDirsUsage.begin() + idx, false);
     if (!isAngled)
       AngledDirIdx++;
     SystemDirIdx++;
@@ -507,6 +516,10 @@ class HeaderSearch {
     return FI && FI->isImport;
   }
 
+  /// Determine which HeaderSearchOptions::UserEntries have been successfully
+  /// used so far and mark their index with 'true' in the resulting bit vector.
+  std::vector<bool> computeUserEntryUsage() const;
+
   /// This method returns a HeaderMap for the specified
   /// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
   const HeaderMap *CreateHeaderMap(const FileEntry *FE);
@@ -562,6 +575,8 @@ class HeaderSearch {
   ///
   /// \param ModuleName The name of the module we're looking for.
   ///
+  /// \param ImportLoc Location of the module include/import.
+  ///
   /// \param AllowSearch Whether we are allowed to search in the various
   /// search directories to produce a module definition. If not, this lookup
   /// will only return an already-known module.
@@ -570,7 +585,9 @@ class HeaderSearch {
   /// in subdirectories.
   ///
   /// \returns The module with the given name.
-  Module *lookupModule(StringRef ModuleName, bool AllowSearch = true,
+  Module *lookupModule(StringRef ModuleName,
+                       SourceLocation ImportLoc = SourceLocation(),
+                       bool AllowSearch = true,
                        bool AllowExtraModuleMapSearch = false);
 
   /// Try to find a module map file in the given directory, returning
@@ -640,11 +657,14 @@ class HeaderSearch {
   /// but for compatibility with some buggy frameworks, additional attempts
   /// may be made to find the module under a related-but-different search-name.
   ///
+  /// \param ImportLoc Location of the module include/import.
+  ///
   /// \param AllowExtraModuleMapSearch Whether we allow to search modulemaps
   /// in subdirectories.
   ///
   /// \returns The module named ModuleName.
   Module *lookupModule(StringRef ModuleName, StringRef SearchName,
+                       SourceLocation ImportLoc,
                        bool AllowExtraModuleMapSearch = false);
 
   /// Retrieve the name of the (to-be-)cached module file that should
@@ -709,6 +729,14 @@ class HeaderSearch {
                           Module *RequestingModule,
                           ModuleMap::KnownHeader *SuggestedModule);
 
+  /// Cache the result of a successful lookup at the given include location
+  /// using the search path at index `HitIdx`.
+  void cacheLookupSuccess(LookupFileCacheInfo &CacheLookup, unsigned HitIdx,
+                          SourceLocation IncludeLoc);
+  /// Note that a lookup at the given include location was successful using the
+  /// search path at index `HitIdx`.
+  void noteLookupUsage(unsigned HitIdx, SourceLocation IncludeLoc);
+
 public:
   /// Retrieve the module map.
   ModuleMap &getModuleMap() { return ModMap; }
@@ -758,6 +786,9 @@ class HeaderSearch {
 
   search_dir_iterator system_dir_end() const { return SearchDirs.end(); }
 
+  /// Get the index of the given search directory.
+  Optional<unsigned> searchDirIdx(const DirectoryLookup &DL) const;
+
   /// Retrieve a uniqued framework name.
   StringRef getUniqueFrameworkName(StringRef Framework);
 

clang/include/clang/Serialization/ASTBitCodes.h

@@ -405,6 +405,9 @@ enum UnhashedControlBlockRecordTypes {
 
   /// Record code for \#pragma diagnostic mappings.
   DIAG_PRAGMA_MAPPINGS,
+
+  /// Record code for the indices of used header search entries.
+  HEADER_SEARCH_ENTRY_USAGE,
 };
 
 /// Record code for extension blocks.

clang/include/clang/Serialization/ModuleFile.h

@@ -20,6 +20,7 @@
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ContinuousRangeMap.h"
 #include "clang/Serialization/ModuleFileExtension.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SetVector.h"
@@ -173,6 +174,9 @@ class ModuleFile {
   /// unique module files based on AST contents.
   ASTFileSignature ASTBlockHash;
 
+  /// The bit vector denoting usage of each header search entry (true = used).
+  llvm::BitVector SearchPathUsage;
+
   /// Whether this module has been directly imported by the
   /// user.
   bool DirectlyImported = false;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

@@ -48,7 +48,8 @@ class DependencyScanningService {
 public:
   DependencyScanningService(ScanningMode Mode, ScanningOutputFormat Format,
                             bool ReuseFileManager = true,
-                            bool SkipExcludedPPRanges = true);
+                            bool SkipExcludedPPRanges = true,
+                            bool OptimizeArgs = false);
 
   ScanningMode getMode() const { return Mode; }
 
@@ -58,6 +59,8 @@ class DependencyScanningService {
 
   bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
 
+  bool canOptimizeArgs() const { return OptimizeArgs; }
+
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
@@ -70,6 +73,8 @@ class DependencyScanningService {
   /// ranges by bumping the buffer pointer in the lexer instead of lexing the
   /// tokens in the range until reaching the corresponding directive.
   const bool SkipExcludedPPRanges;
+  /// Whether to optimize the modules' command-line arguments.
+  const bool OptimizeArgs;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
 };

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -87,6 +87,8 @@ class DependencyScanningWorker {
   /// worker. If null, the file manager will not be reused.
   llvm::IntrusiveRefCntPtr<FileManager> Files;
   ScanningOutputFormat Format;
+  /// Whether to optimize the modules' command-line arguments.
+  bool OptimizeArgs;
 };
 
 } // end namespace dependencies

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -102,7 +102,7 @@ struct ModuleDeps {
   bool ImportedByMainFile = false;
 
   /// Compiler invocation that can be used to build this module (without paths).
-  CompilerInvocation Invocation;
+  CompilerInvocation BuildInvocation;
 
   /// Gets the canonical command line suitable for passing to clang.
   ///
@@ -147,8 +147,7 @@ class ModuleDepCollector;
 /// \c DependencyConsumer of the parent \c ModuleDepCollector.
 class ModuleDepCollectorPP final : public PPCallbacks {
 public:
-  ModuleDepCollectorPP(CompilerInstance &I, ModuleDepCollector &MDC)
-      : Instance(I), MDC(MDC) {}
+  ModuleDepCollectorPP(ModuleDepCollector &MDC) : MDC(MDC) {}
 
   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                    SrcMgr::CharacteristicKind FileType,
@@ -165,8 +164,6 @@ class ModuleDepCollectorPP final : public PPCallbacks {
   void EndOfMainFile() override;
 
 private:
-  /// The compiler instance for the current translation unit.
-  CompilerInstance &Instance;
   /// The parent dependency collector.
   ModuleDepCollector &MDC;
   /// Working set of direct modular dependencies.
@@ -199,17 +196,17 @@ class ModuleDepCollectorPP final : public PPCallbacks {
 class ModuleDepCollector final : public DependencyCollector {
 public:
   ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
-                     CompilerInstance &I, DependencyConsumer &C,
-                     CompilerInvocation &&OriginalCI);
+                     CompilerInstance &ScanInstance, DependencyConsumer &C,
+                     CompilerInvocation &&OriginalCI, bool OptimizeArgs);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
 
 private:
   friend ModuleDepCollectorPP;
 
-  /// The compiler instance for the current translation unit.
-  CompilerInstance &Instance;
+  /// The compiler instance for scanning the current translation unit.
+  CompilerInstance &ScanInstance;
   /// The consumer of collected dependency information.
   DependencyConsumer &Consumer;
   /// Path to the main source file.
@@ -225,15 +222,18 @@ class ModuleDepCollector final : public DependencyCollector {
   std::unique_ptr<DependencyOutputOptions> Opts;
   /// The original Clang invocation passed to dependency scanner.
   CompilerInvocation OriginalInvocation;
+  /// Whether to optimize the modules' command-line arguments.
+  bool OptimizeArgs;
 
   /// Checks whether the module is known as being prebuilt.
   bool isPrebuiltModule(const Module *M);
 
   /// Constructs a CompilerInvocation that can be used to build the given
   /// module, excluding paths to discovered modular dependencies that are yet to
   /// be built.
-  CompilerInvocation
-  makeInvocationForModuleBuildWithoutPaths(const ModuleDeps &Deps) const;
+  CompilerInvocation makeInvocationForModuleBuildWithoutPaths(
+      const ModuleDeps &Deps,
+      llvm::function_ref<void(CompilerInvocation &)> Optimize) const;
 };
 
 } // end namespace dependencies