[clang][lex] Remark on search path usage

For dependency scanning, it would be useful to collect header search paths (provided on command-line via `-I` and friends) that were actually used during preprocessing. This patch adds that feature to `HeaderSearch` along with a new remark that reports such paths as they get used.

Previous version of this patch tried to use the existing `LookupFileCache` to report used paths via `HitIdx`. That doesn't work for `ComputeUserEntryUsage` (which is intended to be called *after* preprocessing), because it indexes used search paths by the file name. This means the values get overwritten when the code contains `#include_next`.

Note that `HeaderSearch` doesn't use `HeaderSearchOptions::UserEntries` directly. Instead, `InitHeaderSearch` pre-processes them (adds platform-specific paths, removes duplicates, removes paths that don't exist) and creates `DirectoryLookup` instances. This means we need a mechanism for translating between those two. It's not possible to go from `DirectoryLookup` back to the original `HeaderSearch`, so `InitHeaderSearch` now tracks the relationships explicitly.

Depends on D111557.

Reviewed By: dexonsmith

Differential Revision: https://reviews.llvm.org/D102923

Author: jansvoboda11

clang/include/clang/Basic/DiagnosticGroups.td

@@ -751,6 +751,7 @@ def UnusedLocalTypedef : DiagGroup<"unused-local-typedef">;
 def UnusedPropertyIvar :  DiagGroup<"unused-property-ivar">;
 def UnusedGetterReturnValue : DiagGroup<"unused-getter-return-value">;
 def UsedButMarkedUnused : DiagGroup<"used-but-marked-unused">;
+def UsedSearchPath : DiagGroup<"search-path-usage">;
 def UserDefinedLiterals : DiagGroup<"user-defined-literals">;
 def UserDefinedWarnings : DiagGroup<"user-defined-warnings">;
 def ReorderCtor : DiagGroup<"reorder-ctor">;

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -446,6 +446,9 @@ def warn_pp_hdrstop_filename_ignored : Warning<
   "#pragma hdrstop filename not supported, "
   "/Fp can be used to specify precompiled header filename">,
   InGroup<ClangClPch>;
+def remark_pp_search_path_usage : Remark<
+  "search path used: '%0'">,
+  InGroup<UsedSearchPath>;
 def err_pp_file_not_found_angled_include_not_fatal : Error<
   "'%0' file not found with <angled> %select{include|import}1; "
   "use \"quotes\" instead">;

clang/include/clang/Lex/HeaderMap.h

@@ -77,13 +77,6 @@ class HeaderMap : private HeaderMapImpl {
   static std::unique_ptr<HeaderMap> Create(const FileEntry *FE,
                                            FileManager &FM);
 
-  /// Check to see if the specified relative filename is located in this
-  /// HeaderMap.  If so, open it and return its FileEntry.  If RawPath is not
-  /// NULL and the file is found, RawPath will be set to the raw path at which
-  /// the file was found in the file system. For example, for a search path
-  /// ".." and a filename "../file.h" this would be "../../file.h".
-  Optional<FileEntryRef> LookupFile(StringRef Filename, FileManager &FM) const;
-
   using HeaderMapImpl::dump;
   using HeaderMapImpl::getFileName;
   using HeaderMapImpl::lookupFilename;

clang/include/clang/Lex/HeaderSearch.h

@@ -172,6 +172,9 @@ class HeaderSearch {
   /// Header-search options used to initialize this header search.
   std::shared_ptr<HeaderSearchOptions> HSOpts;
 
+  /// Mapping from SearchDir to HeaderSearchOptions::UserEntries indices.
+  llvm::DenseMap<unsigned, unsigned> SearchDirToHSEntry;
+
   DiagnosticsEngine &Diags;
   FileManager &FileMgr;
 
@@ -182,6 +185,9 @@ class HeaderSearch {
   /// NoCurDirSearch is true, then the check for the file in the current
   /// directory is suppressed.
   std::vector<DirectoryLookup> SearchDirs;
+  /// Whether the DirectoryLookup at the corresponding index in SearchDirs has
+  /// been successfully used to lookup a file.
+  std::vector<bool> SearchDirsUsage;
   unsigned AngledDirIdx = 0;
   unsigned SystemDirIdx = 0;
   bool NoCurDirSearch = false;
@@ -280,22 +286,25 @@ class HeaderSearch {
   DiagnosticsEngine &getDiags() const { return Diags; }
 
   /// Interface for setting the file search paths.
-  void SetSearchPaths(const std::vector<DirectoryLookup> &dirs,
-                      unsigned angledDirIdx, unsigned systemDirIdx,
-                      bool noCurDirSearch) {
+  void SetSearchPaths(std::vector<DirectoryLookup> dirs, unsigned angledDirIdx,
+                      unsigned systemDirIdx, bool noCurDirSearch,
+                      llvm::DenseMap<unsigned, unsigned> searchDirToHSEntry) {
     assert(angledDirIdx <= systemDirIdx && systemDirIdx <= dirs.size() &&
         "Directory indices are unordered");
-    SearchDirs = dirs;
+    SearchDirs = std::move(dirs);
+    SearchDirsUsage.assign(SearchDirs.size(), false);
     AngledDirIdx = angledDirIdx;
     SystemDirIdx = systemDirIdx;
     NoCurDirSearch = noCurDirSearch;
+    SearchDirToHSEntry = std::move(searchDirToHSEntry);
     //LookupFileCache.clear();
   }
 
   /// Add an additional search path.
   void AddSearchPath(const DirectoryLookup &dir, bool isAngled) {
     unsigned idx = isAngled ? SystemDirIdx : AngledDirIdx;
     SearchDirs.insert(SearchDirs.begin() + idx, dir);
+    SearchDirsUsage.insert(SearchDirsUsage.begin() + idx, false);
     if (!isAngled)
       AngledDirIdx++;
     SystemDirIdx++;
@@ -505,6 +514,10 @@ class HeaderSearch {
     return FI && FI->isImport;
   }
 
+  /// Determine which HeaderSearchOptions::UserEntries have been successfully
+  /// used so far and mark their index with 'true' in the resulting bit vector.
+  std::vector<bool> computeUserEntryUsage() const;
+
   /// This method returns a HeaderMap for the specified
   /// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
   const HeaderMap *CreateHeaderMap(const FileEntry *FE);
@@ -714,6 +727,14 @@ class HeaderSearch {
                           Module *RequestingModule,
                           ModuleMap::KnownHeader *SuggestedModule);
 
+  /// Cache the result of a successful lookup at the given include location
+  /// using the search path at index `HitIdx`.
+  void cacheLookupSuccess(LookupFileCacheInfo &CacheLookup, unsigned HitIdx,
+                          SourceLocation IncludeLoc);
+  /// Note that a lookup at the given include location was successful using the
+  /// search path at index `HitIdx`.
+  void noteLookupUsage(unsigned HitIdx, SourceLocation IncludeLoc);
+
 public:
   /// Retrieve the module map.
   ModuleMap &getModuleMap() { return ModMap; }
@@ -763,6 +784,9 @@ class HeaderSearch {
 
   search_dir_iterator system_dir_end() const { return SearchDirs.end(); }
 
+  /// Get the index of the given search directory.
+  Optional<unsigned> searchDirIdx(const DirectoryLookup &DL) const;
+
   /// Retrieve a uniqued framework name.
   StringRef getUniqueFrameworkName(StringRef Framework);
 

clang/lib/Frontend/InitHeaderSearch.cpp

@@ -36,9 +36,11 @@ namespace {
 struct DirectoryLookupInfo {
   IncludeDirGroup Group;
   DirectoryLookup Lookup;
+  Optional<unsigned> UserEntryIdx;
 
-  DirectoryLookupInfo(IncludeDirGroup Group, DirectoryLookup Lookup)
-      : Group(Group), Lookup(Lookup) {}
+  DirectoryLookupInfo(IncludeDirGroup Group, DirectoryLookup Lookup,
+                      Optional<unsigned> UserEntryIdx)
+      : Group(Group), Lookup(Lookup), UserEntryIdx(UserEntryIdx) {}
 };
 
 /// InitHeaderSearch - This class makes it easier to set the search paths of
@@ -60,13 +62,15 @@ class InitHeaderSearch {
   /// AddPath - Add the specified path to the specified group list, prefixing
   /// the sysroot if used.
   /// Returns true if the path exists, false if it was ignored.
-  bool AddPath(const Twine &Path, IncludeDirGroup Group, bool isFramework);
+  bool AddPath(const Twine &Path, IncludeDirGroup Group, bool isFramework,
+               Optional<unsigned> UserEntryIdx = None);
 
   /// AddUnmappedPath - Add the specified path to the specified group list,
   /// without performing any sysroot remapping.
   /// Returns true if the path exists, false if it was ignored.
   bool AddUnmappedPath(const Twine &Path, IncludeDirGroup Group,
-                       bool isFramework);
+                       bool isFramework,
+                       Optional<unsigned> UserEntryIdx = None);
 
   /// AddSystemHeaderPrefix - Add the specified prefix to the system header
   /// prefix list.
@@ -119,22 +123,25 @@ static bool CanPrefixSysroot(StringRef Path) {
 }
 
 bool InitHeaderSearch::AddPath(const Twine &Path, IncludeDirGroup Group,
-                               bool isFramework) {
+                               bool isFramework,
+                               Optional<unsigned> UserEntryIdx) {
   // Add the path with sysroot prepended, if desired and this is a system header
   // group.
   if (HasSysroot) {
     SmallString<256> MappedPathStorage;
     StringRef MappedPathStr = Path.toStringRef(MappedPathStorage);
     if (CanPrefixSysroot(MappedPathStr)) {
-      return AddUnmappedPath(IncludeSysroot + Path, Group, isFramework);
+      return AddUnmappedPath(IncludeSysroot + Path, Group, isFramework,
+                             UserEntryIdx);
     }
   }
 
-  return AddUnmappedPath(Path, Group, isFramework);
+  return AddUnmappedPath(Path, Group, isFramework, UserEntryIdx);
 }
 
 bool InitHeaderSearch::AddUnmappedPath(const Twine &Path, IncludeDirGroup Group,
-                                       bool isFramework) {
+                                       bool isFramework,
+                                       Optional<unsigned> UserEntryIdx) {
   assert(!Path.isTriviallyEmpty() && "can't handle empty path here");
 
   FileManager &FM = Headers.getFileMgr();
@@ -160,7 +167,8 @@ bool InitHeaderSearch::AddUnmappedPath(const Twine &Path, IncludeDirGroup Group,
 
   // If the directory exists, add it.
   if (auto DE = FM.getOptionalDirectoryRef(MappedPathStr)) {
-    IncludePath.emplace_back(Group, DirectoryLookup(*DE, Type, isFramework));
+    IncludePath.emplace_back(Group, DirectoryLookup(*DE, Type, isFramework),
+                             UserEntryIdx);
     return true;
   }
 
@@ -171,7 +179,8 @@ bool InitHeaderSearch::AddUnmappedPath(const Twine &Path, IncludeDirGroup Group,
       if (const HeaderMap *HM = Headers.CreateHeaderMap(*FE)) {
         // It is a headermap, add it to the search path.
         IncludePath.emplace_back(
-            Group, DirectoryLookup(HM, Type, Group == IndexHeaderMap));
+            Group, DirectoryLookup(HM, Type, Group == IndexHeaderMap),
+            UserEntryIdx);
         return true;
       }
     }
@@ -471,7 +480,7 @@ void InitHeaderSearch::AddDefaultIncludePaths(const LangOptions &Lang,
 /// RemoveDuplicates - If there are duplicate directory entries in the specified
 /// search list, remove the later (dead) ones.  Returns the number of non-system
 /// headers removed, which is used to update NumAngled.
-static unsigned RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
+static unsigned RemoveDuplicates(std::vector<DirectoryLookupInfo> &SearchList,
                                  unsigned First, bool Verbose) {
   llvm::SmallPtrSet<const DirectoryEntry *, 8> SeenDirs;
   llvm::SmallPtrSet<const DirectoryEntry *, 8> SeenFrameworkDirs;
@@ -480,7 +489,7 @@ static unsigned RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
   for (unsigned i = First; i != SearchList.size(); ++i) {
     unsigned DirToRemove = i;
 
-    const DirectoryLookup &CurEntry = SearchList[i];
+    const DirectoryLookup &CurEntry = SearchList[i].Lookup;
 
     if (CurEntry.isNormalDir()) {
       // If this isn't the first time we've seen this dir, remove it.
@@ -510,7 +519,7 @@ static unsigned RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
       for (FirstDir = First;; ++FirstDir) {
         assert(FirstDir != i && "Didn't find dupe?");
 
-        const DirectoryLookup &SearchEntry = SearchList[FirstDir];
+        const DirectoryLookup &SearchEntry = SearchList[FirstDir].Lookup;
 
         // If these are different lookup types, then they can't be the dupe.
         if (SearchEntry.getLookupType() != CurEntry.getLookupType())
@@ -532,7 +541,7 @@ static unsigned RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
 
       // If the first dir in the search path is a non-system dir, zap it
       // instead of the system one.
-      if (SearchList[FirstDir].getDirCharacteristic() == SrcMgr::C_User)
+      if (SearchList[FirstDir].Lookup.getDirCharacteristic() == SrcMgr::C_User)
         DirToRemove = FirstDir;
     }
 
@@ -554,24 +563,45 @@ static unsigned RemoveDuplicates(std::vector<DirectoryLookup> &SearchList,
   return NonSystemRemoved;
 }
 
+/// Extract DirectoryLookups from DirectoryLookupInfos.
+static std::vector<DirectoryLookup>
+extractLookups(const std::vector<DirectoryLookupInfo> &Infos) {
+  std::vector<DirectoryLookup> Lookups;
+  Lookups.reserve(Infos.size());
+  llvm::transform(Infos, std::back_inserter(Lookups),
+                  [](const DirectoryLookupInfo &Info) { return Info.Lookup; });
+  return Lookups;
+}
+
+/// Collect the mapping between indices of DirectoryLookups and UserEntries.
+static llvm::DenseMap<unsigned, unsigned>
+mapToUserEntries(const std::vector<DirectoryLookupInfo> &Infos) {
+  llvm::DenseMap<unsigned, unsigned> LookupsToUserEntries;
+  for (unsigned I = 0, E = Infos.size(); I < E; ++I) {
+    // Check whether this DirectoryLookup maps to a HeaderSearch::UserEntry.
+    if (Infos[I].UserEntryIdx)
+      LookupsToUserEntries.insert({I, *Infos[I].UserEntryIdx});
+  }
+  return LookupsToUserEntries;
+}
 
 void InitHeaderSearch::Realize(const LangOptions &Lang) {
   // Concatenate ANGLE+SYSTEM+AFTER chains together into SearchList.
-  std::vector<DirectoryLookup> SearchList;
+  std::vector<DirectoryLookupInfo> SearchList;
   SearchList.reserve(IncludePath.size());
 
   // Quoted arguments go first.
   for (auto &Include : IncludePath)
     if (Include.Group == Quoted)
-      SearchList.push_back(Include.Lookup);
+      SearchList.push_back(Include);
 
   // Deduplicate and remember index.
   RemoveDuplicates(SearchList, 0, Verbose);
   unsigned NumQuoted = SearchList.size();
 
   for (auto &Include : IncludePath)
     if (Include.Group == Angled || Include.Group == IndexHeaderMap)
-      SearchList.push_back(Include.Lookup);
+      SearchList.push_back(Include);
 
   RemoveDuplicates(SearchList, NumQuoted, Verbose);
   unsigned NumAngled = SearchList.size();
@@ -583,11 +613,11 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) {
          Include.Group == CXXSystem) ||
         (Lang.ObjC && !Lang.CPlusPlus && Include.Group == ObjCSystem) ||
         (Lang.ObjC && Lang.CPlusPlus && Include.Group == ObjCXXSystem))
-      SearchList.push_back(Include.Lookup);
+      SearchList.push_back(Include);
 
   for (auto &Include : IncludePath)
     if (Include.Group == After)
-      SearchList.push_back(Include.Lookup);
+      SearchList.push_back(Include);
 
   // Remove duplicates across both the Angled and System directories.  GCC does
   // this and failing to remove duplicates across these two groups breaks
@@ -596,7 +626,8 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) {
   NumAngled -= NonSystemRemoved;
 
   bool DontSearchCurDir = false;  // TODO: set to true if -I- is set?
-  Headers.SetSearchPaths(SearchList, NumQuoted, NumAngled, DontSearchCurDir);
+  Headers.SetSearchPaths(extractLookups(SearchList), NumQuoted, NumAngled,
+                         DontSearchCurDir, mapToUserEntries(SearchList));
 
   Headers.SetSystemHeaderPrefixes(SystemHeaderPrefixes);
 
@@ -606,14 +637,14 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) {
     for (unsigned i = 0, e = SearchList.size(); i != e; ++i) {
       if (i == NumQuoted)
         llvm::errs() << "#include <...> search starts here:\n";
-      StringRef Name = SearchList[i].getName();
+      StringRef Name = SearchList[i].Lookup.getName();
       const char *Suffix;
-      if (SearchList[i].isNormalDir())
+      if (SearchList[i].Lookup.isNormalDir())
         Suffix = "";
-      else if (SearchList[i].isFramework())
+      else if (SearchList[i].Lookup.isFramework())
         Suffix = " (framework directory)";
       else {
-        assert(SearchList[i].isHeaderMap() && "Unknown DirectoryLookup");
+        assert(SearchList[i].Lookup.isHeaderMap() && "Unknown DirectoryLookup");
         Suffix = " (headermap)";
       }
       llvm::errs() << " " << Name << Suffix << "\n";
@@ -632,9 +663,9 @@ void clang::ApplyHeaderSearchOptions(HeaderSearch &HS,
   for (unsigned i = 0, e = HSOpts.UserEntries.size(); i != e; ++i) {
     const HeaderSearchOptions::Entry &E = HSOpts.UserEntries[i];
     if (E.IgnoreSysRoot) {
-      Init.AddUnmappedPath(E.Path, E.Group, E.IsFramework);
+      Init.AddUnmappedPath(E.Path, E.Group, E.IsFramework, i);
     } else {
-      Init.AddPath(E.Path, E.Group, E.IsFramework);
+      Init.AddPath(E.Path, E.Group, E.IsFramework, i);
     }
   }
 

clang/lib/Lex/HeaderMap.cpp

@@ -194,19 +194,6 @@ LLVM_DUMP_METHOD void HeaderMapImpl::dump() const {
   }
 }
 
-/// LookupFile - Check to see if the specified relative filename is located in
-/// this HeaderMap.  If so, open it and return its FileEntry.
-Optional<FileEntryRef> HeaderMap::LookupFile(StringRef Filename,
-                                             FileManager &FM) const {
-
-  SmallString<1024> Path;
-  StringRef Dest = HeaderMapImpl::lookupFilename(Filename, Path);
-  if (Dest.empty())
-    return None;
-
-  return FM.getOptionalFileRef(Dest);
-}
-
 StringRef HeaderMapImpl::lookupFilename(StringRef Filename,
                                         SmallVectorImpl<char> &DestPath) const {
   const HMapHeader &Hdr = getHeader();