[index] Improve macro indexing support

The major change here is to index macro occurrences in more places than
before, specifically

* In non-expansion references such as `#if`, `#ifdef`, etc.
* When the macro is a reference to a builtin macro such as __LINE__.
* When using the preprocessor state instead of callbacks, we now include
  all definition locations and undefinitions instead of just the latest
  one (which may also have had the wrong location previously).
* When indexing an existing module file (.pcm), we now include module
  macros, and we no longer report unrelated preprocessor macros during
  indexing the module, which could have caused duplication.

Additionally, we now correctly obey the system symbol filter for macros,
so by default in system headers only definition/undefinition occurrences
are reported, but it can be configured to report references as well if
desired.

Extends FileIndexRecord to support occurrences of macros. Since the
design of this type is to keep a single list of entities organized by
source location, we incorporate macros into the existing DeclOccurrence
struct.

Differential Revision: https://reviews.llvm.org/D99758

Author: benlangmuir

clang/include/clang/Index/DeclOccurrence.h

@@ -9,26 +9,31 @@
 #ifndef LLVM_CLANG_INDEX_DECLOCCURRENCE_H
 #define LLVM_CLANG_INDEX_DECLOCCURRENCE_H
 
+#include "clang/AST/DeclBase.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Index/IndexSymbol.h"
+#include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace clang {
-class Decl;
-
 namespace index {
 
 struct DeclOccurrence {
   SymbolRoleSet Roles;
   unsigned Offset;
-  const Decl *Dcl;
+  llvm::PointerUnion<const Decl *, const MacroInfo *> DeclOrMacro;
+  const IdentifierInfo *MacroName = nullptr;
   SmallVector<SymbolRelation, 3> Relations;
 
   DeclOccurrence(SymbolRoleSet R, unsigned Offset, const Decl *D,
                  ArrayRef<SymbolRelation> Relations)
-      : Roles(R), Offset(Offset), Dcl(D),
+      : Roles(R), Offset(Offset), DeclOrMacro(D),
         Relations(Relations.begin(), Relations.end()) {}
+  DeclOccurrence(SymbolRoleSet R, unsigned Offset, const IdentifierInfo *Name,
+                 const MacroInfo *MI)
+      : Roles(R), Offset(Offset), DeclOrMacro(MI), MacroName(Name) {}
 
   friend bool operator<(const DeclOccurrence &LHS, const DeclOccurrence &RHS) {
     return LHS.Offset < RHS.Offset;

clang/include/clang/Index/IndexingOptions.h

@@ -28,6 +28,7 @@ struct IndexingOptions {
       SystemSymbolFilterKind::DeclarationsOnly;
   bool IndexFunctionLocals = false;
   bool IndexImplicitInstantiation = false;
+  bool IndexMacros = true;
   // Whether to index macro definitions in the Preprocesor when preprocessor
   // callback is not available (e.g. after parsing has finished). Note that
   // macro references are not available in Proprocessor.

clang/lib/Index/FileIndexRecord.cpp

@@ -17,42 +17,68 @@
 using namespace clang;
 using namespace clang::index;
 
-void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
-                                       const Decl *D,
-                                       ArrayRef<SymbolRelation> Relations) {
-  assert(D->isCanonicalDecl() &&
-         "Occurrences should be associated with their canonical decl");
-
+static void addOccurrence(std::vector<DeclOccurrence> &Decls,
+                          DeclOccurrence Info) {
   auto IsNextOccurence = [&]() -> bool {
     if (Decls.empty())
       return true;
     auto &Last = Decls.back();
-    return Last.Offset < Offset;
+    return Last.Offset < Info.Offset;
   };
 
   if (IsNextOccurence()) {
-    Decls.emplace_back(Roles, Offset, D, Relations);
+    Decls.push_back(std::move(Info));
     return;
   }
 
-  DeclOccurrence NewInfo(Roles, Offset, D, Relations);
   // We keep Decls in order as we need to access them in this order in all cases.
-  auto It = llvm::upper_bound(Decls, NewInfo);
-  Decls.insert(It, std::move(NewInfo));
+  auto It = llvm::upper_bound(Decls, Info);
+  Decls.insert(It, std::move(Info));
+}
+
+void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                       const Decl *D,
+                                       ArrayRef<SymbolRelation> Relations) {
+  assert(D->isCanonicalDecl() &&
+         "Occurrences should be associated with their canonical decl");
+  addOccurrence(Decls, DeclOccurrence(Roles, Offset, D, Relations));
 }
 
-void FileIndexRecord::print(llvm::raw_ostream &OS) const {
+void FileIndexRecord::addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                        const IdentifierInfo *Name,
+                                        const MacroInfo *MI) {
+  addOccurrence(Decls, DeclOccurrence(Roles, Offset, Name, MI));
+}
+
+void FileIndexRecord::removeHeaderGuardMacros() {
+  auto It =
+      std::remove_if(Decls.begin(), Decls.end(), [](const DeclOccurrence &D) {
+        if (const auto *MI = D.DeclOrMacro.dyn_cast<const MacroInfo *>())
+          return MI->isUsedForHeaderGuard();
+        return false;
+      });
+  Decls.erase(It, Decls.end());
+}
+
+void FileIndexRecord::print(llvm::raw_ostream &OS, SourceManager &SM) const {
   OS << "DECLS BEGIN ---\n";
   for (auto &DclInfo : Decls) {
-    const Decl *D = DclInfo.Dcl;
-    SourceManager &SM = D->getASTContext().getSourceManager();
-    SourceLocation Loc = SM.getFileLoc(D->getLocation());
-    PresumedLoc PLoc = SM.getPresumedLoc(Loc);
-    OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine()
-       << ':' << PLoc.getColumn();
-
-    if (auto ND = dyn_cast<NamedDecl>(D)) {
-      OS << ' ' << ND->getDeclName();
+    if (const auto *D = DclInfo.DeclOrMacro.dyn_cast<const Decl *>()) {
+      SourceLocation Loc = SM.getFileLoc(D->getLocation());
+      PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+      OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
+         << PLoc.getLine() << ':' << PLoc.getColumn();
+
+      if (const auto *ND = dyn_cast<NamedDecl>(D)) {
+        OS << ' ' << ND->getDeclName();
+      }
+    } else {
+      const auto *MI = DclInfo.DeclOrMacro.get<const MacroInfo *>();
+      SourceLocation Loc = SM.getFileLoc(MI->getDefinitionLoc());
+      PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+      OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
+         << PLoc.getLine() << ':' << PLoc.getColumn();
+      OS << ' ' << DclInfo.MacroName->getName();
     }
 
     OS << '\n';

clang/lib/Index/FileIndexRecord.h

@@ -48,7 +48,21 @@ class FileIndexRecord {
   /// \param Relations the set of symbols related to this occurrence.
   void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D,
                         ArrayRef<SymbolRelation> Relations);
-  void print(llvm::raw_ostream &OS) const;
+
+  /// Adds an occurrence of the given macro at the supplied \c Offset.
+  ///
+  /// \param Roles the roles the occurrence fulfills in this position.
+  /// \param Offset the offset in the file of this occurrence.
+  /// \param Name the name of the macro.
+  /// \param MI the canonical declaration this is an occurrence of.
+  void addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
+                         const IdentifierInfo *Name, const MacroInfo *MI);
+
+  /// Remove any macro occurrences for header guards. When preprocessing, this
+  /// will only be accurate after HandleEndOfFile.
+  void removeHeaderGuardMacros();
+
+  void print(llvm::raw_ostream &OS, SourceManager &SM) const;
 };
 
 } // end namespace index

clang/lib/Index/IndexingAction.cpp

@@ -57,6 +57,32 @@ class IndexPPCallbacks final : public PPCallbacks {
                                    MacroNameTok.getLocation(),
                                    *MD.getMacroInfo());
   }
+
+  void Defined(const Token &MacroNameTok, const MacroDefinition &MD,
+               SourceRange Range) override {
+    if (!MD.getMacroInfo()) // Ignore nonexistent macro.
+      return;
+    // Note: this is defined(M), not #define M
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
+  void Ifdef(SourceLocation Loc, const Token &MacroNameTok,
+             const MacroDefinition &MD) override {
+    if (!MD.getMacroInfo()) // Ignore non-existent macro.
+      return;
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
+  void Ifndef(SourceLocation Loc, const Token &MacroNameTok,
+              const MacroDefinition &MD) override {
+    if (!MD.getMacroInfo()) // Ignore nonexistent macro.
+      return;
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
 };
 
 class IndexASTConsumer final : public ASTConsumer {
@@ -168,23 +194,54 @@ static void indexTranslationUnit(ASTUnit &Unit, IndexingContext &IndexCtx) {
   Unit.visitLocalTopLevelDecls(&IndexCtx, topLevelDeclVisitor);
 }
 
-static void indexPreprocessorMacros(const Preprocessor &PP,
+static void indexPreprocessorMacro(const IdentifierInfo *II,
+                                   const MacroInfo *MI,
+                                   MacroDirective::Kind DirectiveKind,
+                                   SourceLocation Loc,
+                                   IndexDataConsumer &DataConsumer) {
+  // When using modules, it may happen that we find #undef of a macro that
+  // was defined in another module. In such case, MI may be nullptr, since
+  // we only look for macro definitions in the current TU. In that case,
+  // there is nothing to index.
+  if (!MI)
+    return;
+
+  // Skip implicit visibility change.
+  if (DirectiveKind == MacroDirective::MD_Visibility)
+    return;
+
+  auto Role = DirectiveKind == MacroDirective::MD_Define
+                  ? SymbolRole::Definition
+                  : SymbolRole::Undefinition;
+  DataConsumer.handleMacroOccurrence(II, MI, static_cast<unsigned>(Role), Loc);
+}
+
+static void indexPreprocessorMacros(Preprocessor &PP,
                                     IndexDataConsumer &DataConsumer) {
-  for (const auto &M : PP.macros())
-    if (MacroDirective *MD = M.second.getLatest()) {
-      auto *MI = MD->getMacroInfo();
-      // When using modules, it may happen that we find #undef of a macro that
-      // was defined in another module. In such case, MI may be nullptr, since
-      // we only look for macro definitions in the current TU. In that case,
-      // there is nothing to index.
-      if (!MI)
-        continue;
-
-      DataConsumer.handleMacroOccurrence(
-          M.first, MD->getMacroInfo(),
-          static_cast<unsigned>(index::SymbolRole::Definition),
-          MD->getLocation());
+  for (const auto &M : PP.macros()) {
+    for (auto *MD = M.second.getLatest(); MD; MD = MD->getPrevious()) {
+      indexPreprocessorMacro(M.first, MD->getMacroInfo(), MD->getKind(),
+                             MD->getLocation(), DataConsumer);
     }
+  }
+}
+
+static void indexPreprocessorModuleMacros(Preprocessor &PP,
+                                          serialization::ModuleFile &Mod,
+                                          IndexDataConsumer &DataConsumer) {
+  for (const auto &M : PP.macros()) {
+    if (M.second.getLatest() == nullptr) {
+      for (auto *MM : PP.getLeafModuleMacros(M.first)) {
+        auto *OwningMod = MM->getOwningModule();
+        if (OwningMod && OwningMod->getASTFile() == Mod.File) {
+          if (auto *MI = MM->getMacroInfo()) {
+            indexPreprocessorMacro(M.first, MI, MacroDirective::MD_Define,
+                                   MI->getDefinitionLoc(), DataConsumer);
+          }
+        }
+      }
+    }
+  }
 }
 
 void index::indexASTUnit(ASTUnit &Unit, IndexDataConsumer &DataConsumer,
@@ -231,8 +288,9 @@ void index::indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
   IndexCtx.setASTContext(Ctx);
   DataConsumer.initialize(Ctx);
 
-  if (Opts.IndexMacrosInPreprocessor)
-    indexPreprocessorMacros(Reader.getPreprocessor(), DataConsumer);
+  if (Opts.IndexMacrosInPreprocessor) {
+    indexPreprocessorModuleMacros(Reader.getPreprocessor(), Mod, DataConsumer);
+  }
 
   for (const Decl *D : Reader.getModuleFileLevelDecls(Mod)) {
     IndexCtx.indexTopLevelDecl(D);

clang/lib/Index/IndexingContext.cpp

@@ -495,20 +495,55 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
 void IndexingContext::handleMacroDefined(const IdentifierInfo &Name,
                                          SourceLocation Loc,
                                          const MacroInfo &MI) {
+  if (!shouldIndexMacroOccurrence(/*IsRef=*/false, Loc))
+    return;
   SymbolRoleSet Roles = (unsigned)SymbolRole::Definition;
   DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
 }
 
 void IndexingContext::handleMacroUndefined(const IdentifierInfo &Name,
                                            SourceLocation Loc,
                                            const MacroInfo &MI) {
+  if (!shouldIndexMacroOccurrence(/*IsRef=*/false, Loc))
+    return;
   SymbolRoleSet Roles = (unsigned)SymbolRole::Undefinition;
   DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
 }
 
 void IndexingContext::handleMacroReference(const IdentifierInfo &Name,
                                            SourceLocation Loc,
                                            const MacroInfo &MI) {
+  if (!shouldIndexMacroOccurrence(/*IsRef=*/true, Loc))
+    return;
   SymbolRoleSet Roles = (unsigned)SymbolRole::Reference;
   DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
 }
+
+bool IndexingContext::shouldIndexMacroOccurrence(bool IsRef,
+                                                 SourceLocation Loc) {
+  if (!IndexOpts.IndexMacros)
+    return false;
+
+  switch (IndexOpts.SystemSymbolFilter) {
+  case IndexingOptions::SystemSymbolFilterKind::None:
+    break;
+  case IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly:
+    if (!IsRef)
+      return true;
+    break;
+  case IndexingOptions::SystemSymbolFilterKind::All:
+    return true;
+  }
+
+  SourceManager &SM = Ctx->getSourceManager();
+  FileID FID = SM.getFileID(SM.getFileLoc(Loc));
+  if (FID.isInvalid())
+    return false;
+
+  bool Invalid = false;
+  const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid);
+  if (Invalid || !SEntry.isFile())
+    return false;
+
+  return SEntry.getFile().getFileCharacteristic() == SrcMgr::C_User;
+}

clang/lib/Index/IndexingContext.h

@@ -137,6 +137,8 @@ class IndexingContext {
 private:
   bool shouldIgnoreIfImplicit(const Decl *D);
 
+  bool shouldIndexMacroOccurrence(bool IsRef, SourceLocation Loc);
+
   bool handleDeclOccurrence(const Decl *D, SourceLocation Loc,
                             bool IsRef, const Decl *Parent,
                             SymbolRoleSet Roles,

clang/lib/Index/USRGeneration.cpp

@@ -1101,15 +1101,14 @@ bool clang::index::generateUSRForMacro(const MacroDefinitionRecord *MD,
 bool clang::index::generateUSRForMacro(StringRef MacroName, SourceLocation Loc,
                                        const SourceManager &SM,
                                        SmallVectorImpl<char> &Buf) {
-  // Don't generate USRs for things with invalid locations.
-  if (MacroName.empty() || Loc.isInvalid())
+  if (MacroName.empty())
     return true;
 
   llvm::raw_svector_ostream Out(Buf);
 
   // Assume that system headers are sane.  Don't put source location
   // information into the USR if the macro comes from a system header.
-  bool ShouldGenerateLocation = !SM.isInSystemHeader(Loc);
+  bool ShouldGenerateLocation = Loc.isValid() && !SM.isInSystemHeader(Loc);
 
   Out << getUSRSpacePrefix();
   if (ShouldGenerateLocation)

clang/test/Index/Core/Inputs/module/ModA.h

@@ -1,2 +1,3 @@
 
 void ModA_func(void);
+#define MODA_MACRO 1

clang/test/Index/Core/Inputs/module/SubModA.h

@@ -1,2 +1,3 @@
 
 void SubModA_func(void);
+#define SUBMODA_MACRO 1

clang/test/Index/Core/Inputs/sys/system-head.h

@@ -34,3 +34,18 @@ class SubCls1 : public Cls {
   // CHECK-NOT: [[@LINE+1]]:3 | class/C++ | SubCls1 |
   SubCls1 *f;
 };
+
+// FIXME: this decl gets reported after the macro definitions, immediately
+// before the next declaration. Add a dummy declaration so that the checks work.
+void reset_parser();
+
+// CHECK: [[@LINE+1]]:9 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Def
+#define SYSTEM_MACRO 1
+// CHECK: [[@LINE+1]]:8 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Undef
+#undef SYSTEM_MACRO
+// CHECK: [[@LINE+1]]:9 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Def
+#define SYSTEM_MACRO int fromSystemMacro = 1
+
+// CHECK-NOT: [[@LINE+2]]:1 | macro/C
+// CHECK: [[@LINE+1]]:1 | variable/C | fromSystemMacro
+SYSTEM_MACRO;