Merge pull request #2803 from benlangmuir/index-macros-20210107

[20210107][index] Add macro definitions to the index store

Author: benlangmuir

clang/include/clang/Driver/Options.td

@@ -620,6 +620,8 @@ def index_record_codegen_name : Flag<["-"], "index-record-codegen-name">, Flags<
   HelpText<"Record the codegen name for symbols">;
 def index_unit_output_path : Separate<["-"], "index-unit-output-path">, MetaVarName<"<path>">, Flags<[CC1Option]>,
   HelpText<"Use <path> as the output path for this compilation unit in the index unit file">;
+def index_ignore_macros : Flag<["-"], "index-ignore-macros">, Flags<[CC1Option]>,
+  HelpText<"Ignore macros during indexing">;
 
 // Make sure all other -ccc- options are rejected.
 def ccc_ : Joined<["-"], "ccc-">, Group<internal_Group>, Flags<[Unsupported]>;

clang/include/clang/Frontend/FrontendOptions.h

@@ -302,6 +302,7 @@ class FrontendOptions {
 
   unsigned IndexIgnoreSystemSymbols : 1;
   unsigned IndexRecordCodegenName : 1;
+  unsigned IndexIgnoreMacros : 1;
 
   /// Output (and read) PCM files regardless of compiler errors.
   unsigned AllowPCMWithCompilerErrors : 1;

clang/include/clang/Index/DeclOccurrence.h

@@ -9,26 +9,31 @@
 #ifndef LLVM_CLANG_INDEX_DECLOCCURRENCE_H
 #define LLVM_CLANG_INDEX_DECLOCCURRENCE_H
 
+#include "clang/AST/DeclBase.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Index/IndexSymbol.h"
+#include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace clang {
-class Decl;
-
 namespace index {
 
 struct DeclOccurrence {
   SymbolRoleSet Roles;
   unsigned Offset;
-  const Decl *Dcl;
+  llvm::PointerUnion<const Decl *, const MacroInfo *> DeclOrMacro;
+  const IdentifierInfo *MacroName = nullptr;
   SmallVector<SymbolRelation, 3> Relations;
 
   DeclOccurrence(SymbolRoleSet R, unsigned Offset, const Decl *D,
                  ArrayRef<SymbolRelation> Relations)
-      : Roles(R), Offset(Offset), Dcl(D),
+      : Roles(R), Offset(Offset), DeclOrMacro(D),
         Relations(Relations.begin(), Relations.end()) {}
+  DeclOccurrence(SymbolRoleSet R, unsigned Offset, const IdentifierInfo *Name,
+                 const MacroInfo *MI)
+      : Roles(R), Offset(Offset), DeclOrMacro(MI), MacroName(Name) {}
 
   friend bool operator<(const DeclOccurrence &LHS, const DeclOccurrence &RHS) {
     return LHS.Offset < RHS.Offset;

clang/include/clang/Index/IndexingOptions.h

@@ -28,6 +28,7 @@ struct IndexingOptions {
       SystemSymbolFilterKind::DeclarationsOnly;
   bool IndexFunctionLocals = false;
   bool IndexImplicitInstantiation = false;
+  bool IndexMacros = true;
   // Whether to index macro definitions in the Preprocesor when preprocessor
   // callback is not available (e.g. after parsing has finished). Note that
   // macro references are not available in Proprocessor.

clang/lib/Driver/ToolChains/Clang.cpp

@@ -5098,6 +5098,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     Args.AddLastArg(CmdArgs, options::OPT_index_ignore_system_symbols);
     Args.AddLastArg(CmdArgs, options::OPT_index_record_codegen_name);
     Args.AddLastArg(CmdArgs, options::OPT_index_unit_output_path);
+    Args.AddLastArg(CmdArgs, options::OPT_index_ignore_macros);
 
     // If '-o' is passed along with '-fsyntax-only' pass it along the cc1
     // invocation so that the index action knows what the out file is.

clang/lib/Driver/ToolChains/Darwin.cpp

@@ -525,6 +525,7 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   Args.ClaimAllArgs(options::OPT_index_store_path);
   Args.ClaimAllArgs(options::OPT_index_ignore_system_symbols);
   Args.ClaimAllArgs(options::OPT_index_record_codegen_name);
+  Args.ClaimAllArgs(options::OPT_index_ignore_macros);
 
   /// Hack(tm) to ignore linking errors when we are doing ARC migration.
   if (Args.hasArg(options::OPT_ccc_arcmt_check,

clang/lib/Frontend/CompilerInvocation.cpp

@@ -1894,6 +1894,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.IndexUnitOutputPath = std::string(Args.getLastArgValue(OPT_index_unit_output_path));
   Opts.IndexIgnoreSystemSymbols = Args.hasArg(OPT_index_ignore_system_symbols);
   Opts.IndexRecordCodegenName = Args.hasArg(OPT_index_record_codegen_name);
+  Opts.IndexIgnoreMacros = Args.hasArg(OPT_index_ignore_macros);
 
   InputKind DashX(Language::Unknown);
   if (const Arg *A = Args.getLastArg(OPT_x)) {

clang/lib/Index/ClangIndexRecordWriter.cpp

@@ -28,6 +28,16 @@ StringRef ClangIndexRecordWriter::getUSR(const Decl *D) {
   return Insert.first->second;
 }
 
+StringRef ClangIndexRecordWriter::getUSR(const IdentifierInfo *Name,
+                                         const MacroInfo *MI) {
+  assert(Name && MI);
+  auto Insert = USRByDecl.insert(std::make_pair(MI, StringRef()));
+  if (Insert.second) {
+    Insert.first->second = getUSRNonCached(Name, MI);
+  }
+  return Insert.first->second;
+}
+
 StringRef ClangIndexRecordWriter::getUSRNonCached(const Decl *D) {
   SmallString<256> Buf;
   bool Ignore = generateUSRForDecl(D, Buf);
@@ -39,6 +49,19 @@ StringRef ClangIndexRecordWriter::getUSRNonCached(const Decl *D) {
   return StringRef(Ptr, USR.size());
 }
 
+StringRef ClangIndexRecordWriter::getUSRNonCached(const IdentifierInfo *Name,
+                                                  const MacroInfo *MI) {
+  SmallString<256> Buf;
+  bool Ignore = generateUSRForMacro(Name->getName(), MI->getDefinitionLoc(),
+                                    Ctx.getSourceManager(), Buf);
+  if (Ignore)
+    return StringRef();
+  StringRef USR = Buf.str();
+  char *Ptr = Allocator.Allocate<char>(USR.size());
+  std::copy(USR.begin(), USR.end(), Ptr);
+  return StringRef(Ptr, USR.size());
+}
+
 ClangIndexRecordWriter::ClangIndexRecordWriter(ASTContext &Ctx,
                                                RecordingOptions Opts)
     : Impl(Opts.DataDirPath), Ctx(Ctx), RecordOpts(std::move(Opts)),
@@ -74,47 +97,63 @@ bool ClangIndexRecordWriter::writeRecord(StringRef Filename,
     return std::make_pair(LineNo, ColNo);
   };
 
+  llvm::DenseMap<const MacroInfo *, const IdentifierInfo *> MacroNames;
+
   for (auto &Occur : IdxRecord.getDeclOccurrencesSortedByOffset()) {
     unsigned Line, Col;
     std::tie(Line, Col) = getLineCol(Occur.Offset);
     SmallVector<writer::SymbolRelation, 3> Related;
     Related.reserve(Occur.Relations.size());
     for (auto &Rel : Occur.Relations)
       Related.push_back(writer::SymbolRelation{Rel.RelatedSymbol, Rel.Roles});
+    if (Occur.MacroName)
+      MacroNames[Occur.DeclOrMacro.get<const MacroInfo *>()] = Occur.MacroName;
 
-    Impl.addOccurrence(Occur.Dcl, Occur.Roles, Line, Col, Related);
+    Impl.addOccurrence(Occur.DeclOrMacro.getOpaqueValue(), Occur.Roles, Line,
+                       Col, Related);
   }
 
   PrintingPolicy Policy(Ctx.getLangOpts());
   Policy.SuppressTemplateArgsInCXXConstructors = true;
 
   auto Result = Impl.endRecord(Error,
       [&](writer::OpaqueDecl OD, SmallVectorImpl<char> &Scratch) {
-    const Decl *D = static_cast<const Decl *>(OD);
-    auto Info = getSymbolInfo(D);
-
     writer::Symbol Sym;
-    Sym.SymInfo = Info;
-
-    auto *ND = dyn_cast<NamedDecl>(D);
-    if (ND) {
-      llvm::raw_svector_ostream OS(Scratch);
-      DeclarationName DeclName = ND->getDeclName();
-      if (!DeclName.isEmpty())
-        DeclName.print(OS, Policy);
+    auto DeclOrMacro =
+        llvm::PointerUnion<const Decl *, const MacroInfo *>::getFromOpaqueValue(
+            const_cast<void *>(OD));
+    if (auto *MI = DeclOrMacro.dyn_cast<const MacroInfo *>()) {
+      auto *II = MacroNames[MI];
+      assert(II);
+      Sym.SymInfo = getSymbolInfoForMacro(*MI);
+      Sym.Name = II->getName();
+      Sym.USR = getUSR(II, MI);
+      assert(!Sym.USR.empty() && "Recorded macro without USR!");
+    } else {
+      const Decl *D = DeclOrMacro.get<const Decl *>();
+      Sym.SymInfo = getSymbolInfo(D);
+
+      auto *ND = dyn_cast<NamedDecl>(D);
+      if (ND) {
+        llvm::raw_svector_ostream OS(Scratch);
+        DeclarationName DeclName = ND->getDeclName();
+        if (!DeclName.isEmpty())
+          DeclName.print(OS, Policy);
+      }
+      unsigned NameLen = Scratch.size();
+      Sym.Name = StringRef(Scratch.data(), NameLen);
+
+      Sym.USR = getUSR(D);
+      assert(!Sym.USR.empty() && "Recorded decl without USR!");
+
+      if (ASTNameGen && ND) {
+        llvm::raw_svector_ostream OS(Scratch);
+        ASTNameGen->writeName(ND, OS);
+      }
+      unsigned CGNameLen = Scratch.size() - NameLen;
+      Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen);
     }
-    unsigned NameLen = Scratch.size();
-    Sym.Name = StringRef(Scratch.data(), NameLen);
 
-    Sym.USR = getUSR(D);
-    assert(!Sym.USR.empty() && "Recorded decl without USR!");
-
-    if (ASTNameGen && ND) {
-      llvm::raw_svector_ostream OS(Scratch);
-      ASTNameGen->writeName(ND, OS);
-    }
-    unsigned CGNameLen = Scratch.size() - NameLen;
-    Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen);
     return Sym;
   });
 

clang/lib/Index/ClangIndexRecordWriter.h

@@ -19,6 +19,8 @@
 namespace clang {
   class ASTContext;
   class Decl;
+  class IdentifierInfo;
+  class MacroInfo;
 
 namespace index {
   class FileIndexRecord;
@@ -31,7 +33,7 @@ class ClangIndexRecordWriter {
 
   std::unique_ptr<ASTNameGenerator> ASTNameGen;
   llvm::BumpPtrAllocator Allocator;
-  llvm::DenseMap<const Decl *, StringRef> USRByDecl;
+  llvm::DenseMap<const void *, StringRef> USRByDecl;
   IndexRecordHasher Hasher;
 
 public:
@@ -44,9 +46,11 @@ class ClangIndexRecordWriter {
   bool writeRecord(StringRef Filename, const FileIndexRecord &Record,
                    std::string &Error, std::string *RecordFile = nullptr);
   StringRef getUSR(const Decl *D);
+  StringRef getUSR(const IdentifierInfo *Name, const MacroInfo *MI);
 
 private:
   StringRef getUSRNonCached(const Decl *D);
+  StringRef getUSRNonCached(const IdentifierInfo *Name, const MacroInfo *MI);
 };
 
 } // end namespace index

clang/lib/Index/FileIndexRecord.cpp

@@ -17,42 +17,68 @@
 using namespace clang;
 using namespace clang::index;
 
-void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
-                                       const Decl *D,
-                                       ArrayRef<SymbolRelation> Relations) {
-  assert(D->isCanonicalDecl() &&
-         "Occurrences should be associated with their canonical decl");
-
+static void addOccurrence(std::vector<DeclOccurrence> &Decls,
+                          DeclOccurrence Info) {
   auto IsNextOccurence = [&]() -> bool {
     if (Decls.empty())
       return true;
     auto &Last = Decls.back();
-    return Last.Offset < Offset;
+    return Last.Offset < Info.Offset;
   };
 
   if (IsNextOccurence()) {
-    Decls.emplace_back(Roles, Offset, D, Relations);
+    Decls.push_back(std::move(Info));
     return;
   }
 
-  DeclOccurrence NewInfo(Roles, Offset, D, Relations);
   // We keep Decls in order as we need to access them in this order in all cases.
-  auto It = llvm::upper_bound(Decls, NewInfo);
-  Decls.insert(It, std::move(NewInfo));
+  auto It = llvm::upper_bound(Decls, Info);
+  Decls.insert(It, std::move(Info));
+}
+
+void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                       const Decl *D,
+                                       ArrayRef<SymbolRelation> Relations) {
+  assert(D->isCanonicalDecl() &&
+         "Occurrences should be associated with their canonical decl");
+  addOccurrence(Decls, DeclOccurrence(Roles, Offset, D, Relations));
 }
 
-void FileIndexRecord::print(llvm::raw_ostream &OS) const {
+void FileIndexRecord::addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                        const IdentifierInfo *Name,
+                                        const MacroInfo *MI) {
+  addOccurrence(Decls, DeclOccurrence(Roles, Offset, Name, MI));
+}
+
+void FileIndexRecord::removeHeaderGuardMacros() {
+  auto It =
+      std::remove_if(Decls.begin(), Decls.end(), [](const DeclOccurrence &D) {
+        if (const auto *MI = D.DeclOrMacro.dyn_cast<const MacroInfo *>())
+          return MI->isUsedForHeaderGuard();
+        return false;
+      });
+  Decls.erase(It, Decls.end());
+}
+
+void FileIndexRecord::print(llvm::raw_ostream &OS, SourceManager &SM) const {
   OS << "DECLS BEGIN ---\n";
   for (auto &DclInfo : Decls) {
-    const Decl *D = DclInfo.Dcl;
-    SourceManager &SM = D->getASTContext().getSourceManager();
-    SourceLocation Loc = SM.getFileLoc(D->getLocation());
-    PresumedLoc PLoc = SM.getPresumedLoc(Loc);
-    OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine()
-       << ':' << PLoc.getColumn();
-
-    if (auto ND = dyn_cast<NamedDecl>(D)) {
-      OS << ' ' << ND->getDeclName();
+    if (const auto *D = DclInfo.DeclOrMacro.dyn_cast<const Decl *>()) {
+      SourceLocation Loc = SM.getFileLoc(D->getLocation());
+      PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+      OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
+         << PLoc.getLine() << ':' << PLoc.getColumn();
+
+      if (const auto *ND = dyn_cast<NamedDecl>(D)) {
+        OS << ' ' << ND->getDeclName();
+      }
+    } else {
+      const auto *MI = DclInfo.DeclOrMacro.get<const MacroInfo *>();
+      SourceLocation Loc = SM.getFileLoc(MI->getDefinitionLoc());
+      PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+      OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
+         << PLoc.getLine() << ':' << PLoc.getColumn();
+      OS << ' ' << DclInfo.MacroName->getName();
     }
 
     OS << '\n';

clang/lib/Index/FileIndexRecord.h

@@ -48,7 +48,21 @@ class FileIndexRecord {
   /// \param Relations the set of symbols related to this occurrence.
   void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D,
                         ArrayRef<SymbolRelation> Relations);
-  void print(llvm::raw_ostream &OS) const;
+
+  /// Adds an occurrence of the given macro at the supplied \c Offset.
+  ///
+  /// \param Roles the roles the occurrence fulfills in this position.
+  /// \param Offset the offset in the file of this occurrence.
+  /// \param Name the name of the macro.
+  /// \param MI the canonical declaration this is an occurrence of.
+  void addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
+                         const IdentifierInfo *Name, const MacroInfo *MI);
+
+  /// Remove any macro occurrences for header guards. When preprocessing, this
+  /// will only be accurate after HandleEndOfFile.
+  void removeHeaderGuardMacros();
+
+  void print(llvm::raw_ostream &OS, SourceManager &SM) const;
 };
 
 } // end namespace index