[index] Add macro definitions to the index store

Add support for indexing macro definitions, undefs, and references in
the index store. In the normal case of compiling source files the macros
are indexed using preprocessor callbacks that include every reference.
When we are instead indexing a module that has already been compiled
(.pcm), we get the module macros from the preprocessor, and only
one definition is found.

Macro indexing is enabled by default, but can be disabled using a new
-index-ignore-macros driver flag.

rdar://47815401

Author: benlangmuir

clang/include/clang/Driver/Options.td

@@ -620,6 +620,8 @@ def index_record_codegen_name : Flag<["-"], "index-record-codegen-name">, Flags<
   HelpText<"Record the codegen name for symbols">;
 def index_unit_output_path : Separate<["-"], "index-unit-output-path">, MetaVarName<"<path>">, Flags<[CC1Option]>,
   HelpText<"Use <path> as the output path for this compilation unit in the index unit file">;
+def index_ignore_macros : Flag<["-"], "index-ignore-macros">, Flags<[CC1Option]>,
+  HelpText<"Ignore macros during indexing">;
 
 // Make sure all other -ccc- options are rejected.
 def ccc_ : Joined<["-"], "ccc-">, Group<internal_Group>, Flags<[Unsupported]>;

clang/include/clang/Frontend/FrontendOptions.h

@@ -302,6 +302,7 @@ class FrontendOptions {
 
   unsigned IndexIgnoreSystemSymbols : 1;
   unsigned IndexRecordCodegenName : 1;
+  unsigned IndexIgnoreMacros : 1;
 
   /// Output (and read) PCM files regardless of compiler errors.
   unsigned AllowPCMWithCompilerErrors : 1;

clang/lib/Driver/ToolChains/Clang.cpp

@@ -5098,6 +5098,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     Args.AddLastArg(CmdArgs, options::OPT_index_ignore_system_symbols);
     Args.AddLastArg(CmdArgs, options::OPT_index_record_codegen_name);
     Args.AddLastArg(CmdArgs, options::OPT_index_unit_output_path);
+    Args.AddLastArg(CmdArgs, options::OPT_index_ignore_macros);
 
     // If '-o' is passed along with '-fsyntax-only' pass it along the cc1
     // invocation so that the index action knows what the out file is.

clang/lib/Driver/ToolChains/Darwin.cpp

@@ -525,6 +525,7 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   Args.ClaimAllArgs(options::OPT_index_store_path);
   Args.ClaimAllArgs(options::OPT_index_ignore_system_symbols);
   Args.ClaimAllArgs(options::OPT_index_record_codegen_name);
+  Args.ClaimAllArgs(options::OPT_index_ignore_macros);
 
   /// Hack(tm) to ignore linking errors when we are doing ARC migration.
   if (Args.hasArg(options::OPT_ccc_arcmt_check,

clang/lib/Frontend/CompilerInvocation.cpp

@@ -1894,6 +1894,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.IndexUnitOutputPath = std::string(Args.getLastArgValue(OPT_index_unit_output_path));
   Opts.IndexIgnoreSystemSymbols = Args.hasArg(OPT_index_ignore_system_symbols);
   Opts.IndexRecordCodegenName = Args.hasArg(OPT_index_record_codegen_name);
+  Opts.IndexIgnoreMacros = Args.hasArg(OPT_index_ignore_macros);
 
   InputKind DashX(Language::Unknown);
   if (const Arg *A = Args.getLastArg(OPT_x)) {

clang/lib/Index/ClangIndexRecordWriter.cpp

@@ -28,6 +28,16 @@ StringRef ClangIndexRecordWriter::getUSR(const Decl *D) {
   return Insert.first->second;
 }
 
+StringRef ClangIndexRecordWriter::getUSR(const IdentifierInfo *Name,
+                                         const MacroInfo *MI) {
+  assert(Name && MI);
+  auto Insert = USRByDecl.insert(std::make_pair(MI, StringRef()));
+  if (Insert.second) {
+    Insert.first->second = getUSRNonCached(Name, MI);
+  }
+  return Insert.first->second;
+}
+
 StringRef ClangIndexRecordWriter::getUSRNonCached(const Decl *D) {
   SmallString<256> Buf;
   bool Ignore = generateUSRForDecl(D, Buf);
@@ -39,6 +49,19 @@ StringRef ClangIndexRecordWriter::getUSRNonCached(const Decl *D) {
   return StringRef(Ptr, USR.size());
 }
 
+StringRef ClangIndexRecordWriter::getUSRNonCached(const IdentifierInfo *Name,
+                                                  const MacroInfo *MI) {
+  SmallString<256> Buf;
+  bool Ignore = generateUSRForMacro(Name->getName(), MI->getDefinitionLoc(),
+                                    Ctx.getSourceManager(), Buf);
+  if (Ignore)
+    return StringRef();
+  StringRef USR = Buf.str();
+  char *Ptr = Allocator.Allocate<char>(USR.size());
+  std::copy(USR.begin(), USR.end(), Ptr);
+  return StringRef(Ptr, USR.size());
+}
+
 ClangIndexRecordWriter::ClangIndexRecordWriter(ASTContext &Ctx,
                                                RecordingOptions Opts)
     : Impl(Opts.DataDirPath), Ctx(Ctx), RecordOpts(std::move(Opts)),
@@ -74,49 +97,63 @@ bool ClangIndexRecordWriter::writeRecord(StringRef Filename,
     return std::make_pair(LineNo, ColNo);
   };
 
+  llvm::DenseMap<const MacroInfo *, const IdentifierInfo *> MacroNames;
+
   for (auto &Occur : IdxRecord.getDeclOccurrencesSortedByOffset()) {
     unsigned Line, Col;
     std::tie(Line, Col) = getLineCol(Occur.Offset);
     SmallVector<writer::SymbolRelation, 3> Related;
     Related.reserve(Occur.Relations.size());
     for (auto &Rel : Occur.Relations)
       Related.push_back(writer::SymbolRelation{Rel.RelatedSymbol, Rel.Roles});
+    if (Occur.MacroName)
+      MacroNames[Occur.DeclOrMacro.get<const MacroInfo *>()] = Occur.MacroName;
 
-    // FIXME: handle macro occurrence
-    if (auto *D = Occur.DeclOrMacro.dyn_cast<const Decl *>())
-      Impl.addOccurrence(D, Occur.Roles, Line, Col, Related);
+    Impl.addOccurrence(Occur.DeclOrMacro.getOpaqueValue(), Occur.Roles, Line,
+                       Col, Related);
   }
 
   PrintingPolicy Policy(Ctx.getLangOpts());
   Policy.SuppressTemplateArgsInCXXConstructors = true;
 
   auto Result = Impl.endRecord(Error,
       [&](writer::OpaqueDecl OD, SmallVectorImpl<char> &Scratch) {
-    const Decl *D = static_cast<const Decl *>(OD);
-    auto Info = getSymbolInfo(D);
-
     writer::Symbol Sym;
-    Sym.SymInfo = Info;
-
-    auto *ND = dyn_cast<NamedDecl>(D);
-    if (ND) {
-      llvm::raw_svector_ostream OS(Scratch);
-      DeclarationName DeclName = ND->getDeclName();
-      if (!DeclName.isEmpty())
-        DeclName.print(OS, Policy);
+    auto DeclOrMacro =
+        llvm::PointerUnion<const Decl *, const MacroInfo *>::getFromOpaqueValue(
+            const_cast<void *>(OD));
+    if (auto *MI = DeclOrMacro.dyn_cast<const MacroInfo *>()) {
+      auto *II = MacroNames[MI];
+      assert(II);
+      Sym.SymInfo = getSymbolInfoForMacro(*MI);
+      Sym.Name = II->getName();
+      Sym.USR = getUSR(II, MI);
+      assert(!Sym.USR.empty() && "Recorded macro without USR!");
+    } else {
+      const Decl *D = DeclOrMacro.get<const Decl *>();
+      Sym.SymInfo = getSymbolInfo(D);
+
+      auto *ND = dyn_cast<NamedDecl>(D);
+      if (ND) {
+        llvm::raw_svector_ostream OS(Scratch);
+        DeclarationName DeclName = ND->getDeclName();
+        if (!DeclName.isEmpty())
+          DeclName.print(OS, Policy);
+      }
+      unsigned NameLen = Scratch.size();
+      Sym.Name = StringRef(Scratch.data(), NameLen);
+
+      Sym.USR = getUSR(D);
+      assert(!Sym.USR.empty() && "Recorded decl without USR!");
+
+      if (ASTNameGen && ND) {
+        llvm::raw_svector_ostream OS(Scratch);
+        ASTNameGen->writeName(ND, OS);
+      }
+      unsigned CGNameLen = Scratch.size() - NameLen;
+      Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen);
     }
-    unsigned NameLen = Scratch.size();
-    Sym.Name = StringRef(Scratch.data(), NameLen);
 
-    Sym.USR = getUSR(D);
-    assert(!Sym.USR.empty() && "Recorded decl without USR!");
-
-    if (ASTNameGen && ND) {
-      llvm::raw_svector_ostream OS(Scratch);
-      ASTNameGen->writeName(ND, OS);
-    }
-    unsigned CGNameLen = Scratch.size() - NameLen;
-    Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen);
     return Sym;
   });
 

clang/lib/Index/ClangIndexRecordWriter.h

@@ -19,6 +19,8 @@
 namespace clang {
   class ASTContext;
   class Decl;
+  class IdentifierInfo;
+  class MacroInfo;
 
 namespace index {
   class FileIndexRecord;
@@ -31,7 +33,7 @@ class ClangIndexRecordWriter {
 
   std::unique_ptr<ASTNameGenerator> ASTNameGen;
   llvm::BumpPtrAllocator Allocator;
-  llvm::DenseMap<const Decl *, StringRef> USRByDecl;
+  llvm::DenseMap<const void *, StringRef> USRByDecl;
   IndexRecordHasher Hasher;
 
 public:
@@ -44,9 +46,11 @@ class ClangIndexRecordWriter {
   bool writeRecord(StringRef Filename, const FileIndexRecord &Record,
                    std::string &Error, std::string *RecordFile = nullptr);
   StringRef getUSR(const Decl *D);
+  StringRef getUSR(const IdentifierInfo *Name, const MacroInfo *MI);
 
 private:
   StringRef getUSRNonCached(const Decl *D);
+  StringRef getUSRNonCached(const IdentifierInfo *Name, const MacroInfo *MI);
 };
 
 } // end namespace index

clang/lib/Index/IndexRecordHasher.cpp

@@ -26,6 +26,8 @@ using namespace llvm;
 
 static hash_code computeHash(const TemplateArgument &Arg,
                              IndexRecordHasher &Hasher);
+static hash_code hashLoc(const SourceManager &SM, SourceLocation Loc,
+                         bool IncludeOffset);
 
 namespace {
 class DeclHashVisitor : public ConstDeclVisitor<DeclHashVisitor, hash_code> {
@@ -53,7 +55,8 @@ class DeclHashVisitor : public ConstDeclVisitor<DeclHashVisitor, hash_code> {
 
       hash_code Hash = VisitDeclContext(D->getDeclContext());
       if (D->isEmbeddedInDeclarator() && !D->isFreeStanding()) {
-        COMBINE_HASH(hashLoc(D->getLocation(), /*IncludeOffset=*/true));
+        COMBINE_HASH(hashLoc(Hasher.getASTContext().getSourceManager(),
+                             D->getLocation(), /*IncludeOffset=*/true));
       } else
         COMBINE_HASH('a');
       return Hash;
@@ -127,30 +130,6 @@ class DeclHashVisitor : public ConstDeclVisitor<DeclHashVisitor, hash_code> {
     else
       return 0;
   }
-
-  hash_code hashLoc(SourceLocation Loc, bool IncludeOffset) {
-    if (Loc.isInvalid()) {
-      return 0;
-    }
-    hash_code Hash = INITIAL_HASH;
-    const SourceManager &SM = Hasher.getASTContext().getSourceManager();
-    Loc = SM.getFileLoc(Loc);
-    const std::pair<FileID, unsigned> &Decomposed = SM.getDecomposedLoc(Loc);
-    const FileEntry *FE = SM.getFileEntryForID(Decomposed.first);
-    if (FE) {
-      COMBINE_HASH(llvm::sys::path::filename(FE->getName()));
-    } else {
-      // This case really isn't interesting.
-      return 0;
-    }
-    if (IncludeOffset) {
-      // Use the offest into the FileID to represent the location.  Using
-      // a line/column can cause us to look back at the original source file,
-      // which is expensive.
-      COMBINE_HASH(Decomposed.second);
-    }
-    return Hash;
-  }
 };
 }
 
@@ -160,6 +139,9 @@ hash_code IndexRecordHasher::hashRecord(const FileIndexRecord &Record) {
     COMBINE_HASH(Info.Roles, Info.Offset);
     if (auto *D = Info.DeclOrMacro.dyn_cast<const Decl *>()) {
       COMBINE_HASH(hash(D));
+    } else {
+      COMBINE_HASH(
+          hash(Info.MacroName, Info.DeclOrMacro.get<const MacroInfo *>()));
     }
     for (auto &Rel : Info.Relations) {
       COMBINE_HASH(hash(Rel.RelatedSymbol));
@@ -168,6 +150,11 @@ hash_code IndexRecordHasher::hashRecord(const FileIndexRecord &Record) {
   return Hash;
 }
 
+hash_code IndexRecordHasher::hash(const IdentifierInfo *Name,
+                                  const MacroInfo *MI) {
+  return tryCache(MI, MacroDef{Name, MI});
+}
+
 hash_code IndexRecordHasher::hash(const Decl *D) {
   assert(D->isCanonicalDecl());
 
@@ -304,6 +291,18 @@ static hash_code computeHash(const IdentifierInfo *II) {
   return hash_value(II->getName());
 }
 
+hash_code IndexRecordHasher::hashImpl(MacroDef MD) {
+  hash_code Hash = INITIAL_HASH;
+  COMBINE_HASH(StringRef("@macro@"), computeHash(MD.Name));
+  auto &SM = getASTContext().getSourceManager();
+  auto Loc = MD.MI->getDefinitionLoc();
+  // Only hash the location if it's not in a system header, to match how
+  // USR generation behaves.
+  if (Loc.isValid() && !SM.isInSystemHeader(Loc))
+    COMBINE_HASH(hashLoc(SM, Loc, /*IncludeOffset=*/true));
+  return Hash;
+}
+
 static hash_code computeHash(Selector Sel) {
   unsigned N = Sel.getNumArgs();
   if (N == 0)
@@ -486,3 +485,27 @@ hash_code IndexRecordHasher::hashImpl(const NestedNameSpecifier *NNS) {
 
   return Hash;
 }
+
+static hash_code hashLoc(const SourceManager &SM, SourceLocation Loc,
+                         bool IncludeOffset) {
+  if (Loc.isInvalid()) {
+    return 0;
+  }
+  hash_code Hash = INITIAL_HASH;
+  Loc = SM.getFileLoc(Loc);
+  const std::pair<FileID, unsigned> &Decomposed = SM.getDecomposedLoc(Loc);
+  const FileEntry *FE = SM.getFileEntryForID(Decomposed.first);
+  if (FE) {
+    COMBINE_HASH(llvm::sys::path::filename(FE->getName()));
+  } else {
+    // This case really isn't interesting.
+    return 0;
+  }
+  if (IncludeOffset) {
+    // Use the offest into the FileID to represent the location.  Using
+    // a line/column can cause us to look back at the original source file,
+    // which is expensive.
+    COMBINE_HASH(Decomposed.second);
+  }
+  return Hash;
+}

clang/lib/Index/IndexRecordHasher.h

@@ -18,6 +18,8 @@ namespace clang {
   class ASTContext;
   class Decl;
   class DeclarationName;
+  class IdentifierInfo;
+  class MacroInfo;
   class NestedNameSpecifier;
   class QualType;
   class Type;
@@ -37,6 +39,7 @@ class IndexRecordHasher {
 
   llvm::hash_code hashRecord(const FileIndexRecord &Record);
   llvm::hash_code hash(const Decl *D);
+  llvm::hash_code hash(const IdentifierInfo *Name, const MacroInfo *M);
   llvm::hash_code hash(QualType Ty);
   llvm::hash_code hash(CanQualType Ty);
   llvm::hash_code hash(DeclarationName Name);
@@ -46,7 +49,13 @@ class IndexRecordHasher {
   template <typename T>
   llvm::hash_code tryCache(const void *Ptr, T Obj);
 
+  struct MacroDef {
+    const IdentifierInfo *Name;
+    const MacroInfo *MI;
+  };
+
   llvm::hash_code hashImpl(const Decl *D);
+  llvm::hash_code hashImpl(MacroDef MD);
   llvm::hash_code hashImpl(CanQualType Ty);
   llvm::hash_code hashImpl(DeclarationName Name);
   llvm::hash_code hashImpl(const NestedNameSpecifier *NNS);