Skip to content

Commit 93c87fc

Browse files
benlangmuirbenlangmuir
committed
[index] Improve macro indexing support
The major change here is to index macro occurrences in more places than before, specifically * In non-expansion references such as `#if`, `#ifdef`, etc. * When the macro is a reference to a builtin macro such as __LINE__. * When using the preprocessor state instead of callbacks, we now include all definition locations and undefinitions instead of just the latest one (which may also have had the wrong location previously). * When indexing an existing module file (.pcm), we now include module macros, and we no longer report unrelated preprocessor macros during indexing the module, which could have caused duplication. Additionally, we now correctly obey the system symbol filter for macros, so by default in system headers only definition/undefinition occurrences are reported, but it can be configured to report references as well if desired. Extends FileIndexRecord to support occurrences of macros. Since the design of this type is to keep a single list of entities organized by source location, we incorporate macros into the existing DeclOccurrence struct. Differential Revision: https://reviews.llvm.org/D99758
1 parent 9c5ebf0 commit 93c87fc

File tree

15 files changed

+285
-55
lines changed

15 files changed

+285
-55
lines changed

‎clang/include/clang/Index/DeclOccurrence.h

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,26 +9,31 @@
99
#ifndef LLVM_CLANG_INDEX_DECLOCCURRENCE_H
1010
#define LLVM_CLANG_INDEX_DECLOCCURRENCE_H
1111

12+
#include "clang/AST/DeclBase.h"
1213
#include "clang/Basic/LLVM.h"
1314
#include "clang/Index/IndexSymbol.h"
15+
#include "clang/Lex/MacroInfo.h"
1416
#include "llvm/ADT/ArrayRef.h"
17+
#include "llvm/ADT/PointerUnion.h"
1518
#include "llvm/ADT/SmallVector.h"
1619

1720
namespace clang {
18-
class Decl;
19-
2021
namespace index {
2122

2223
struct DeclOccurrence {
2324
SymbolRoleSet Roles;
2425
unsigned Offset;
25-
const Decl *Dcl;
26+
llvm::PointerUnion<const Decl *, const MacroInfo *> DeclOrMacro;
27+
const IdentifierInfo *MacroName = nullptr;
2628
SmallVector<SymbolRelation, 3> Relations;
2729

2830
DeclOccurrence(SymbolRoleSet R, unsigned Offset, const Decl *D,
2931
ArrayRef<SymbolRelation> Relations)
30-
: Roles(R), Offset(Offset), Dcl(D),
32+
: Roles(R), Offset(Offset), DeclOrMacro(D),
3133
Relations(Relations.begin(), Relations.end()) {}
34+
DeclOccurrence(SymbolRoleSet R, unsigned Offset, const IdentifierInfo *Name,
35+
const MacroInfo *MI)
36+
: Roles(R), Offset(Offset), DeclOrMacro(MI), MacroName(Name) {}
3237

3338
friend bool operator<(const DeclOccurrence &LHS, const DeclOccurrence &RHS) {
3439
return LHS.Offset < RHS.Offset;

‎clang/include/clang/Index/IndexingOptions.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ struct IndexingOptions {
2828
SystemSymbolFilterKind::DeclarationsOnly;
2929
bool IndexFunctionLocals = false;
3030
bool IndexImplicitInstantiation = false;
31+
bool IndexMacros = true;
3132
// Whether to index macro definitions in the Preprocesor when preprocessor
3233
// callback is not available (e.g. after parsing has finished). Note that
3334
// macro references are not available in Proprocessor.

‎clang/lib/Index/FileIndexRecord.cpp

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,42 +17,68 @@
1717
using namespace clang;
1818
using namespace clang::index;
1919

20-
void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
21-
const Decl *D,
22-
ArrayRef<SymbolRelation> Relations) {
23-
assert(D->isCanonicalDecl() &&
24-
"Occurrences should be associated with their canonical decl");
25-
20+
static void addOccurrence(std::vector<DeclOccurrence> &Decls,
21+
DeclOccurrence Info) {
2622
auto IsNextOccurence = [&]() -> bool {
2723
if (Decls.empty())
2824
return true;
2925
auto &Last = Decls.back();
30-
return Last.Offset < Offset;
26+
return Last.Offset < Info.Offset;
3127
};
3228

3329
if (IsNextOccurence()) {
34-
Decls.emplace_back(Roles, Offset, D, Relations);
30+
Decls.push_back(std::move(Info));
3531
return;
3632
}
3733

38-
DeclOccurrence NewInfo(Roles, Offset, D, Relations);
3934
// We keep Decls in order as we need to access them in this order in all cases.
40-
auto It = llvm::upper_bound(Decls, NewInfo);
41-
Decls.insert(It, std::move(NewInfo));
35+
auto It = llvm::upper_bound(Decls, Info);
36+
Decls.insert(It, std::move(Info));
37+
}
38+
39+
void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
40+
const Decl *D,
41+
ArrayRef<SymbolRelation> Relations) {
42+
assert(D->isCanonicalDecl() &&
43+
"Occurrences should be associated with their canonical decl");
44+
addOccurrence(Decls, DeclOccurrence(Roles, Offset, D, Relations));
4245
}
4346

44-
void FileIndexRecord::print(llvm::raw_ostream &OS) const {
47+
void FileIndexRecord::addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
48+
const IdentifierInfo *Name,
49+
const MacroInfo *MI) {
50+
addOccurrence(Decls, DeclOccurrence(Roles, Offset, Name, MI));
51+
}
52+
53+
void FileIndexRecord::removeHeaderGuardMacros() {
54+
auto It =
55+
std::remove_if(Decls.begin(), Decls.end(), [](const DeclOccurrence &D) {
56+
if (const auto *MI = D.DeclOrMacro.dyn_cast<const MacroInfo *>())
57+
return MI->isUsedForHeaderGuard();
58+
return false;
59+
});
60+
Decls.erase(It, Decls.end());
61+
}
62+
63+
void FileIndexRecord::print(llvm::raw_ostream &OS, SourceManager &SM) const {
4564
OS << "DECLS BEGIN ---\n";
4665
for (auto &DclInfo : Decls) {
47-
const Decl *D = DclInfo.Dcl;
48-
SourceManager &SM = D->getASTContext().getSourceManager();
49-
SourceLocation Loc = SM.getFileLoc(D->getLocation());
50-
PresumedLoc PLoc = SM.getPresumedLoc(Loc);
51-
OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine()
52-
<< ':' << PLoc.getColumn();
53-
54-
if (auto ND = dyn_cast<NamedDecl>(D)) {
55-
OS << ' ' << ND->getDeclName();
66+
if (const auto *D = DclInfo.DeclOrMacro.dyn_cast<const Decl *>()) {
67+
SourceLocation Loc = SM.getFileLoc(D->getLocation());
68+
PresumedLoc PLoc = SM.getPresumedLoc(Loc);
69+
OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
70+
<< PLoc.getLine() << ':' << PLoc.getColumn();
71+
72+
if (const auto *ND = dyn_cast<NamedDecl>(D)) {
73+
OS << ' ' << ND->getDeclName();
74+
}
75+
} else {
76+
const auto *MI = DclInfo.DeclOrMacro.get<const MacroInfo *>();
77+
SourceLocation Loc = SM.getFileLoc(MI->getDefinitionLoc());
78+
PresumedLoc PLoc = SM.getPresumedLoc(Loc);
79+
OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
80+
<< PLoc.getLine() << ':' << PLoc.getColumn();
81+
OS << ' ' << DclInfo.MacroName->getName();
5682
}
5783

5884
OS << '\n';

‎clang/lib/Index/FileIndexRecord.h

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,21 @@ class FileIndexRecord {
4848
/// \param Relations the set of symbols related to this occurrence.
4949
void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D,
5050
ArrayRef<SymbolRelation> Relations);
51-
void print(llvm::raw_ostream &OS) const;
51+
52+
/// Adds an occurrence of the given macro at the supplied \c Offset.
53+
///
54+
/// \param Roles the roles the occurrence fulfills in this position.
55+
/// \param Offset the offset in the file of this occurrence.
56+
/// \param Name the name of the macro.
57+
/// \param MI the canonical declaration this is an occurrence of.
58+
void addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
59+
const IdentifierInfo *Name, const MacroInfo *MI);
60+
61+
/// Remove any macro occurrences for header guards. When preprocessing, this
62+
/// will only be accurate after HandleEndOfFile.
63+
void removeHeaderGuardMacros();
64+
65+
void print(llvm::raw_ostream &OS, SourceManager &SM) const;
5266
};
5367

5468
} // end namespace index

‎clang/lib/Index/IndexingAction.cpp

Lines changed: 75 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,32 @@ class IndexPPCallbacks final : public PPCallbacks {
5151
MacroNameTok.getLocation(),
5252
*MD.getMacroInfo());
5353
}
54+
55+
void Defined(const Token &MacroNameTok, const MacroDefinition &MD,
56+
SourceRange Range) override {
57+
if (!MD.getMacroInfo()) // Ignore nonexistent macro.
58+
return;
59+
// Note: this is defined(M), not #define M
60+
IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
61+
MacroNameTok.getLocation(),
62+
*MD.getMacroInfo());
63+
}
64+
void Ifdef(SourceLocation Loc, const Token &MacroNameTok,
65+
const MacroDefinition &MD) override {
66+
if (!MD.getMacroInfo()) // Ignore non-existent macro.
67+
return;
68+
IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
69+
MacroNameTok.getLocation(),
70+
*MD.getMacroInfo());
71+
}
72+
void Ifndef(SourceLocation Loc, const Token &MacroNameTok,
73+
const MacroDefinition &MD) override {
74+
if (!MD.getMacroInfo()) // Ignore nonexistent macro.
75+
return;
76+
IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
77+
MacroNameTok.getLocation(),
78+
*MD.getMacroInfo());
79+
}
5480
};
5581

5682
class IndexASTConsumer final : public ASTConsumer {
@@ -162,23 +188,54 @@ static void indexTranslationUnit(ASTUnit &Unit, IndexingContext &IndexCtx) {
162188
Unit.visitLocalTopLevelDecls(&IndexCtx, topLevelDeclVisitor);
163189
}
164190

165-
static void indexPreprocessorMacros(const Preprocessor &PP,
191+
static void indexPreprocessorMacro(const IdentifierInfo *II,
192+
const MacroInfo *MI,
193+
MacroDirective::Kind DirectiveKind,
194+
SourceLocation Loc,
195+
IndexDataConsumer &DataConsumer) {
196+
// When using modules, it may happen that we find #undef of a macro that
197+
// was defined in another module. In such case, MI may be nullptr, since
198+
// we only look for macro definitions in the current TU. In that case,
199+
// there is nothing to index.
200+
if (!MI)
201+
return;
202+
203+
// Skip implicit visibility change.
204+
if (DirectiveKind == MacroDirective::MD_Visibility)
205+
return;
206+
207+
auto Role = DirectiveKind == MacroDirective::MD_Define
208+
? SymbolRole::Definition
209+
: SymbolRole::Undefinition;
210+
DataConsumer.handleMacroOccurrence(II, MI, static_cast<unsigned>(Role), Loc);
211+
}
212+
213+
static void indexPreprocessorMacros(Preprocessor &PP,
166214
IndexDataConsumer &DataConsumer) {
167-
for (const auto &M : PP.macros())
168-
if (MacroDirective *MD = M.second.getLatest()) {
169-
auto *MI = MD->getMacroInfo();
170-
// When using modules, it may happen that we find #undef of a macro that
171-
// was defined in another module. In such case, MI may be nullptr, since
172-
// we only look for macro definitions in the current TU. In that case,
173-
// there is nothing to index.
174-
if (!MI)
175-
continue;
176-
177-
DataConsumer.handleMacroOccurrence(
178-
M.first, MD->getMacroInfo(),
179-
static_cast<unsigned>(index::SymbolRole::Definition),
180-
MD->getLocation());
215+
for (const auto &M : PP.macros()) {
216+
for (auto *MD = M.second.getLatest(); MD; MD = MD->getPrevious()) {
217+
indexPreprocessorMacro(M.first, MD->getMacroInfo(), MD->getKind(),
218+
MD->getLocation(), DataConsumer);
219+
}
220+
}
221+
}
222+
223+
static void indexPreprocessorModuleMacros(Preprocessor &PP,
224+
serialization::ModuleFile &Mod,
225+
IndexDataConsumer &DataConsumer) {
226+
for (const auto &M : PP.macros()) {
227+
if (M.second.getLatest() == nullptr) {
228+
for (auto *MM : PP.getLeafModuleMacros(M.first)) {
229+
auto *OwningMod = MM->getOwningModule();
230+
if (OwningMod && OwningMod->getASTFile() == Mod.File) {
231+
if (auto *MI = MM->getMacroInfo()) {
232+
indexPreprocessorMacro(M.first, MI, MacroDirective::MD_Define,
233+
MI->getDefinitionLoc(), DataConsumer);
234+
}
235+
}
236+
}
181237
}
238+
}
182239
}
183240

184241
void index::indexASTUnit(ASTUnit &Unit, IndexDataConsumer &DataConsumer,
@@ -225,8 +282,9 @@ void index::indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
225282
IndexCtx.setASTContext(Ctx);
226283
DataConsumer.initialize(Ctx);
227284

228-
if (Opts.IndexMacrosInPreprocessor)
229-
indexPreprocessorMacros(Reader.getPreprocessor(), DataConsumer);
285+
if (Opts.IndexMacrosInPreprocessor) {
286+
indexPreprocessorModuleMacros(Reader.getPreprocessor(), Mod, DataConsumer);
287+
}
230288

231289
for (const Decl *D : Reader.getModuleFileLevelDecls(Mod)) {
232290
IndexCtx.indexTopLevelDecl(D);

‎clang/lib/Index/IndexingContext.cpp

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,20 +457,55 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
457457
void IndexingContext::handleMacroDefined(const IdentifierInfo &Name,
458458
SourceLocation Loc,
459459
const MacroInfo &MI) {
460+
if (!shouldIndexMacroOccurrence(/*IsRef=*/false, Loc))
461+
return;
460462
SymbolRoleSet Roles = (unsigned)SymbolRole::Definition;
461463
DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
462464
}
463465

464466
void IndexingContext::handleMacroUndefined(const IdentifierInfo &Name,
465467
SourceLocation Loc,
466468
const MacroInfo &MI) {
469+
if (!shouldIndexMacroOccurrence(/*IsRef=*/false, Loc))
470+
return;
467471
SymbolRoleSet Roles = (unsigned)SymbolRole::Undefinition;
468472
DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
469473
}
470474

471475
void IndexingContext::handleMacroReference(const IdentifierInfo &Name,
472476
SourceLocation Loc,
473477
const MacroInfo &MI) {
478+
if (!shouldIndexMacroOccurrence(/*IsRef=*/true, Loc))
479+
return;
474480
SymbolRoleSet Roles = (unsigned)SymbolRole::Reference;
475481
DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
476482
}
483+
484+
bool IndexingContext::shouldIndexMacroOccurrence(bool IsRef,
485+
SourceLocation Loc) {
486+
if (!IndexOpts.IndexMacros)
487+
return false;
488+
489+
switch (IndexOpts.SystemSymbolFilter) {
490+
case IndexingOptions::SystemSymbolFilterKind::None:
491+
break;
492+
case IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly:
493+
if (!IsRef)
494+
return true;
495+
break;
496+
case IndexingOptions::SystemSymbolFilterKind::All:
497+
return true;
498+
}
499+
500+
SourceManager &SM = Ctx->getSourceManager();
501+
FileID FID = SM.getFileID(SM.getFileLoc(Loc));
502+
if (FID.isInvalid())
503+
return false;
504+
505+
bool Invalid = false;
506+
const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid);
507+
if (Invalid || !SEntry.isFile())
508+
return false;
509+
510+
return SEntry.getFile().getFileCharacteristic() == SrcMgr::C_User;
511+
}

‎clang/lib/Index/IndexingContext.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ class IndexingContext {
124124
private:
125125
bool shouldIgnoreIfImplicit(const Decl *D);
126126

127+
bool shouldIndexMacroOccurrence(bool IsRef, SourceLocation Loc);
128+
127129
bool handleDeclOccurrence(const Decl *D, SourceLocation Loc,
128130
bool IsRef, const Decl *Parent,
129131
SymbolRoleSet Roles,

‎clang/lib/Index/USRGeneration.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,15 +1103,14 @@ bool clang::index::generateUSRForMacro(const MacroDefinitionRecord *MD,
11031103
bool clang::index::generateUSRForMacro(StringRef MacroName, SourceLocation Loc,
11041104
const SourceManager &SM,
11051105
SmallVectorImpl<char> &Buf) {
1106-
// Don't generate USRs for things with invalid locations.
1107-
if (MacroName.empty() || Loc.isInvalid())
1106+
if (MacroName.empty())
11081107
return true;
11091108

11101109
llvm::raw_svector_ostream Out(Buf);
11111110

11121111
// Assume that system headers are sane. Don't put source location
11131112
// information into the USR if the macro comes from a system header.
1114-
bool ShouldGenerateLocation = !SM.isInSystemHeader(Loc);
1113+
bool ShouldGenerateLocation = Loc.isValid() && !SM.isInSystemHeader(Loc);
11151114

11161115
Out << getUSRSpacePrefix();
11171116
if (ShouldGenerateLocation)

‎clang/test/Index/Core/Inputs/module/ModA.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11

22
void ModA_func(void);
3+
#define MODA_MACRO 1

‎clang/test/Index/Core/Inputs/module/SubModA.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11

22
void SubModA_func(void);
3+
#define SUBMODA_MACRO 1

‎clang/test/Index/Core/Inputs/sys/system-head.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,18 @@ class SubCls1 : public Cls {
3434
// CHECK-NOT: [[@LINE+1]]:3 | class/C++ | SubCls1 |
3535
SubCls1 *f;
3636
};
37+
38+
// FIXME: this decl gets reported after the macro definitions, immediately
39+
// before the next declaration. Add a dummy declaration so that the checks work.
40+
void reset_parser();
41+
42+
// CHECK: [[@LINE+1]]:9 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Def
43+
#define SYSTEM_MACRO 1
44+
// CHECK: [[@LINE+1]]:8 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Undef
45+
#undef SYSTEM_MACRO
46+
// CHECK: [[@LINE+1]]:9 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Def
47+
#define SYSTEM_MACRO int fromSystemMacro = 1
48+
49+
// CHECK-NOT: [[@LINE+2]]:1 | macro/C
50+
// CHECK: [[@LINE+1]]:1 | variable/C | fromSystemMacro
51+
SYSTEM_MACRO;

0 commit comments

Comments
 (0)