Skip to content

Commit e35f922

Browse files
ArcsinXArcsinX
committed
[clangd] Ignore the static index refs from the dynamic index files.
This patch fixes the following problem: - open a file with references to the symbol `Foo` - remove all references to `Foo` (from the dynamic index). - `MergedIndex::refs()` result will contain positions of removed references (from the static index). The idea of this patch is to keep a set of files which were used during index build inside the index. Thus at processing the static index references we can check if the file of processing reference is a part of the dynamic index or not. Reviewed By: sammccall Differential Revision: https://reviews.llvm.org/D93393
1 parent c15c296 commit e35f922

File tree

16 files changed

+199
-20
lines changed

16 files changed

+199
-20
lines changed

‎clang-tools-extra/clangd/index/FileIndex.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -266,11 +266,14 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
266266
std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
267267
std::vector<std::shared_ptr<RefSlab>> RefSlabs;
268268
std::vector<std::shared_ptr<RelationSlab>> RelationSlabs;
269+
llvm::StringSet<> Files;
269270
std::vector<RefSlab *> MainFileRefs;
270271
{
271272
std::lock_guard<std::mutex> Lock(Mutex);
272-
for (const auto &FileAndSymbols : SymbolsSnapshot)
273+
for (const auto &FileAndSymbols : SymbolsSnapshot) {
273274
SymbolSlabs.push_back(FileAndSymbols.second);
275+
Files.insert(FileAndSymbols.first());
276+
}
274277
for (const auto &FileAndRefs : RefsSnapshot) {
275278
RefSlabs.push_back(FileAndRefs.second.Slab);
276279
if (FileAndRefs.second.CountReferences)
@@ -372,14 +375,14 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
372375
case IndexType::Light:
373376
return std::make_unique<MemIndex>(
374377
llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
375-
std::move(AllRelations),
378+
std::move(AllRelations), std::move(Files),
376379
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
377380
std::move(RefsStorage), std::move(SymsStorage)),
378381
StorageSize);
379382
case IndexType::Heavy:
380383
return std::make_unique<dex::Dex>(
381384
llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
382-
std::move(AllRelations),
385+
std::move(AllRelations), std::move(Files),
383386
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
384387
std::move(RefsStorage), std::move(SymsStorage)),
385388
StorageSize);

‎clang-tools-extra/clangd/index/Index.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ void SwapIndex::relations(
7676
return snapshot()->relations(R, CB);
7777
}
7878

79+
llvm::unique_function<bool(llvm::StringRef) const>
80+
SwapIndex::indexedFiles() const {
81+
return snapshot()->indexedFiles();
82+
}
83+
7984
size_t SwapIndex::estimateMemoryUsage() const {
8085
return snapshot()->estimateMemoryUsage();
8186
}

‎clang-tools-extra/clangd/index/Index.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "Symbol.h"
1515
#include "SymbolID.h"
1616
#include "llvm/ADT/DenseSet.h"
17+
#include "llvm/ADT/FunctionExtras.h"
1718
#include "llvm/ADT/Optional.h"
1819
#include "llvm/ADT/StringExtras.h"
1920
#include "llvm/Support/JSON.h"
@@ -121,6 +122,11 @@ class SymbolIndex {
121122
llvm::function_ref<void(const SymbolID &Subject, const Symbol &Object)>
122123
Callback) const = 0;
123124

125+
/// Returns function which checks if the specified file was used to build this
126+
/// index or not. The function must only be called while the index is alive.
127+
virtual llvm::unique_function<bool(llvm::StringRef) const>
128+
indexedFiles() const = 0;
129+
124130
/// Returns estimated size of index (in bytes).
125131
virtual size_t estimateMemoryUsage() const = 0;
126132
};
@@ -145,6 +151,9 @@ class SwapIndex : public SymbolIndex {
145151
llvm::function_ref<void(const SymbolID &, const Symbol &)>)
146152
const override;
147153

154+
llvm::unique_function<bool(llvm::StringRef) const>
155+
indexedFiles() const override;
156+
148157
size_t estimateMemoryUsage() const override;
149158

150159
private:

‎clang-tools-extra/clangd/index/MemIndex.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,18 @@ void MemIndex::relations(
109109
}
110110
}
111111

112+
llvm::unique_function<bool(llvm::StringRef) const>
113+
MemIndex::indexedFiles() const {
114+
return [this](llvm::StringRef FileURI) {
115+
auto Path = URI::resolve(FileURI);
116+
if (!Path) {
117+
llvm::consumeError(Path.takeError());
118+
return false;
119+
}
120+
return Files.contains(*Path);
121+
};
122+
}
123+
112124
size_t MemIndex::estimateMemoryUsage() const {
113125
return Index.getMemorySize() + Refs.getMemorySize() +
114126
Relations.getMemorySize() + BackingDataSize;

‎clang-tools-extra/clangd/index/MemIndex.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_MEMINDEX_H
1111

1212
#include "Index.h"
13+
#include "llvm/ADT/StringSet.h"
1314
#include <mutex>
1415

1516
namespace clang {
@@ -44,6 +45,17 @@ class MemIndex : public SymbolIndex {
4445
this->BackingDataSize = BackingDataSize;
4546
}
4647

48+
template <typename SymbolRange, typename RefRange, typename RelationRange,
49+
typename FileRange, typename Payload>
50+
MemIndex(SymbolRange &&Symbols, RefRange &&Refs, RelationRange &&Relations,
51+
FileRange &&Files, Payload &&BackingData, size_t BackingDataSize)
52+
: MemIndex(std::forward<SymbolRange>(Symbols),
53+
std::forward<RefRange>(Refs),
54+
std::forward<RelationRange>(Relations),
55+
std::forward<Payload>(BackingData), BackingDataSize) {
56+
this->Files = std::forward<FileRange>(Files);
57+
}
58+
4759
/// Builds an index from slabs. The index takes ownership of the data.
4860
static std::unique_ptr<SymbolIndex> build(SymbolSlab Symbols, RefSlab Refs,
4961
RelationSlab Relations);
@@ -62,6 +74,9 @@ class MemIndex : public SymbolIndex {
6274
llvm::function_ref<void(const SymbolID &, const Symbol &)>
6375
Callback) const override;
6476

77+
llvm::unique_function<bool(llvm::StringRef) const>
78+
indexedFiles() const override;
79+
6580
size_t estimateMemoryUsage() const override;
6681

6782
private:
@@ -73,6 +88,8 @@ class MemIndex : public SymbolIndex {
7388
static_assert(sizeof(RelationKind) == sizeof(uint8_t),
7489
"RelationKind should be of same size as a uint8_t");
7590
llvm::DenseMap<std::pair<SymbolID, uint8_t>, std::vector<SymbolID>> Relations;
91+
// Set of files which were used during this index build.
92+
llvm::StringSet<> Files;
7693
std::shared_ptr<void> KeepAlive; // poor man's move-only std::any
7794
// Size of memory retained by KeepAlive.
7895
size_t BackingDataSize = 0;

‎clang-tools-extra/clangd/index/Merge.cpp

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,23 +99,18 @@ bool MergedIndex::refs(const RefsRequest &Req,
9999
// and we can't reliably deduplicate them because offsets may differ slightly.
100100
// We consider the dynamic index authoritative and report all its refs,
101101
// and only report static index refs from other files.
102-
//
103-
// FIXME: The heuristic fails if the dynamic index contains a file, but all
104-
// refs were removed (we will report stale ones from the static index).
105-
// Ultimately we should explicit check which index has the file instead.
106-
llvm::StringSet<> DynamicIndexFileURIs;
107102
More |= Dynamic->refs(Req, [&](const Ref &O) {
108-
DynamicIndexFileURIs.insert(O.Location.FileURI);
109103
Callback(O);
110104
assert(Remaining != 0);
111105
--Remaining;
112106
});
113107
if (Remaining == 0 && More)
114108
return More;
109+
auto DynamicContainsFile = Dynamic->indexedFiles();
115110
// We return less than Req.Limit if static index returns more refs for dirty
116111
// files.
117-
bool StaticHadMore = Static->refs(Req, [&](const Ref &O) {
118-
if (DynamicIndexFileURIs.count(O.Location.FileURI))
112+
bool StaticHadMore = Static->refs(Req, [&](const Ref &O) {
113+
if (DynamicContainsFile(O.Location.FileURI))
119114
return; // ignore refs that have been seen from dynamic index.
120115
if (Remaining == 0) {
121116
More = true;
@@ -127,6 +122,14 @@ bool MergedIndex::refs(const RefsRequest &Req,
127122
return More || StaticHadMore;
128123
}
129124

125+
llvm::unique_function<bool(llvm::StringRef) const>
126+
MergedIndex::indexedFiles() const {
127+
return [DynamicContainsFile{Dynamic->indexedFiles()},
128+
StaticContainsFile{Static->indexedFiles()}](llvm::StringRef FileURI) {
129+
return DynamicContainsFile(FileURI) || StaticContainsFile(FileURI);
130+
};
131+
}
132+
130133
void MergedIndex::relations(
131134
const RelationsRequest &Req,
132135
llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {

‎clang-tools-extra/clangd/index/Merge.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ class MergedIndex : public SymbolIndex {
4545
void relations(const RelationsRequest &,
4646
llvm::function_ref<void(const SymbolID &, const Symbol &)>)
4747
const override;
48+
llvm::unique_function<bool(llvm::StringRef) const>
49+
indexedFiles() const override;
4850
size_t estimateMemoryUsage() const override {
4951
return Dynamic->estimateMemoryUsage() + Static->estimateMemoryUsage();
5052
}

‎clang-tools-extra/clangd/index/ProjectAware.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ class ProjectAwareIndex : public SymbolIndex {
5454
llvm::function_ref<void(const SymbolID &, const Symbol &)>
5555
Callback) const override;
5656

57+
llvm::unique_function<bool(llvm::StringRef) const>
58+
indexedFiles() const override;
59+
5760
ProjectAwareIndex(IndexFactory Gen) : Gen(std::move(Gen)) {}
5861

5962
private:
@@ -112,6 +115,14 @@ void ProjectAwareIndex::relations(
112115
return Idx->relations(Req, Callback);
113116
}
114117

118+
llvm::unique_function<bool(llvm::StringRef) const>
119+
ProjectAwareIndex::indexedFiles() const {
120+
trace::Span Tracer("ProjectAwareIndex::indexedFiles");
121+
if (auto *Idx = getIndex())
122+
return Idx->indexedFiles();
123+
return [](llvm::StringRef) { return false; };
124+
}
125+
115126
SymbolIndex *ProjectAwareIndex::getIndex() const {
116127
const auto &C = Config::current();
117128
if (!C.Index.External)

‎clang-tools-extra/clangd/index/dex/Dex.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,17 @@ void Dex::relations(
313313
}
314314
}
315315

316+
llvm::unique_function<bool(llvm::StringRef) const> Dex::indexedFiles() const {
317+
return [this](llvm::StringRef FileURI) {
318+
auto Path = URI::resolve(FileURI);
319+
if (!Path) {
320+
llvm::consumeError(Path.takeError());
321+
return false;
322+
}
323+
return Files.contains(*Path);
324+
};
325+
}
326+
316327
size_t Dex::estimateMemoryUsage() const {
317328
size_t Bytes = Symbols.size() * sizeof(const Symbol *);
318329
Bytes += SymbolQuality.size() * sizeof(float);

‎clang-tools-extra/clangd/index/dex/Dex.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,16 @@ class Dex : public SymbolIndex {
6767
this->BackingDataSize = BackingDataSize;
6868
}
6969

70+
template <typename SymbolRange, typename RefsRange, typename RelationsRange,
71+
typename FileRange, typename Payload>
72+
Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
73+
FileRange &&Files, Payload &&BackingData, size_t BackingDataSize)
74+
: Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),
75+
std::forward<RelationsRange>(Relations),
76+
std::forward<Payload>(BackingData), BackingDataSize) {
77+
this->Files = std::forward<FileRange>(Files);
78+
}
79+
7080
/// Builds an index from slabs. The index takes ownership of the slab.
7181
static std::unique_ptr<SymbolIndex> build(SymbolSlab, RefSlab, RelationSlab);
7282

@@ -84,6 +94,9 @@ class Dex : public SymbolIndex {
8494
llvm::function_ref<void(const SymbolID &, const Symbol &)>
8595
Callback) const override;
8696

97+
llvm::unique_function<bool(llvm::StringRef) const>
98+
indexedFiles() const override;
99+
87100
size_t estimateMemoryUsage() const override;
88101

89102
private:
@@ -112,6 +125,8 @@ class Dex : public SymbolIndex {
112125
"RelationKind should be of same size as a uint8_t");
113126
llvm::DenseMap<std::pair<SymbolID, uint8_t>, std::vector<SymbolID>> Relations;
114127
std::shared_ptr<void> KeepAlive; // poor man's move-only std::any
128+
// Set of files which were used during this index build.
129+
llvm::StringSet<> Files;
115130
// Size of memory retained by KeepAlive.
116131
size_t BackingDataSize = 0;
117132
};

‎clang-tools-extra/clangd/index/remote/Client.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,14 @@ class IndexClient : public clangd::SymbolIndex {
152152
});
153153
}
154154

155+
llvm::unique_function<bool(llvm::StringRef) const> indexedFiles() const {
156+
// FIXME: For now we always return "false" regardless of whether the file
157+
// was indexed or not. A possible implementation could be based on
158+
// the idea that we do not want to send a request at every
159+
// call of a function returned by IndexClient::indexedFiles().
160+
return [](llvm::StringRef) { return false; };
161+
}
162+
155163
// IndexClient does not take any space since the data is stored on the
156164
// server.
157165
size_t estimateMemoryUsage() const override { return 0; }

‎clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,11 @@ class IndexRequestCollector : public SymbolIndex {
13491349
llvm::function_ref<void(const SymbolID &, const Symbol &)>)
13501350
const override {}
13511351

1352+
llvm::unique_function<bool(llvm::StringRef) const>
1353+
indexedFiles() const override {
1354+
return [](llvm::StringRef) { return false; };
1355+
}
1356+
13521357
// This is incorrect, but IndexRequestCollector is not an actual index and it
13531358
// isn't used in production code.
13541359
size_t estimateMemoryUsage() const override { return 0; }

‎clang-tools-extra/clangd/unittests/DexTests.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,20 @@ TEST(DexTests, Relations) {
732732
EXPECT_THAT(Results, UnorderedElementsAre(Child1.ID, Child2.ID));
733733
}
734734

735+
TEST(DexIndex, IndexedFiles) {
736+
SymbolSlab Symbols;
737+
RefSlab Refs;
738+
auto Size = Symbols.bytes() + Refs.bytes();
739+
auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
740+
llvm::StringSet<> Files = {testPath("foo.cc"), testPath("bar.cc")};
741+
Dex I(std::move(Data.first), std::move(Data.second), RelationSlab(),
742+
std::move(Files), std::move(Data), Size);
743+
auto ContainsFile = I.indexedFiles();
744+
EXPECT_TRUE(ContainsFile("unittest:///foo.cc"));
745+
EXPECT_TRUE(ContainsFile("unittest:///bar.cc"));
746+
EXPECT_FALSE(ContainsFile("unittest:///foobar.cc"));
747+
}
748+
735749
TEST(DexTest, PreferredTypesBoosting) {
736750
auto Sym1 = symbol("t1");
737751
Sym1.Type = "T1";

0 commit comments

Comments
 (0)