Skip to content

Commit 419ba04

Browse files
ahoppenahoppen
committed
[libSyntax] Record reused node IDs
This is cheaper than recording reused region offsets and the reused node IDs will later be used to incrementally transfer the syntax to SwiftSyntax.
1 parent 705f5b7 commit 419ba04

File tree

7 files changed

+106
-86
lines changed

7 files changed

+106
-86
lines changed

‎include/swift/Parse/SyntaxParsingCache.h

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "swift/Syntax/SyntaxNodes.h"
1717
#include "llvm/Support/FileSystem.h"
1818
#include "llvm/Support/raw_ostream.h"
19+
#include <unordered_set>
1920

2021
namespace {
2122

@@ -48,10 +49,8 @@ namespace swift {
4849
using namespace swift::syntax;
4950

5051
struct SyntaxReuseRegion {
51-
/// The byte offset at which the range begins
52-
uintptr_t Start;
53-
/// The byte offset at which the end ends
54-
uintptr_t End;
52+
AbsolutePosition Start;
53+
AbsolutePosition End;
5554
};
5655

5756
class SyntaxParsingCache {
@@ -62,13 +61,8 @@ class SyntaxParsingCache {
6261
/// the source file that is now parsed incrementally
6362
llvm::SmallVector<SourceEdit, 4> Edits;
6463

65-
/// Whether or not information about reused nodes shall be recored in
66-
/// \c ReusedRanges
67-
bool RecordReuseInformation = false;
68-
69-
/// If \c RecordReuseInformation buffer offsets of ranges that have been
70-
/// successfully looked up in this cache are stored.
71-
std::vector<SyntaxReuseRegion> ReusedRanges;
64+
/// The IDs of all syntax nodes that got reused are collected in this vector.
65+
std::unordered_set<unsigned> ReusedNodeIds;
7266

7367
public:
7468
SyntaxParsingCache(SourceFileSyntax OldSyntaxTree)
@@ -86,16 +80,15 @@ class SyntaxParsingCache {
8680
/// reused for a new syntax tree.
8781
llvm::Optional<Syntax> lookUp(size_t NewPosition, SyntaxKind Kind);
8882

89-
/// Turn recording of reused ranges on
90-
void setRecordReuseInformation() { RecordReuseInformation = true; }
91-
92-
/// Return the ranges of the new source file that have been successfully
93-
/// looked up in this cache as a (start, end) pair of byte offsets in the
94-
/// post-edit file.
95-
std::vector<SyntaxReuseRegion> getReusedRanges() const {
96-
return ReusedRanges;
83+
const std::unordered_set<unsigned> &getReusedNodeIds() const {
84+
return ReusedNodeIds;
9785
}
9886

87+
/// Get the source regions of the new source file, represented by
88+
/// \p SyntaxTree that have been reused as part of the incremental parse.
89+
std::vector<SyntaxReuseRegion>
90+
getReusedRegions(const SourceFileSyntax &SyntaxTree) const;
91+
9992
private:
10093
llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t Position,
10194
SyntaxKind Kind);

‎include/swift/Syntax/RawSyntax.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,4 +523,8 @@ class RawSyntax final
523523
} // end namespace syntax
524524
} // end namespace swift
525525

526+
namespace llvm {
527+
raw_ostream &operator<<(raw_ostream &OS, swift::syntax::AbsolutePosition Pos);
528+
} // end namespace llvm
529+
526530
#endif // SWIFT_SYNTAX_RAWSYNTAX_H

‎lib/Parse/SyntaxParsingCache.cpp

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//===----------------------------------------------------------------------===//
1212

1313
#include "swift/Parse/SyntaxParsingCache.h"
14+
#include "swift/Syntax/SyntaxVisitor.h"
1415

1516
using namespace swift;
1617
using namespace swift::syntax;
@@ -84,10 +85,54 @@ llvm::Optional<Syntax> SyntaxParsingCache::lookUp(size_t NewPosition,
8485

8586
auto Node = lookUpFrom(OldSyntaxTree, OldPosition, Kind);
8687
if (Node.hasValue()) {
87-
if (RecordReuseInformation) {
88-
ReusedRanges.push_back(
89-
{NewPosition, NewPosition + Node->getTextLength()});
90-
}
88+
ReusedNodeIds.insert(Node->getId());
9189
}
9290
return Node;
9391
}
92+
93+
std::vector<SyntaxReuseRegion>
94+
SyntaxParsingCache::getReusedRegions(const SourceFileSyntax &SyntaxTree) const {
95+
/// Determines the reused source regions from reused syntax node IDs
96+
class ReusedRegionsCollector : public SyntaxVisitor {
97+
std::unordered_set<unsigned> ReusedNodeIds;
98+
std::vector<SyntaxReuseRegion> ReusedRegions;
99+
100+
bool didReuseNode(unsigned NodeId) {
101+
return ReusedNodeIds.count(NodeId) > 0;
102+
}
103+
104+
public:
105+
ReusedRegionsCollector(std::unordered_set<unsigned> ReusedNodeIds)
106+
: ReusedNodeIds(ReusedNodeIds) {}
107+
108+
const std::vector<SyntaxReuseRegion> &getReusedRegions() {
109+
std::sort(ReusedRegions.begin(), ReusedRegions.end(),
110+
[](const SyntaxReuseRegion &Lhs,
111+
const SyntaxReuseRegion &Rhs) -> bool {
112+
return Lhs.Start.getOffset() < Rhs.Start.getOffset();
113+
});
114+
return ReusedRegions;
115+
}
116+
117+
void visit(Syntax Node) override {
118+
if (didReuseNode(Node.getId())) {
119+
// Node has been reused, add it to the list
120+
auto Start = Node.getAbsolutePositionBeforeLeadingTrivia();
121+
auto End = Node.getAbsoluteEndPositionAfterTrailingTrivia();
122+
ReusedRegions.push_back({Start, End});
123+
} else {
124+
SyntaxVisitor::visit(Node);
125+
}
126+
}
127+
128+
void collectReusedRegions(SourceFileSyntax Node) {
129+
assert(ReusedRegions.empty() &&
130+
"ReusedRegionsCollector cannot be reused");
131+
Node.accept(*this);
132+
}
133+
};
134+
135+
ReusedRegionsCollector ReuseRegionsCollector(getReusedNodeIds());
136+
ReuseRegionsCollector.collectReusedRegions(SyntaxTree);
137+
return ReuseRegionsCollector.getReusedRegions();
138+
}

‎lib/Syntax/RawSyntax.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,3 +343,8 @@ void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
343343
for (auto &Piece : TrailingTrivia)
344344
Piece.Profile(ID);
345345
}
346+
347+
llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, AbsolutePosition Pos) {
348+
Pos.printLineAndColumn(OS);
349+
return OS;
350+
}

‎tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

Lines changed: 11 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2013,18 +2013,6 @@ SwiftEditorDocument::getSyntaxTree() const {
20132013
return Impl.SyntaxTree;
20142014
}
20152015

2016-
const SourceManager &SwiftEditorDocument::getSourceManager() const {
2017-
return Impl.SyntaxInfo->getSourceManager();
2018-
}
2019-
2020-
SourceManager &SwiftEditorDocument::getSourceManager() {
2021-
return Impl.SyntaxInfo->getSourceManager();
2022-
}
2023-
2024-
unsigned SwiftEditorDocument::getBufferID() const {
2025-
return Impl.SyntaxInfo->getBufferID();
2026-
}
2027-
20282016
std::string SwiftEditorDocument::getFilePath() const { return Impl.FilePath; }
20292017

20302018
bool SwiftEditorDocument::hasUpToDateAST() const {
@@ -2371,7 +2359,6 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
23712359
if (EditorDoc->getSyntaxTree().hasValue()) {
23722360
SyntaxCache.emplace(EditorDoc->getSyntaxTree().getValue());
23732361
SyntaxCache->addEdit(Offset, Offset + Length, Buf->getBufferSize());
2374-
SyntaxCache->setRecordReuseInformation();
23752362
}
23762363

23772364
SyntaxParsingCache *SyntaxCachePtr = nullptr;
@@ -2386,37 +2373,35 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
23862373
// Avoid computing the reused ranges if the consumer doesn't care about
23872374
// them
23882375
if (Consumer.syntaxReuseInfoEnabled()) {
2389-
auto ReuseRegions = SyntaxCache->getReusedRanges();
2376+
auto &SyntaxTree = EditorDoc->getSyntaxTree();
2377+
auto ReuseRegions = SyntaxCache->getReusedRegions(*SyntaxTree);
2378+
2379+
// Abstract away from SyntaxReuseRegions to std::pair<unsigned, unsigned>
2380+
// so that SourceKit doesn't have to import swiftParse
23902381
std::vector<SourceFileRange> ReuseRegionOffsets;
23912382
ReuseRegionOffsets.reserve(ReuseRegions.size());
23922383
for (auto ReuseRegion : ReuseRegions) {
2393-
auto Start = ReuseRegion.Start;
2394-
auto End = ReuseRegion.End;
2384+
auto Start = ReuseRegion.Start.getOffset();
2385+
auto End = ReuseRegion.End.getOffset();
23952386
ReuseRegionOffsets.push_back({Start, End});
23962387
}
23972388
Consumer.handleSyntaxReuseRegions(ReuseRegionOffsets);
23982389
}
23992390
if (LogReuseRegions) {
2391+
auto &SyntaxTree = EditorDoc->getSyntaxTree();
2392+
auto ReuseRegions = SyntaxCache->getReusedRegions(*SyntaxTree);
24002393
LOG_SECTION("SyntaxCache", InfoHighPrio) {
24012394
Log->getOS() << "Reused ";
24022395

24032396
bool FirstIteration = true;
2404-
unsigned LastPrintedBufferID;
2405-
for (auto ReuseRegion : SyntaxCache->getReusedRanges()) {
2397+
for (auto ReuseRegion : ReuseRegions) {
24062398
if (!FirstIteration) {
24072399
Log->getOS() << ", ";
24082400
} else {
24092401
FirstIteration = false;
24102402
}
24112403

2412-
const SourceManager &SM = EditorDoc->getSourceManager();
2413-
unsigned BufferID = EditorDoc->getBufferID();
2414-
auto Start = SM.getLocForOffset(BufferID, ReuseRegion.Start);
2415-
auto End = SM.getLocForOffset(BufferID, ReuseRegion.End);
2416-
2417-
Start.print(Log->getOS(), SM, LastPrintedBufferID);
2418-
Log->getOS() << " - ";
2419-
End.print(Log->getOS(), SM, LastPrintedBufferID);
2404+
Log->getOS() << ReuseRegion.Start << " - " << ReuseRegion.End;
24202405
}
24212406
}
24222407
}

‎tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,12 +110,6 @@ class SwiftEditorDocument :
110110

111111
const llvm::Optional<swift::SourceFileSyntax> &getSyntaxTree() const;
112112

113-
const swift::SourceManager &getSourceManager() const;
114-
swift::SourceManager &getSourceManager();
115-
116-
/// Get the buffer ID of this file in its source manager
117-
unsigned getBufferID() const;
118-
119113
std::string getFilePath() const;
120114

121115
/// Whether or not the AST stored for this document is up-to-date or just an

‎tools/swift-syntax-test/swift-syntax-test.cpp

Lines changed: 25 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -192,14 +192,13 @@ namespace {
192192
// A utility class to wrap a source range consisting of a byte start and end
193193
// offset
194194
struct ByteBasedSourceRange {
195-
unsigned Start;
196-
unsigned End;
195+
uintptr_t Start;
196+
uintptr_t End;
197197

198-
ByteBasedSourceRange(unsigned Start, unsigned End) : Start(Start), End(End) {
198+
ByteBasedSourceRange(uintptr_t Start, uintptr_t End)
199+
: Start(Start), End(End) {
199200
assert(Start <= End);
200201
}
201-
ByteBasedSourceRange(SyntaxReuseRegion Pair)
202-
: ByteBasedSourceRange(Pair.Start, Pair.End) {}
203202
ByteBasedSourceRange() : ByteBasedSourceRange(0, 0) {}
204203

205204
ByteBasedSourceRange intersect(const ByteBasedSourceRange &Other) {
@@ -228,9 +227,9 @@ struct ByteBasedSourceRangeSet {
228227

229228
ByteBasedSourceRangeSet() {}
230229

231-
ByteBasedSourceRangeSet(std::vector<SyntaxReuseRegion> PairVector) {
232-
for (auto Pair : PairVector) {
233-
addRange(Pair);
230+
ByteBasedSourceRangeSet(std::vector<SyntaxReuseRegion> Ranges) {
231+
for (auto Range : Ranges) {
232+
addRange({Range.Start.getOffset(), Range.End.getOffset()});
234233
}
235234
}
236235

@@ -410,7 +409,8 @@ bool useColoredOutput() {
410409

411410
void printVisualNodeReuseInformation(SourceManager &SourceMgr,
412411
unsigned BufferID,
413-
SyntaxParsingCache *Cache) {
412+
SyntaxParsingCache *Cache,
413+
const SourceFileSyntax &NewSyntaxTree) {
414414
unsigned CurrentOffset = 0;
415415
auto SourceText = SourceMgr.getEntireTextForBuffer(BufferID);
416416
if (useColoredOutput()) {
@@ -436,13 +436,14 @@ void printVisualNodeReuseInformation(SourceManager &SourceMgr,
436436
}
437437
};
438438

439-
for (auto ReuseRange : Cache->getReusedRanges()) {
439+
for (auto ReuseRange : Cache->getReusedRegions(NewSyntaxTree)) {
440+
auto StartOffset = ReuseRange.Start.getOffset();
441+
auto EndOffset = ReuseRange.End.getOffset();
440442
// Print region that was not reused
441-
PrintReparsedRegion(SourceText, CurrentOffset, ReuseRange.Start);
443+
PrintReparsedRegion(SourceText, CurrentOffset, StartOffset);
442444

443-
llvm::outs() << SourceText.substr(ReuseRange.Start,
444-
ReuseRange.End - ReuseRange.Start);
445-
CurrentOffset = ReuseRange.End;
445+
llvm::outs() << SourceText.substr(StartOffset, EndOffset - StartOffset);
446+
CurrentOffset = EndOffset;
446447
}
447448
PrintReparsedRegion(SourceText, CurrentOffset, SourceText.size());
448449
if (useColoredOutput())
@@ -451,21 +452,16 @@ void printVisualNodeReuseInformation(SourceManager &SourceMgr,
451452
llvm::outs() << '\n';
452453
}
453454

454-
void saveReuseLog(SourceManager &SourceMgr, unsigned BufferID,
455-
SyntaxParsingCache *Cache) {
455+
void saveReuseLog(SyntaxParsingCache *Cache,
456+
const SourceFileSyntax &NewSyntaxTree) {
456457
std::error_code ErrorCode;
457458
llvm::raw_fd_ostream ReuseLog(options::IncrementalReuseLog, ErrorCode,
458459
llvm::sys::fs::OpenFlags::F_RW);
459460
assert(!ErrorCode && "Unable to open incremental usage log");
460461

461-
for (auto ReuseRange : Cache->getReusedRanges()) {
462-
SourceLoc Start = SourceMgr.getLocForOffset(BufferID, ReuseRange.Start);
463-
SourceLoc End = SourceMgr.getLocForOffset(BufferID, ReuseRange.End);
464-
465-
ReuseLog << "Reused ";
466-
Start.printLineAndColumn(ReuseLog, SourceMgr, BufferID);
467-
ReuseLog << " to ";
468-
End.printLineAndColumn(ReuseLog, SourceMgr, BufferID);
462+
for (auto ReuseRange : Cache->getReusedRegions(NewSyntaxTree)) {
463+
ReuseLog << "Reused " << ReuseRange.Start << " to " << ReuseRange.End
464+
<< '\n';
469465
ReuseLog << '\n';
470466
}
471467
}
@@ -494,7 +490,8 @@ bool verifyReusedRegions(ByteBasedSourceRangeSet ExpectedReparseRegions,
494490
auto FileLength = SourceMgr.getRangeForBuffer(BufferID).getByteLength();
495491

496492
// Compute the repared regions by inverting the reused regions
497-
auto ReusedRanges = ByteBasedSourceRangeSet(SyntaxCache->getReusedRanges());
493+
auto ReusedRanges = ByteBasedSourceRangeSet(
494+
SyntaxCache->getReusedRegions(SF->getSyntaxRoot()));
498495
auto ReparsedRegions = ReusedRanges.invert(FileLength);
499496

500497
// Same for expected reuse regions
@@ -552,8 +549,6 @@ int parseFile(const char *MainExecutablePath, const StringRef InputFileName,
552549
}
553550
SyntaxCache = new SyntaxParsingCache(OldSyntaxTree.getValue());
554551

555-
SyntaxCache->setRecordReuseInformation();
556-
557552
if (options::OldSourceFilename.empty()) {
558553
llvm::errs() << "The old syntax file must be provided to translate "
559554
"line:column edits to byte offsets";
@@ -608,10 +603,10 @@ int parseFile(const char *MainExecutablePath, const StringRef InputFileName,
608603
if (SyntaxCache) {
609604
if (options::PrintVisualReuseInfo) {
610605
printVisualNodeReuseInformation(Instance.getSourceMgr(), BufferID,
611-
SyntaxCache);
606+
SyntaxCache, SF->getSyntaxRoot());
612607
}
613608
if (!options::IncrementalReuseLog.empty()) {
614-
saveReuseLog(Instance.getSourceMgr(), BufferID, SyntaxCache);
609+
saveReuseLog(SyntaxCache, SF->getSyntaxRoot());
615610
}
616611
ByteBasedSourceRangeSet ExpectedReparseRegions;
617612

@@ -670,8 +665,7 @@ int doDumpRawTokenSyntax(const StringRef InputFile) {
670665
}
671666

672667
for (auto TokAndPos : Tokens) {
673-
TokAndPos.second.printLineAndColumn(llvm::outs());
674-
llvm::outs() << "\n";
668+
llvm::outs() << TokAndPos.second << "\n";
675669
TokAndPos.first->dump(llvm::outs());
676670
llvm::outs() << "\n";
677671
}

0 commit comments

Comments
 (0)