Skip to content

Commit 97757e4

Browse files
Nathan Hawesrintaro
Nathan Hawes
authored and
rintaro
committed
[Parse] Assert that the children of syntax tree nodes have contiguous source ranges
This should help us catch syntax tree round-tripping issues.
1 parent 7b5dfd6 commit 97757e4

10 files changed

+126
-62
lines changed

‎include/swift/Parse/ParsedRawSyntaxNode.h

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class ParsedRawSyntaxNode {
5252
};
5353
struct DeferredLayoutNode {
5454
MutableArrayRef<ParsedRawSyntaxNode> Children;
55+
CharSourceRange Range;
5556
};
5657
struct DeferredTokenNode {
5758
const ParsedTriviaPiece *TriviaPieces;
@@ -72,9 +73,9 @@ class ParsedRawSyntaxNode {
7273
/// Primary used for capturing a deferred missing token.
7374
bool IsMissing = false;
7475

75-
ParsedRawSyntaxNode(syntax::SyntaxKind k,
76+
ParsedRawSyntaxNode(syntax::SyntaxKind k, CharSourceRange r,
7677
MutableArrayRef<ParsedRawSyntaxNode> deferredNodes)
77-
: DeferredLayout({deferredNodes}),
78+
: DeferredLayout({deferredNodes, r}),
7879
SynKind(uint16_t(k)), TokKind(uint16_t(tok::unknown)),
7980
DK(DataKind::DeferredLayout) {
8081
assert(getKind() == k && "Syntax kind with too large value!");
@@ -109,10 +110,12 @@ class ParsedRawSyntaxNode {
109110
}
110111

111112
ParsedRawSyntaxNode(syntax::SyntaxKind k, tok tokKind,
112-
CharSourceRange r, OpaqueSyntaxNode n)
113+
CharSourceRange r, OpaqueSyntaxNode n,
114+
bool IsMissing = false)
113115
: RecordedData{n, r},
114116
SynKind(uint16_t(k)), TokKind(uint16_t(tokKind)),
115-
DK(DataKind::Recorded) {
117+
DK(DataKind::Recorded),
118+
IsMissing(IsMissing) {
116119
assert(getKind() == k && "Syntax kind with too large value!");
117120
assert(getTokenKind() == tokKind && "Token kind with too large value!");
118121
}
@@ -209,9 +212,20 @@ class ParsedRawSyntaxNode {
209212
return copy;
210213
}
211214

215+
CharSourceRange getDeferredRange() const {
216+
switch (DK) {
217+
case DataKind::DeferredLayout:
218+
return getDeferredLayoutRange();
219+
case DataKind::DeferredToken:
220+
return getDeferredTokenRangeWithTrivia();
221+
default:
222+
llvm_unreachable("node not deferred");
223+
}
224+
}
225+
212226
// Recorded Data ===========================================================//
213227

214-
CharSourceRange getRange() const {
228+
CharSourceRange getRecordedRange() const {
215229
assert(isRecorded());
216230
return RecordedData.Range;
217231
}
@@ -228,6 +242,10 @@ class ParsedRawSyntaxNode {
228242

229243
// Deferred Layout Data ====================================================//
230244

245+
CharSourceRange getDeferredLayoutRange() const {
246+
assert(DK == DataKind::DeferredLayout);
247+
return DeferredLayout.Range;
248+
}
231249
ArrayRef<ParsedRawSyntaxNode> getDeferredChildren() const {
232250
assert(DK == DataKind::DeferredLayout);
233251
return DeferredLayout.Children;
@@ -259,6 +277,19 @@ class ParsedRawSyntaxNode {
259277

260278
// Deferred Token Data =====================================================//
261279

280+
CharSourceRange getDeferredTokenRangeWithTrivia() const {
281+
assert(DK == DataKind::DeferredToken);
282+
auto leadTriviaPieces = getDeferredLeadingTriviaPieces();
283+
auto trailTriviaPieces = getDeferredTrailingTriviaPieces();
284+
285+
auto leadTriviaLen = ParsedTriviaPiece::getTotalLength(leadTriviaPieces);
286+
auto trailTriviaLen = ParsedTriviaPiece::getTotalLength(trailTriviaPieces);
287+
288+
SourceLoc begin = DeferredToken.TokLoc.getAdvancedLoc(-leadTriviaLen);
289+
unsigned len = leadTriviaLen + DeferredToken.TokLength + trailTriviaLen;
290+
291+
return CharSourceRange{begin, len};
292+
}
262293
CharSourceRange getDeferredTokenRange() const {
263294
assert(DK == DataKind::DeferredToken);
264295
return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};

‎include/swift/Parse/ParsedRawSyntaxRecorder.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ class ParsedRawSyntaxRecorder {
7373
/// Used for incremental re-parsing.
7474
ParsedRawSyntaxNode lookupNode(size_t lexerOffset, SourceLoc loc,
7575
syntax::SyntaxKind kind);
76+
77+
#ifndef NDEBUG
78+
static void verifyElementRanges(ArrayRef<ParsedRawSyntaxNode> elements);
79+
#endif
7680
};
7781

7882
} // end namespace swift

‎include/swift/Parse/ParsedSyntaxBuilders.h.gyb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,9 @@ public:
6262
% end
6363

6464
Parsed${node.name} build();
65-
Parsed${node.name} makeDeferred();
6665

6766
private:
67+
Parsed${node.name} makeDeferred();
6868
Parsed${node.name} record();
6969
void finishLayout(bool deferred);
7070
};

‎include/swift/Parse/ParsedSyntaxRecorder.h.gyb

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,23 +43,22 @@ struct ParsedSyntaxRecorder {
4343
% end
4444
% child_params = ', '.join(child_params)
4545
private:
46-
static Parsed${node.name} record${node.syntax_kind}(${child_params},
46+
static Parsed${node.name} record${node.syntax_kind}(MutableArrayRef<ParsedRawSyntaxNode> layout,
4747
ParsedRawSyntaxRecorder &rec);
48-
public:
49-
static Parsed${node.name} defer${node.syntax_kind}(${child_params},
48+
static Parsed${node.name} defer${node.syntax_kind}(MutableArrayRef<ParsedRawSyntaxNode> layout,
5049
SyntaxParsingContext &SPCtx);
50+
public:
5151
static Parsed${node.name} make${node.syntax_kind}(${child_params},
5252
SyntaxParsingContext &SPCtx);
5353
% elif node.is_syntax_collection():
5454
private:
5555
static Parsed${node.name} record${node.syntax_kind}(
56-
MutableArrayRef<Parsed${node.collection_element_type}> elts,
56+
MutableArrayRef<ParsedRawSyntaxNode> layout,
5757
ParsedRawSyntaxRecorder &rec);
58-
59-
public:
6058
static Parsed${node.name} defer${node.syntax_kind}(
61-
MutableArrayRef<Parsed${node.collection_element_type}> elts,
59+
MutableArrayRef<ParsedRawSyntaxNode> layout,
6260
SyntaxParsingContext &SPCtx);
61+
public:
6362
static Parsed${node.name} make${node.syntax_kind}(
6463
MutableArrayRef<Parsed${node.collection_element_type}> elts,
6564
SyntaxParsingContext &SPCtx);

‎lib/Parse/ParseExpr.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1466,6 +1466,7 @@ ParserResult<Expr> Parser::parseExprPrimary(Diag<> ID, bool isExprBasic) {
14661466
// name, not a binding, because it is the start of an enum pattern or
14671467
// call pattern.
14681468
peekToken().isNot(tok::period, tok::period_prefix, tok::l_paren)) {
1469+
DeferringContextRAII Deferring(*SyntaxContext);
14691470
Identifier name;
14701471
SourceLoc loc = consumeIdentifier(&name, /*allowDollarIdentifier=*/true);
14711472
auto introducer = (InVarOrLetPattern != IVOLP_InVar
@@ -1474,10 +1475,10 @@ ParserResult<Expr> Parser::parseExprPrimary(Diag<> ID, bool isExprBasic) {
14741475
auto pattern = createBindingFromPattern(loc, name, introducer);
14751476
if (SyntaxContext->isEnabled()) {
14761477
ParsedPatternSyntax PatternNode =
1477-
ParsedSyntaxRecorder::deferIdentifierPattern(
1478+
ParsedSyntaxRecorder::makeIdentifierPattern(
14781479
SyntaxContext->popToken(), *SyntaxContext);
14791480
ParsedExprSyntax ExprNode =
1480-
ParsedSyntaxRecorder::deferUnresolvedPatternExpr(std::move(PatternNode),
1481+
ParsedSyntaxRecorder::makeUnresolvedPatternExpr(std::move(PatternNode),
14811482
*SyntaxContext);
14821483
SyntaxContext->addSyntax(std::move(ExprNode));
14831484
}

‎lib/Parse/ParsedRawSyntaxNode.cpp

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,31 @@ ParsedRawSyntaxNode
2121
ParsedRawSyntaxNode::makeDeferred(SyntaxKind k,
2222
MutableArrayRef<ParsedRawSyntaxNode> deferredNodes,
2323
SyntaxParsingContext &ctx) {
24+
CharSourceRange range;
2425
if (deferredNodes.empty()) {
25-
return ParsedRawSyntaxNode(k, {});
26+
return ParsedRawSyntaxNode(k, range, {});
2627
}
2728
ParsedRawSyntaxNode *newPtr =
2829
ctx.getScratchAlloc().Allocate<ParsedRawSyntaxNode>(deferredNodes.size());
2930

30-
// uninitialized move;
3131
auto ptr = newPtr;
32-
for (auto &node : deferredNodes)
33-
:: new (static_cast<void *>(ptr++)) ParsedRawSyntaxNode(std::move(node));
32+
for (auto &node : deferredNodes) {
33+
// Cached range.
34+
if (!node.isNull() && !node.isMissing()) {
35+
auto nodeRange = node.getDeferredRange();
36+
if (nodeRange.isValid()) {
37+
if (range.isInvalid())
38+
range = nodeRange;
39+
else
40+
range.widen(nodeRange);
41+
}
42+
}
3443

35-
return ParsedRawSyntaxNode(k, makeMutableArrayRef(newPtr, deferredNodes.size()));
44+
// uninitialized move;
45+
:: new (static_cast<void *>(ptr++)) ParsedRawSyntaxNode(std::move(node));
46+
}
47+
return ParsedRawSyntaxNode(k, range,
48+
makeMutableArrayRef(newPtr, deferredNodes.size()));
3649
}
3750

3851
ParsedRawSyntaxNode

‎lib/Parse/ParsedRawSyntaxRecorder.cpp

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ ParsedRawSyntaxNode
5353
ParsedRawSyntaxRecorder::recordMissingToken(tok tokenKind, SourceLoc loc) {
5454
CharSourceRange range{loc, 0};
5555
OpaqueSyntaxNode n = SPActions->recordMissingToken(tokenKind, loc);
56-
return ParsedRawSyntaxNode{SyntaxKind::Token, tokenKind, range, n};
56+
return ParsedRawSyntaxNode{SyntaxKind::Token, tokenKind, range, n,
57+
/*isMissing=*/true};
5758
}
5859

5960
static ParsedRawSyntaxNode
@@ -84,11 +85,11 @@ ParsedRawSyntaxRecorder::recordRawSyntax(SyntaxKind kind,
8485
if (subnode.isNull()) {
8586
subnodes.push_back(nullptr);
8687
} else if (subnode.isRecorded()) {
87-
localRange = subnode.getRange();
88+
localRange = subnode.getRecordedRange();
8889
subnodes.push_back(subnode.takeOpaqueNode());
8990
} else {
9091
auto recorded = getRecordedNode(subnode.copyDeferred(), *this);
91-
localRange = recorded.getRange();
92+
localRange = recorded.getRecordedRange();
9293
subnodes.push_back(recorded.takeOpaqueNode());
9394
}
9495

@@ -128,3 +129,21 @@ ParsedRawSyntaxRecorder::lookupNode(size_t lexerOffset, SourceLoc loc,
128129
CharSourceRange range{loc, unsigned(length)};
129130
return ParsedRawSyntaxNode{kind, tok::unknown, range, n};
130131
}
132+
133+
#ifndef NDEBUG
134+
void ParsedRawSyntaxRecorder::verifyElementRanges(ArrayRef<ParsedRawSyntaxNode> elements) {
135+
SourceLoc prevEndLoc;
136+
for (const auto &elem: elements) {
137+
if (elem.isMissing() || elem.isNull())
138+
continue;
139+
CharSourceRange range = elem.isRecorded()
140+
? elem.getRecordedRange()
141+
: elem.getDeferredRange();
142+
if (range.isValid()) {
143+
assert((prevEndLoc.isInvalid() || range.getStart() == prevEndLoc)
144+
&& "Non-contiguous child ranges?");
145+
prevEndLoc = range.getEnd();
146+
}
147+
}
148+
}
149+
#endif

‎lib/Parse/ParsedSyntaxBuilders.cpp.gyb

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ Parsed${node.name}Builder::makeDeferred() {
7474

7575
Parsed${node.name}
7676
Parsed${node.name}Builder::build() {
77-
if (SPCtx.isBacktracking())
77+
if (SPCtx.shouldDefer())
7878
return makeDeferred();
7979
return record();
8080
}
@@ -120,6 +120,10 @@ void Parsed${node.name}Builder::finishLayout(bool deferred) {
120120
}
121121
% end
122122
% end
123+
124+
#ifndef NDEBUG
125+
ParsedRawSyntaxRecorder::verifyElementRanges(Layout);
126+
#endif
123127
% end
124128
}
125129

‎lib/Parse/ParsedSyntaxRecorder.cpp.gyb

Lines changed: 29 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -90,23 +90,21 @@ bool ParsedSyntaxRecorder::formExactLayoutFor(syntax::SyntaxKind Kind,
9090
% child_params = ', '.join(child_params)
9191
% child_move_args = ', '.join(child_move_args)
9292
Parsed${node.name}
93-
ParsedSyntaxRecorder::record${node.syntax_kind}(${child_params},
93+
ParsedSyntaxRecorder::record${node.syntax_kind}(MutableArrayRef<ParsedRawSyntaxNode> layout,
9494
ParsedRawSyntaxRecorder &rec) {
95-
ParsedRawSyntaxNode layout[] = {
96-
% for child in node.children:
97-
% if child.is_optional:
98-
${child.name}.hasValue() ? ${child.name}->takeRaw() : ParsedRawSyntaxNode::null(),
99-
% else:
100-
${child.name}.takeRaw(),
101-
% end
102-
% end
103-
};
10495
auto raw = rec.recordRawSyntax(SyntaxKind::${node.syntax_kind}, layout);
10596
return Parsed${node.name}(std::move(raw));
10697
}
10798

10899
Parsed${node.name}
109-
ParsedSyntaxRecorder::defer${node.syntax_kind}(${child_params}, SyntaxParsingContext &SPCtx) {
100+
ParsedSyntaxRecorder::defer${node.syntax_kind}(MutableArrayRef<ParsedRawSyntaxNode> layout, SyntaxParsingContext &SPCtx) {
101+
auto raw = ParsedRawSyntaxNode::makeDeferred(SyntaxKind::${node.syntax_kind}, layout, SPCtx);
102+
return Parsed${node.name}(std::move(raw));
103+
}
104+
105+
Parsed${node.name}
106+
ParsedSyntaxRecorder::make${node.syntax_kind}(${child_params},
107+
SyntaxParsingContext &SPCtx) {
110108
ParsedRawSyntaxNode layout[] = {
111109
% for child in node.children:
112110
% if child.is_optional:
@@ -116,41 +114,27 @@ ParsedSyntaxRecorder::defer${node.syntax_kind}(${child_params}, SyntaxParsingCon
116114
% end
117115
% end
118116
};
119-
auto raw = ParsedRawSyntaxNode::makeDeferred(SyntaxKind::${node.syntax_kind}, llvm::makeMutableArrayRef(layout, ${len(node.children)}), SPCtx);
120-
return Parsed${node.name}(std::move(raw));
121-
}
122-
123-
Parsed${node.name}
124-
ParsedSyntaxRecorder::make${node.syntax_kind}(${child_params},
125-
SyntaxParsingContext &SPCtx) {
126-
if (SPCtx.isBacktracking())
127-
return defer${node.syntax_kind}(${child_move_args}, SPCtx);
128-
return record${node.syntax_kind}(${child_move_args}, SPCtx.getRecorder());
117+
#ifndef NDEBUG
118+
ParsedRawSyntaxRecorder::verifyElementRanges(layout);
119+
#endif
120+
if (SPCtx.shouldDefer())
121+
return defer${node.syntax_kind}(layout, SPCtx);
122+
return record${node.syntax_kind}(layout, SPCtx.getRecorder());
129123
}
130124

131125
% elif node.is_syntax_collection():
132126
Parsed${node.name}
133127
ParsedSyntaxRecorder::record${node.syntax_kind}(
134-
MutableArrayRef<Parsed${node.collection_element_type}> elements,
128+
MutableArrayRef<ParsedRawSyntaxNode> layout,
135129
ParsedRawSyntaxRecorder &rec) {
136-
SmallVector<ParsedRawSyntaxNode, 16> layout;
137-
layout.reserve(elements.size());
138-
for (auto &element : elements) {
139-
layout.push_back(element.takeRaw());
140-
}
141130
auto raw = rec.recordRawSyntax(SyntaxKind::${node.syntax_kind}, layout);
142131
return Parsed${node.name}(std::move(raw));
143132
}
144133

145134
Parsed${node.name}
146135
ParsedSyntaxRecorder::defer${node.syntax_kind}(
147-
MutableArrayRef<Parsed${node.collection_element_type}> elements,
136+
MutableArrayRef<ParsedRawSyntaxNode> layout,
148137
SyntaxParsingContext &SPCtx) {
149-
SmallVector<ParsedRawSyntaxNode, 16> layout;
150-
layout.reserve(elements.size());
151-
for (auto &element : elements) {
152-
layout.push_back(element.takeRaw());
153-
}
154138
auto raw = ParsedRawSyntaxNode::makeDeferred(SyntaxKind::${node.syntax_kind},
155139
layout, SPCtx);
156140
return Parsed${node.name}(std::move(raw));
@@ -160,16 +144,24 @@ Parsed${node.name}
160144
ParsedSyntaxRecorder::make${node.syntax_kind}(
161145
MutableArrayRef<Parsed${node.collection_element_type}> elements,
162146
SyntaxParsingContext &SPCtx) {
163-
if (SPCtx.isBacktracking())
164-
return defer${node.syntax_kind}(elements, SPCtx);
165-
return record${node.syntax_kind}(elements, SPCtx.getRecorder());
147+
SmallVector<ParsedRawSyntaxNode, 16> layout;
148+
layout.reserve(elements.size());
149+
for (auto &element : elements) {
150+
layout.push_back(element.takeRaw());
151+
}
152+
#ifndef NDEBUG
153+
ParsedRawSyntaxRecorder::verifyElementRanges(layout);
154+
#endif
155+
if (SPCtx.shouldDefer())
156+
return defer${node.syntax_kind}(layout, SPCtx);
157+
return record${node.syntax_kind}(layout, SPCtx.getRecorder());
166158
}
167159

168160
Parsed${node.name}
169161
ParsedSyntaxRecorder::makeBlank${node.syntax_kind}(SourceLoc loc,
170162
SyntaxParsingContext &SPCtx) {
171163
ParsedRawSyntaxNode raw;
172-
if (SPCtx.isBacktracking()) {
164+
if (SPCtx.shouldDefer()) {
173165
// FIXME: 'loc' is not preserved when capturing a deferred layout.
174166
raw = ParsedRawSyntaxNode::makeDeferred(SyntaxKind::${node.syntax_kind}, {}, SPCtx);
175167
} else {

‎lib/Parse/SyntaxParsingContext.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "swift/AST/SourceFile.h"
2020
#include "swift/Basic/Defer.h"
2121
#include "swift/Parse/ParsedSyntax.h"
22+
#include "swift/Parse/ParsedRawSyntaxRecorder.h"
2223
#include "swift/Parse/ParsedSyntaxRecorder.h"
2324
#include "swift/Parse/SyntaxParseActions.h"
2425
#include "swift/Parse/SyntaxParsingCache.h"
@@ -58,7 +59,7 @@ size_t SyntaxParsingContext::lookupNode(size_t LexerOffset, SourceLoc Loc) {
5859
return 0;
5960
}
6061
Mode = AccumulationMode::SkippedForIncrementalUpdate;
61-
auto length = foundNode.getRange().getByteLength();
62+
auto length = foundNode.getRecordedRange().getByteLength();
6263
getStorage().push_back(std::move(foundNode));
6364
return length;
6465
}

0 commit comments

Comments
 (0)