Skip to content

Commit 9a8749c

Browse files
ahoppenahoppen
committed
[libSyntax] Allow adding garbage nodes in between any two children of a syntax node
When the source code is invalid, this allows us to represent tokens that could not be used to form a valid syntax tree with more fidelity. This commit does not start using GarbageNodes yet, it just sets everything up for them.
1 parent 497ae4e commit 9a8749c

23 files changed

+1052
-248
lines changed

‎lib/Parse/ParseExpr.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1688,10 +1688,12 @@ ParserResult<Expr> Parser::parseExprPrimary(Diag<> ID, bool isExprBasic) {
16881688
if (SyntaxContext->isEnabled()) {
16891689
ParsedPatternSyntax PatternNode =
16901690
ParsedSyntaxRecorder::makeIdentifierPattern(
1691-
SyntaxContext->popToken(), *SyntaxContext);
1691+
/*GarbageNodes=*/None,
1692+
/*Identifier=*/SyntaxContext->popToken(), *SyntaxContext);
16921693
ParsedExprSyntax ExprNode =
1693-
ParsedSyntaxRecorder::makeUnresolvedPatternExpr(std::move(PatternNode),
1694-
*SyntaxContext);
1694+
ParsedSyntaxRecorder::makeUnresolvedPatternExpr(
1695+
/*GarbageNodes=*/None,
1696+
/*Pattern=*/std::move(PatternNode), *SyntaxContext);
16951697
SyntaxContext->addSyntax(std::move(ExprNode));
16961698
}
16971699
return makeParserResult(new (Context) UnresolvedPatternExpr(pattern));

‎lib/Parse/ParseIfConfig.cpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -791,8 +791,20 @@ ParserResult<IfConfigDecl> Parser::parseIfConfig(
791791
// clause unless we're doing a parse-only pass.
792792
if (isElse) {
793793
isActive = !foundActive && shouldEvaluate;
794-
if (SyntaxContext->isEnabled())
794+
if (SyntaxContext->isEnabled()) {
795+
// Because we use the same libSyntax node for #elseif and #else, we need
796+
// to disambiguate whether a postfix expression is the condition of
797+
// #elseif or a postfix expression of the #else body.
798+
// To do this, push three empty syntax nodes onto the stack.
799+
// - First one for garbage nodes between the #else keyword and the
800+
// condition
801+
// - One for the condition itself (whcih doesn't exist)
802+
// - And finally one for the garbage nodes between the condition and
803+
// the elements
795804
SyntaxContext->addRawSyntax(ParsedRawSyntaxNode());
805+
SyntaxContext->addRawSyntax(ParsedRawSyntaxNode());
806+
SyntaxContext->addRawSyntax(ParsedRawSyntaxNode());
807+
}
796808
} else {
797809
llvm::SaveAndRestore<bool> S(InPoundIfEnvironment, true);
798810
ParserResult<Expr> Result = parseExprSequence(diag::expected_expr,

‎lib/Parse/ParseType.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -916,7 +916,10 @@ Parser::parseTypeSimpleOrComposition(Diag<> MessageID, ParseTypeReason reason) {
916916
if (SyntaxContext->isEnabled()) {
917917
if (auto synType = SyntaxContext->popIf<ParsedTypeSyntax>()) {
918918
auto LastNode = ParsedSyntaxRecorder::makeCompositionTypeElement(
919-
std::move(*synType), None, *SyntaxContext);
919+
/*GarbageNodes=*/None,
920+
/*Type=*/std::move(*synType),
921+
/*GarbageNodes=*/None,
922+
/*Ampersand=*/None, *SyntaxContext);
920923
SyntaxContext->addSyntax(std::move(LastNode));
921924
}
922925
}

‎lib/Parse/ParsedSyntaxRecorder.cpp.gyb

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,23 @@ ParsedTupleTypeElementSyntax
172172
ParsedSyntaxRecorder::makeTupleTypeElement(ParsedTypeSyntax Type,
173173
llvm::Optional<ParsedTokenSyntax> TrailingComma,
174174
SyntaxParsingContext &SPCtx) {
175-
return makeTupleTypeElement(None, None, None, None, std::move(Type), None, None,
176-
std::move(TrailingComma), SPCtx);
175+
return makeTupleTypeElement(
176+
/*GarbageNodes=*/None,
177+
/*InOut=*/None,
178+
/*GarbageNodes=*/None,
179+
/*Name=*/None,
180+
/*GarbageNodes=*/None,
181+
/*SecondName=*/None,
182+
/*GarbageNodes=*/None,
183+
/*Colon=*/None,
184+
/*GarbageNodes=*/None,
185+
std::move(Type),
186+
/*GarbageNodes=*/None,
187+
/*Ellipsis=*/None,
188+
/*GarbageNodes=*/None,
189+
/*Initializer=*/None,
190+
/*GarbageNodes=*/None,
191+
std::move(TrailingComma),
192+
SPCtx
193+
);
177194
}

‎lib/Parse/SyntaxParsingContext.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,21 +322,21 @@ void SyntaxParsingContext::collectNodesInPlace(SyntaxKind CollectionKind,
322322
static ParsedRawSyntaxNode finalizeSourceFile(RootContextData &RootData,
323323
MutableArrayRef<ParsedRawSyntaxNode> Parts) {
324324
ParsedRawSyntaxRecorder &Recorder = RootData.Recorder;
325-
ParsedRawSyntaxNode Layout[2];
325+
ParsedRawSyntaxNode Layout[4];
326326

327327
assert(!Parts.empty() && Parts.back().isToken(tok::eof));
328-
Layout[1] = std::move(Parts.back());
328+
Layout[3] = std::move(Parts.back());
329329
Parts = Parts.drop_back();
330330

331331

332332
assert(llvm::all_of(Parts, [](const ParsedRawSyntaxNode& node) {
333333
return node.getKind() == SyntaxKind::CodeBlockItem;
334334
}) && "all top level element must be 'CodeBlockItem'");
335335

336-
Layout[0] = Recorder.recordRawSyntax(SyntaxKind::CodeBlockItemList, Parts);
336+
Layout[1] = Recorder.recordRawSyntax(SyntaxKind::CodeBlockItemList, Parts);
337337

338338
return Recorder.recordRawSyntax(SyntaxKind::SourceFile,
339-
llvm::makeMutableArrayRef(Layout, 2));
339+
llvm::makeMutableArrayRef(Layout, 4));
340340
}
341341

342342
OpaqueSyntaxNode SyntaxParsingContext::finalizeRoot() {

‎lib/Syntax/SyntaxFactory.cpp.gyb

Lines changed: 69 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ SyntaxFactory::countChildren(SyntaxKind Kind){
6565
% for node in SYNTAX_NODES:
6666
% if not node.is_syntax_collection():
6767
case SyntaxKind::${node.syntax_kind}:
68-
% child_count = len(node.children)
69-
% non_optional_child_count = sum(0 if child.is_optional else 1 for child in node.children)
68+
% child_count = len(node.non_garbage_children)
69+
% non_optional_child_count = sum(0 if child.is_optional else 1 for child in node.non_garbage_children)
7070
return {${non_optional_child_count}, ${child_count}};
7171
% end
7272
% end
@@ -243,39 +243,95 @@ SyntaxFactory::makeBlank${node.syntax_kind}() {
243243
% end
244244

245245
TupleTypeSyntax SyntaxFactory::makeVoidTupleType() {
246-
return makeTupleType(makeLeftParenToken({}, {}),
247-
makeBlankTupleTypeElementList(),
248-
makeRightParenToken({}, {}));
246+
return makeTupleType(
247+
/*GarbageNodes=*/None,
248+
/*LeftParen=*/makeLeftParenToken({}, {}),
249+
/*GarbageNodes=*/None,
250+
/*Elements=*/makeBlankTupleTypeElementList(),
251+
/*GarbageNodes=*/None,
252+
/*RightParen=*/makeRightParenToken({}, {})
253+
);
249254
}
250255

251256
TupleTypeElementSyntax
252257
SyntaxFactory::makeTupleTypeElement(llvm::Optional<TokenSyntax> Label,
253258
llvm::Optional<TokenSyntax> Colon,
254259
TypeSyntax Type,
255260
llvm::Optional<TokenSyntax> TrailingComma) {
256-
return makeTupleTypeElement(None, Label, None, Colon, Type, None, None,
257-
TrailingComma);
261+
return makeTupleTypeElement(
262+
/*GarbageNodes=*/None,
263+
/*InOut=*/None,
264+
/*GarbageNodes=*/None,
265+
/*Name=*/Label,
266+
/*GarbageNodes=*/None,
267+
/*SecondName=*/None,
268+
/*GarbageNodes=*/None,
269+
/*Colon=*/Colon,
270+
/*GarbageNodes=*/None,
271+
/*Type=*/Type,
272+
/*GarbageNodes=*/None,
273+
/*Ellipsis=*/None,
274+
/*GarbageNodes=*/None,
275+
/*Intitializer=*/None,
276+
/*GarbageNodes=*/None,
277+
/*TrailingComma=*/TrailingComma
278+
);
258279
}
259280

260281
TupleTypeElementSyntax
261282
SyntaxFactory::makeTupleTypeElement(TypeSyntax Type,
262283
llvm::Optional<TokenSyntax> TrailingComma) {
263-
return makeTupleTypeElement(None, None, None, None, Type, None, None,
264-
TrailingComma);
284+
return makeTupleTypeElement(
285+
/*GarbageNodes=*/None,
286+
/*InOut=*/None,
287+
/*GarbageNodes=*/None,
288+
/*Name=*/None,
289+
/*GarbageNodes=*/None,
290+
/*SecondName=*/None,
291+
/*GarbageNodes=*/None,
292+
/*Colon=*/None,
293+
/*GarbageNodes=*/None,
294+
/*Type=*/Type,
295+
/*GarbageNodes=*/None,
296+
/*Ellipsis=*/None,
297+
/*GarbageNodes=*/None,
298+
/*Initializer=*/None,
299+
/*GarbageNodes=*/None,
300+
/*TrailingComma=*/TrailingComma
301+
);
265302
}
266303

267304
GenericParameterSyntax
268305
SyntaxFactory::makeGenericParameter(TokenSyntax Name,
269306
llvm::Optional<TokenSyntax> TrailingComma) {
270-
return makeGenericParameter(None, Name, None, None, TrailingComma);
307+
return makeGenericParameter(
308+
/*GarbageNodes=*/None,
309+
/*Attributes=*/None,
310+
/*GarbageNodes=*/None,
311+
/*Name=*/Name,
312+
/*GarbageNodes=*/None,
313+
/*Colon=*/None,
314+
/*GarbageNodes=*/None,
315+
/*InheritedType=*/None,
316+
/*GarbageNodes=*/None,
317+
/*TrailingComma=*/TrailingComma
318+
);
271319
}
272320

273321
TypeSyntax SyntaxFactory::makeTypeIdentifier(StringRef TypeName,
274322
StringRef LeadingTrivia,
275323
StringRef TrailingTrivia) {
276-
auto identifier =
277-
makeIdentifier(TypeName, LeadingTrivia, TrailingTrivia);
278-
return makeSimpleTypeIdentifier(identifier, None);
324+
auto identifier = makeIdentifier(
325+
TypeName,
326+
LeadingTrivia,
327+
TrailingTrivia
328+
);
329+
return makeSimpleTypeIdentifier(
330+
/*GarbageNodes=*/None,
331+
/*Name=*/identifier,
332+
/*GarbageNodes=*/None,
333+
/*GenerigArgumentClause=*/None
334+
);
279335
}
280336

281337
TypeSyntax SyntaxFactory::makeAnyTypeIdentifier(StringRef LeadingTrivia,

‎test/Syntax/Inputs/serialize_class_decl.json

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,27 @@
11
{
22
"kind": "SourceFile",
33
"layout": [
4+
null,
45
{
56
"kind": "CodeBlockItemList",
67
"layout": [
78
{
89
"kind": "CodeBlockItem",
910
"layout": [
11+
null,
1012
{
1113
"kind": "ClassDecl",
1214
"layout": [
15+
null,
16+
null,
1317
null,
1418
{
1519
"kind": "ModifierList",
1620
"layout": [
1721
{
1822
"kind": "DeclModifier",
1923
"layout": [
24+
null,
2025
{
2126
"tokenKind": {
2227
"kind": "contextual_keyword",
@@ -26,13 +31,16 @@
2631
"trailingTrivia": " ",
2732
"presence": "Present"
2833
},
34+
null,
35+
null,
2936
null
3037
],
3138
"presence": "Present"
3239
}
3340
],
3441
"presence": "Present"
3542
},
43+
null,
3644
{
3745
"tokenKind": {
3846
"kind": "kw_class"
@@ -41,6 +49,7 @@
4149
"trailingTrivia": " ",
4250
"presence": "Present"
4351
},
52+
null,
4453
{
4554
"tokenKind": {
4655
"kind": "identifier",
@@ -53,9 +62,14 @@
5362
null,
5463
null,
5564
null,
65+
null,
66+
null,
67+
null,
68+
null,
5669
{
5770
"kind": "MemberDeclBlock",
5871
"layout": [
72+
null,
5973
{
6074
"tokenKind": {
6175
"kind": "l_brace"
@@ -64,33 +78,41 @@
6478
"trailingTrivia": "",
6579
"presence": "Present"
6680
},
81+
null,
6782
{
6883
"kind": "MemberDeclList",
6984
"layout": [],
7085
"presence": "Present"
7186
},
87+
null,
7288
{
7389
"tokenKind": {
7490
"kind": "r_brace"
7591
},
7692
"leadingTrivia": "",
7793
"trailingTrivia": "",
7894
"presence": "Present"
79-
}
95+
},
96+
null
8097
],
8198
"presence": "Present"
82-
}
99+
},
100+
null
83101
],
84102
"presence": "Present"
85103
},
86104
null,
105+
null,
106+
null,
107+
null,
87108
null
88109
],
89110
"presence": "Present"
90111
}
91112
],
92113
"presence": "Present"
93114
},
115+
null,
94116
{
95117
"tokenKind": {
96118
"kind": "eof",
@@ -99,7 +121,8 @@
99121
"leadingTrivia": "\n",
100122
"trailingTrivia": "",
101123
"presence": "Present"
102-
}
124+
},
125+
null
103126
],
104127
"presence": "Present"
105128
}

0 commit comments

Comments
 (0)