Skip to content

Commit 43ebf73

Browse files
ahoppenahoppen
committed
[libSyntax] Allow adding garbage nodes in between any two children of a syntax node
When the source code is invalid, this allows us to represent tokens that could not be used to form a valid syntax tree with more fidelity. This commit does not start using GarbageNodes yet, it just sets everything up for them.
1 parent 57807fb commit 43ebf73

23 files changed

+906
-202
lines changed

‎lib/Parse/ParseExpr.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1688,10 +1688,12 @@ ParserResult<Expr> Parser::parseExprPrimary(Diag<> ID, bool isExprBasic) {
16881688
if (SyntaxContext->isEnabled()) {
16891689
ParsedPatternSyntax PatternNode =
16901690
ParsedSyntaxRecorder::makeIdentifierPattern(
1691-
SyntaxContext->popToken(), *SyntaxContext);
1691+
/*GarbageNodes=*/None,
1692+
/*Identifier=*/SyntaxContext->popToken(), *SyntaxContext);
16921693
ParsedExprSyntax ExprNode =
1693-
ParsedSyntaxRecorder::makeUnresolvedPatternExpr(std::move(PatternNode),
1694-
*SyntaxContext);
1694+
ParsedSyntaxRecorder::makeUnresolvedPatternExpr(
1695+
/*GarbageNodes=*/None,
1696+
/*Pattern=*/std::move(PatternNode), *SyntaxContext);
16951697
SyntaxContext->addSyntax(std::move(ExprNode));
16961698
}
16971699
return makeParserResult(new (Context) UnresolvedPatternExpr(pattern));

‎lib/Parse/ParseIfConfig.cpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -791,8 +791,20 @@ ParserResult<IfConfigDecl> Parser::parseIfConfig(
791791
// clause unless we're doing a parse-only pass.
792792
if (isElse) {
793793
isActive = !foundActive && shouldEvaluate;
794-
if (SyntaxContext->isEnabled())
794+
if (SyntaxContext->isEnabled()) {
795+
// Because we use the same libSyntax node for #elseif and #else, we need
796+
// to disambiguate whether a postfix expression is the condition of
797+
// #elseif or a postfix expression of the #else body.
798+
// To do this, push three empty syntax nodes onto the stack.
799+
// - First one for garbage nodes between the #else keyword and the
800+
// condition
801+
// - One for the condition itself (whcih doesn't exist)
802+
// - And finally one for the garbage nodes between the condition and
803+
// the elements
795804
SyntaxContext->addRawSyntax(ParsedRawSyntaxNode());
805+
SyntaxContext->addRawSyntax(ParsedRawSyntaxNode());
806+
SyntaxContext->addRawSyntax(ParsedRawSyntaxNode());
807+
}
796808
} else {
797809
llvm::SaveAndRestore<bool> S(InPoundIfEnvironment, true);
798810
ParserResult<Expr> Result = parseExprSequence(diag::expected_expr,

‎lib/Parse/ParseType.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -916,7 +916,10 @@ Parser::parseTypeSimpleOrComposition(Diag<> MessageID, ParseTypeReason reason) {
916916
if (SyntaxContext->isEnabled()) {
917917
if (auto synType = SyntaxContext->popIf<ParsedTypeSyntax>()) {
918918
auto LastNode = ParsedSyntaxRecorder::makeCompositionTypeElement(
919-
std::move(*synType), None, *SyntaxContext);
919+
/*GarbageNodes=*/None,
920+
/*Type=*/std::move(*synType),
921+
/*GarbageNodes=*/None,
922+
/*Ampersand=*/None, *SyntaxContext);
920923
SyntaxContext->addSyntax(std::move(LastNode));
921924
}
922925
}

‎lib/Parse/ParsedSyntaxRecorder.cpp.gyb

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,23 @@ ParsedTupleTypeElementSyntax
172172
ParsedSyntaxRecorder::makeTupleTypeElement(ParsedTypeSyntax Type,
173173
llvm::Optional<ParsedTokenSyntax> TrailingComma,
174174
SyntaxParsingContext &SPCtx) {
175-
return makeTupleTypeElement(None, None, None, None, std::move(Type), None, None,
176-
std::move(TrailingComma), SPCtx);
175+
return makeTupleTypeElement(
176+
/*GarbageNodes=*/None,
177+
/*InOut=*/None,
178+
/*GarbageNodes=*/None,
179+
/*Name=*/None,
180+
/*GarbageNodes=*/None,
181+
/*SecondName=*/None,
182+
/*GarbageNodes=*/None,
183+
/*Colon=*/None,
184+
/*GarbageNodes=*/None,
185+
std::move(Type),
186+
/*GarbageNodes=*/None,
187+
/*Ellipsis=*/None,
188+
/*GarbageNodes=*/None,
189+
/*Initializer=*/None,
190+
/*GarbageNodes=*/None,
191+
std::move(TrailingComma),
192+
SPCtx
193+
);
177194
}

‎lib/Parse/SyntaxParsingContext.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,21 +322,21 @@ void SyntaxParsingContext::collectNodesInPlace(SyntaxKind CollectionKind,
322322
static ParsedRawSyntaxNode finalizeSourceFile(RootContextData &RootData,
323323
MutableArrayRef<ParsedRawSyntaxNode> Parts) {
324324
ParsedRawSyntaxRecorder &Recorder = RootData.Recorder;
325-
ParsedRawSyntaxNode Layout[2];
325+
ParsedRawSyntaxNode Layout[4];
326326

327327
assert(!Parts.empty() && Parts.back().isToken(tok::eof));
328-
Layout[1] = std::move(Parts.back());
328+
Layout[3] = std::move(Parts.back());
329329
Parts = Parts.drop_back();
330330

331331

332332
assert(llvm::all_of(Parts, [](const ParsedRawSyntaxNode& node) {
333333
return node.getKind() == SyntaxKind::CodeBlockItem;
334334
}) && "all top level element must be 'CodeBlockItem'");
335335

336-
Layout[0] = Recorder.recordRawSyntax(SyntaxKind::CodeBlockItemList, Parts);
336+
Layout[1] = Recorder.recordRawSyntax(SyntaxKind::CodeBlockItemList, Parts);
337337

338338
return Recorder.recordRawSyntax(SyntaxKind::SourceFile,
339-
llvm::makeMutableArrayRef(Layout, 2));
339+
llvm::makeMutableArrayRef(Layout, 4));
340340
}
341341

342342
OpaqueSyntaxNode SyntaxParsingContext::finalizeRoot() {

‎lib/Syntax/SyntaxFactory.cpp.gyb

Lines changed: 69 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ SyntaxFactory::countChildren(SyntaxKind Kind){
6565
% for node in SYNTAX_NODES:
6666
% if not node.is_syntax_collection():
6767
case SyntaxKind::${node.syntax_kind}:
68-
% child_count = len(node.children)
69-
% non_optional_child_count = sum(0 if child.is_optional else 1 for child in node.children)
68+
% child_count = len(node.non_garbage_children)
69+
% non_optional_child_count = sum(0 if child.is_optional else 1 for child in node.non_garbage_children)
7070
return {${non_optional_child_count}, ${child_count}};
7171
% end
7272
% end
@@ -243,39 +243,95 @@ SyntaxFactory::makeBlank${node.syntax_kind}() {
243243
% end
244244

245245
TupleTypeSyntax SyntaxFactory::makeVoidTupleType() {
246-
return makeTupleType(makeLeftParenToken({}, {}),
247-
makeBlankTupleTypeElementList(),
248-
makeRightParenToken({}, {}));
246+
return makeTupleType(
247+
/*GarbageNodes=*/None,
248+
/*LeftParen=*/makeLeftParenToken({}, {}),
249+
/*GarbageNodes=*/None,
250+
/*Elements=*/makeBlankTupleTypeElementList(),
251+
/*GarbageNodes=*/None,
252+
/*RightParen=*/makeRightParenToken({}, {})
253+
);
249254
}
250255

251256
TupleTypeElementSyntax
252257
SyntaxFactory::makeTupleTypeElement(llvm::Optional<TokenSyntax> Label,
253258
llvm::Optional<TokenSyntax> Colon,
254259
TypeSyntax Type,
255260
llvm::Optional<TokenSyntax> TrailingComma) {
256-
return makeTupleTypeElement(None, Label, None, Colon, Type, None, None,
257-
TrailingComma);
261+
return makeTupleTypeElement(
262+
/*GarbageNodes=*/None,
263+
/*InOut=*/None,
264+
/*GarbageNodes=*/None,
265+
/*Name=*/Label,
266+
/*GarbageNodes=*/None,
267+
/*SecondName=*/None,
268+
/*GarbageNodes=*/None,
269+
/*Colon=*/Colon,
270+
/*GarbageNodes=*/None,
271+
/*Type=*/Type,
272+
/*GarbageNodes=*/None,
273+
/*Ellipsis=*/None,
274+
/*GarbageNodes=*/None,
275+
/*Intitializer=*/None,
276+
/*GarbageNodes=*/None,
277+
/*TrailingComma=*/TrailingComma
278+
);
258279
}
259280

260281
TupleTypeElementSyntax
261282
SyntaxFactory::makeTupleTypeElement(TypeSyntax Type,
262283
llvm::Optional<TokenSyntax> TrailingComma) {
263-
return makeTupleTypeElement(None, None, None, None, Type, None, None,
264-
TrailingComma);
284+
return makeTupleTypeElement(
285+
/*GarbageNodes=*/None,
286+
/*InOut=*/None,
287+
/*GarbageNodes=*/None,
288+
/*Name=*/None,
289+
/*GarbageNodes=*/None,
290+
/*SecondName=*/None,
291+
/*GarbageNodes=*/None,
292+
/*Colon=*/None,
293+
/*GarbageNodes=*/None,
294+
/*Type=*/Type,
295+
/*GarbageNodes=*/None,
296+
/*Ellipsis=*/None,
297+
/*GarbageNodes=*/None,
298+
/*Initializer=*/None,
299+
/*GarbageNodes=*/None,
300+
/*TrailingComma=*/TrailingComma
301+
);
265302
}
266303

267304
GenericParameterSyntax
268305
SyntaxFactory::makeGenericParameter(TokenSyntax Name,
269306
llvm::Optional<TokenSyntax> TrailingComma) {
270-
return makeGenericParameter(None, Name, None, None, TrailingComma);
307+
return makeGenericParameter(
308+
/*GarbageNodes=*/None,
309+
/*Attributes=*/None,
310+
/*GarbageNodes=*/None,
311+
/*Name=*/Name,
312+
/*GarbageNodes=*/None,
313+
/*Colon=*/None,
314+
/*GarbageNodes=*/None,
315+
/*InheritedType=*/None,
316+
/*GarbageNodes=*/None,
317+
/*TrailingComma=*/TrailingComma
318+
);
271319
}
272320

273321
TypeSyntax SyntaxFactory::makeTypeIdentifier(StringRef TypeName,
274322
StringRef LeadingTrivia,
275323
StringRef TrailingTrivia) {
276-
auto identifier =
277-
makeIdentifier(TypeName, LeadingTrivia, TrailingTrivia);
278-
return makeSimpleTypeIdentifier(identifier, None);
324+
auto identifier = makeIdentifier(
325+
TypeName,
326+
LeadingTrivia,
327+
TrailingTrivia
328+
);
329+
return makeSimpleTypeIdentifier(
330+
/*GarbageNodes=*/None,
331+
/*Name=*/identifier,
332+
/*GarbageNodes=*/None,
333+
/*GenerigArgumentClause=*/None
334+
);
279335
}
280336

281337
TypeSyntax SyntaxFactory::makeAnyTypeIdentifier(StringRef LeadingTrivia,

‎test/Syntax/Inputs/serialize_class_decl.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,27 @@
11
{
22
"kind": "SourceFile",
33
"layout": [
4+
null,
45
{
56
"kind": "CodeBlockItemList",
67
"layout": [
78
{
89
"kind": "CodeBlockItem",
910
"layout": [
11+
null,
1012
{
1113
"kind": "ClassDecl",
1214
"layout": [
15+
null,
16+
null,
1317
null,
1418
{
1519
"kind": "ModifierList",
1620
"layout": [
1721
{
1822
"kind": "DeclModifier",
1923
"layout": [
24+
null,
2025
{
2126
"tokenKind": {
2227
"kind": "contextual_keyword",
@@ -26,13 +31,15 @@
2631
"trailingTrivia": " ",
2732
"presence": "Present"
2833
},
34+
null,
2935
null
3036
],
3137
"presence": "Present"
3238
}
3339
],
3440
"presence": "Present"
3541
},
42+
null,
3643
{
3744
"tokenKind": {
3845
"kind": "kw_class"
@@ -41,6 +48,7 @@
4148
"trailingTrivia": " ",
4249
"presence": "Present"
4350
},
51+
null,
4452
{
4553
"tokenKind": {
4654
"kind": "identifier",
@@ -53,9 +61,14 @@
5361
null,
5462
null,
5563
null,
64+
null,
65+
null,
66+
null,
67+
null,
5668
{
5769
"kind": "MemberDeclBlock",
5870
"layout": [
71+
null,
5972
{
6073
"tokenKind": {
6174
"kind": "l_brace"
@@ -64,11 +77,13 @@
6477
"trailingTrivia": "",
6578
"presence": "Present"
6679
},
80+
null,
6781
{
6882
"kind": "MemberDeclList",
6983
"layout": [],
7084
"presence": "Present"
7185
},
86+
null,
7287
{
7388
"tokenKind": {
7489
"kind": "r_brace"
@@ -84,13 +99,16 @@
8499
"presence": "Present"
85100
},
86101
null,
102+
null,
103+
null,
87104
null
88105
],
89106
"presence": "Present"
90107
}
91108
],
92109
"presence": "Present"
93110
},
111+
null,
94112
{
95113
"tokenKind": {
96114
"kind": "eof",

0 commit comments

Comments
 (0)