Skip to content

Commit a8f5502

Browse files
akyrtziakyrtzi
authored
Merge pull request #21185 from akyrtzi/syntax-parsing-finalize-tree-change
[Parse/Syntax] Simplify how the final SourceFileSyntax root is formed
2 parents 6a18fb8 + cbad349 commit a8f5502

File tree

13 files changed

+125
-87
lines changed

13 files changed

+125
-87
lines changed

‎include/swift/AST/Module.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,6 +1215,8 @@ class SourceFile final : public FileUnit {
12151215

12161216
bool shouldBuildSyntaxTree() const;
12171217

1218+
bool canBeParsedInFull() const;
1219+
12181220
syntax::SourceFileSyntax getSyntaxRoot() const;
12191221
void setSyntaxRoot(syntax::SourceFileSyntax &&Root);
12201222
bool hasSyntaxRoot() const;

‎include/swift/Parse/SyntaxParsingContext.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
363363
/// This function will be called during the destroying of a root syntax
364364
/// parsing context. However, we can explicitly call this function to get
365365
/// the syntax tree before closing the root context.
366-
void finalizeRoot();
366+
RC<RawSyntax> finalizeRoot();
367367

368368
/// Make a missing node corresponding to the given token kind and text, and
369369
/// push this node into the context. The synthesized node can help

‎include/swift/Subsystems.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,14 @@ namespace swift {
121121
PersistentParserState *PersistentState = nullptr,
122122
DelayedParsingCallbacks *DelayedParseCB = nullptr);
123123

124+
/// Parse a single buffer into the given source file, until the full source
125+
/// contents are parsed.
126+
///
127+
/// \return true if the parser found code with side effects.
128+
bool parseIntoSourceFileFull(SourceFile &SF, unsigned BufferID,
129+
PersistentParserState *PersistentState = nullptr,
130+
DelayedParsingCallbacks *DelayedParseCB = nullptr);
131+
124132
/// Finish the parsing by going over the nodes that were delayed
125133
/// during the first parsing pass.
126134
void performDelayedParsing(DeclContext *DC,

‎lib/AST/Module.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1561,11 +1561,15 @@ bool SourceFile::shouldCollectToken() const {
15611561
}
15621562

15631563
bool SourceFile::shouldBuildSyntaxTree() const {
1564+
return canBeParsedInFull() && SyntaxInfo->Enable;
1565+
}
1566+
1567+
bool SourceFile::canBeParsedInFull() const {
15641568
switch (Kind) {
15651569
case SourceFileKind::Library:
15661570
case SourceFileKind::Main:
15671571
case SourceFileKind::Interface:
1568-
return SyntaxInfo->Enable;
1572+
return true;
15691573
case SourceFileKind::REPL:
15701574
case SourceFileKind::SIL:
15711575
return false;

‎lib/Frontend/Frontend.cpp

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,13 +1056,7 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
10561056
SourceFileKind::Library, SourceFile::ImplicitModuleImportKind::None,
10571057
BufferID);
10581058

1059-
bool Done;
1060-
do {
1061-
// Parser may stop at some erroneous constructions like #else, #endif
1062-
// or '}' in some cases, continue parsing until we are done
1063-
parseIntoSourceFile(*NextInput, BufferID, &Done, nullptr,
1064-
&PersistentState, nullptr);
1065-
} while (!Done);
1059+
parseIntoSourceFileFull(*NextInput, BufferID, &PersistentState);
10661060
}
10671061

10681062
// Now parse the main file.
@@ -1071,11 +1065,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
10711065
MainModule->getMainSourceFile(Invocation.getSourceFileKind());
10721066
MainFile.SyntaxParsingCache = Invocation.getMainFileSyntaxParsingCache();
10731067

1074-
bool Done;
1075-
do {
1076-
parseIntoSourceFile(MainFile, MainFile.getBufferID().getValue(), &Done,
1077-
nullptr, &PersistentState, nullptr);
1078-
} while (!Done);
1068+
parseIntoSourceFileFull(MainFile, MainFile.getBufferID().getValue(),
1069+
&PersistentState);
10791070
}
10801071

10811072
assert(Context->LoadedModules.size() == 1 &&

‎lib/Parse/ParseDecl.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2410,8 +2410,12 @@ static void diagnoseOperatorFixityAttributes(Parser &P,
24102410
static unsigned skipUntilMatchingRBrace(Parser &P, bool &HasPoundDirective,
24112411
SyntaxParsingContext *&SyntaxContext) {
24122412
HasPoundDirective = false;
2413+
bool isRootCtx = SyntaxContext->isRoot();
24132414
SyntaxParsingContext BlockItemListContext(SyntaxContext,
24142415
SyntaxKind::CodeBlockItemList);
2416+
if (isRootCtx) {
2417+
BlockItemListContext.setTransparent();
2418+
}
24152419
SyntaxParsingContext BlockItemContext(SyntaxContext,
24162420
SyntaxKind::CodeBlockItem);
24172421
SyntaxParsingContext BodyContext(SyntaxContext, SyntaxKind::TokenList);

‎lib/Parse/ParseStmt.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,12 @@ void Parser::consumeTopLevelDecl(ParserPosition BeginParserPosition,
266266
ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
267267
BraceItemListKind Kind,
268268
BraceItemListKind ConditionalBlockKind) {
269+
bool isRootCtx = SyntaxContext->isRoot();
269270
SyntaxParsingContext ItemListContext(SyntaxContext,
270271
SyntaxKind::CodeBlockItemList);
272+
if (isRootCtx) {
273+
ItemListContext.setTransparent();
274+
}
271275

272276
bool IsTopLevel = (Kind == BraceItemListKind::TopLevelCode) ||
273277
(Kind == BraceItemListKind::TopLevelLibrary);

‎lib/Parse/SyntaxParsingContext.cpp

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -284,41 +284,31 @@ class SyntaxVerifier: public SyntaxVisitor {
284284
}
285285
};
286286

287-
namespace {
288-
void finalizeSourceFile(RootContextData &RootData,
287+
static RC<RawSyntax> finalizeSourceFile(RootContextData &RootData,
289288
ArrayRef<RC<RawSyntax>> Parts) {
290289
SourceFile &SF = RootData.SF;
291290
RC<SyntaxArena> &Arena = RootData.Arena;
292291
std::vector<RC<RawSyntax>> AllTopLevel;
293292
RC<RawSyntax> EOFToken;
294293

295-
if (SF.hasSyntaxRoot()) {
296-
auto SourceRaw = SF.getSyntaxRoot().getRaw();
297-
auto Decls =
298-
SourceRaw->getChild(SourceFileSyntax::Cursor::Statements)->getLayout();
299-
std::copy(Decls.begin(), Decls.end(), std::back_inserter(AllTopLevel));
300-
EOFToken = SourceRaw->getChild(SourceFileSyntax::Cursor::EOFToken);
301-
}
302-
303294
if (!Parts.empty() && Parts.back()->isToken(tok::eof)) {
304295
EOFToken = Parts.back();
305296
Parts = Parts.drop_back();
306297
}
307298

299+
if (!EOFToken)
300+
EOFToken =
301+
RawSyntax::missing(tok::eof, OwnedString::makeUnowned(""), Arena);
302+
308303
for (auto RawNode : Parts) {
309-
if (RawNode->getKind() != SyntaxKind::CodeBlockItemList)
304+
if (RawNode->getKind() != SyntaxKind::CodeBlockItem)
310305
// FIXME: Skip toplevel garbage nodes for now. we shouldn't emit them in
311306
// the first place.
312307
continue;
313308

314-
auto Items = RawNode->getLayout();
315-
std::copy(Items.begin(), Items.end(), std::back_inserter(AllTopLevel));
309+
AllTopLevel.push_back(RawNode);
316310
}
317311

318-
if (!EOFToken)
319-
EOFToken =
320-
RawSyntax::missing(tok::eof, OwnedString::makeUnowned(""), Arena);
321-
322312
auto newRaw = SyntaxFactory::createRaw(
323313
SyntaxKind::SourceFile,
324314
{
@@ -328,29 +318,36 @@ void finalizeSourceFile(RootContextData &RootData,
328318
},
329319
Arena);
330320
assert(newRaw);
331-
SF.setSyntaxRoot(make<SourceFileSyntax>(newRaw));
332321

333322
// Verify the tree if specified.
334323
// Do this only when we see the real EOF token because parseIntoSourceFile()
335324
// can get called multiple times for single source file.
336325
if (EOFToken->isPresent() && SF.getASTContext().LangOpts.VerifySyntaxTree) {
337326
SyntaxVerifier Verifier(RootData);
338-
Verifier.verify(SF.getSyntaxRoot());
327+
Verifier.verify(make<SourceFileSyntax>(newRaw));
339328
}
329+
return newRaw;
340330
}
341-
} // End of anonymous namespace
342331

343-
void SyntaxParsingContext::finalizeRoot() {
332+
RC<RawSyntax> SyntaxParsingContext::finalizeRoot() {
344333
if (!Enabled)
345-
return;
334+
return nullptr;
346335
assert(isTopOfContextStack() && "some sub-contexts are not destructed");
347336
assert(isRoot() && "only root context can finalize the tree");
348337
assert(Mode == AccumulationMode::Root);
349-
finalizeSourceFile(*getRootData(), getParts());
338+
auto &RootData = *getRootData();
339+
SourceFile &SF = RootData.SF;
340+
if (SF.hasSyntaxRoot())
341+
return SF.getSyntaxRoot().getRaw();
342+
343+
auto rawRoot = finalizeSourceFile(RootData, getParts());
350344

351345
// Clear the parts because we will call this function again when destroying
352346
// the root context.
353347
getStorage().clear();
348+
349+
SF.setSyntaxRoot(make<SourceFileSyntax>(rawRoot));
350+
return rawRoot;
354351
}
355352

356353
void SyntaxParsingContext::synthesize(tok Kind, StringRef Text) {
@@ -441,7 +438,7 @@ SyntaxParsingContext::~SyntaxParsingContext() {
441438
}
442439
}
443440

444-
bool shouldCacheNode(tok TokKind, OwnedString &Text,
441+
static bool shouldCacheNode(tok TokKind, OwnedString &Text,
445442
llvm::ArrayRef<TriviaPiece> LeadingTrivia,
446443
llvm::ArrayRef<TriviaPiece> TrailingTrivia) {
447444
// Is string_literal with >16 length.

‎lib/ParseSIL/ParseSIL.cpp

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,16 @@ void PrettyStackTraceParser::print(llvm::raw_ostream &out) const {
107107
out << '\n';
108108
}
109109

110-
bool swift::parseIntoSourceFile(SourceFile &SF,
110+
static bool parseIntoSourceFileImpl(SourceFile &SF,
111111
unsigned BufferID,
112112
bool *Done,
113113
SILParserState *SIL,
114114
PersistentParserState *PersistentState,
115-
DelayedParsingCallbacks *DelayedParseCB) {
115+
DelayedParsingCallbacks *DelayedParseCB,
116+
bool FullParse) {
117+
assert((!FullParse || (SF.canBeParsedInFull() && !SIL)) &&
118+
"cannot parse in full with the given parameters!");
119+
116120
SharedTimer timer("Parsing");
117121
Parser P(BufferID, SF, SIL ? SIL->Impl.get() : nullptr, PersistentState);
118122
PrettyStackTraceParser StackTrace(P);
@@ -122,12 +126,36 @@ bool swift::parseIntoSourceFile(SourceFile &SF,
122126
if (DelayedParseCB)
123127
P.setDelayedParsingCallbacks(DelayedParseCB);
124128

125-
bool FoundSideEffects = P.parseTopLevel();
126-
*Done = P.Tok.is(tok::eof);
129+
bool FoundSideEffects = false;
130+
do {
131+
bool hasSideEffects = P.parseTopLevel();
132+
FoundSideEffects = FoundSideEffects || hasSideEffects;
133+
*Done = P.Tok.is(tok::eof);
134+
} while (FullParse && !*Done);
127135

128136
return FoundSideEffects;
129137
}
130138

139+
bool swift::parseIntoSourceFile(SourceFile &SF,
140+
unsigned BufferID,
141+
bool *Done,
142+
SILParserState *SIL,
143+
PersistentParserState *PersistentState,
144+
DelayedParsingCallbacks *DelayedParseCB) {
145+
return parseIntoSourceFileImpl(SF, BufferID, Done, SIL,
146+
PersistentState, DelayedParseCB,
147+
/*FullParse=*/SF.shouldBuildSyntaxTree());
148+
}
149+
150+
bool swift::parseIntoSourceFileFull(SourceFile &SF, unsigned BufferID,
151+
PersistentParserState *PersistentState,
152+
DelayedParsingCallbacks *DelayedParseCB) {
153+
bool Done = false;
154+
return parseIntoSourceFileImpl(SF, BufferID, &Done, /*SIL=*/nullptr,
155+
PersistentState, DelayedParseCB,
156+
/*FullParse=*/true);
157+
}
158+
131159

132160
//===----------------------------------------------------------------------===//
133161
// SILParser

‎test/Syntax/Inputs/serialize_multiple_decls.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
2-
"id": 18,
2+
"id": 17,
33
"kind": "SourceFile",
44
"layout": [
55
{
6-
"id": 17,
6+
"id": 16,
77
"kind": "CodeBlockItemList",
88
"layout": [
99
{
@@ -204,7 +204,7 @@
204204
"presence": "Present"
205205
},
206206
{
207-
"id": 16,
207+
"id": 15,
208208
"tokenKind": {
209209
"kind": "eof",
210210
"text": ""

‎test/Syntax/Inputs/serialize_struct_decl.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
2-
"id": 40,
2+
"id": 39,
33
"kind": "SourceFile",
44
"layout": [
55
{
6-
"id": 39,
6+
"id": 38,
77
"kind": "CodeBlockItemList",
88
"layout": [
99
{
@@ -426,7 +426,7 @@
426426
"presence": "Present"
427427
},
428428
{
429-
"id": 38,
429+
"id": 37,
430430
"tokenKind": {
431431
"kind": "eof",
432432
"text": ""

‎test/incrParse/Outputs/extend-identifier-at-eof.json

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,33 @@
11
{
2-
"id": 39,
2+
"id": 37,
33
"kind": "SourceFile",
44
"layout": [
55
{
6-
"id": 38,
6+
"id": 36,
77
"kind": "CodeBlockItemList",
88
"layout": [
99
{
1010
"id": 13,
1111
"omitted": true
1212
},
1313
{
14-
"id": 35,
14+
"id": 34,
1515
"kind": "CodeBlockItem",
1616
"layout": [
1717
{
18-
"id": 34,
18+
"id": 33,
1919
"kind": "SequenceExpr",
2020
"layout": [
2121
{
22-
"id": 33,
22+
"id": 32,
2323
"kind": "ExprList",
2424
"layout": [
2525
{
26-
"id": 28,
26+
"id": 27,
2727
"kind": "DiscardAssignmentExpr",
2828
"layout": [
2929
{
30-
"id": 27,
30+
"id": 26,
3131
"tokenKind": {
3232
"kind": "kw__"
3333
},
@@ -65,11 +65,11 @@
6565
"presence": "Present"
6666
},
6767
{
68-
"id": 30,
68+
"id": 29,
6969
"kind": "AssignmentExpr",
7070
"layout": [
7171
{
72-
"id": 29,
72+
"id": 28,
7373
"tokenKind": {
7474
"kind": "equal"
7575
},
@@ -86,11 +86,11 @@
8686
"presence": "Present"
8787
},
8888
{
89-
"id": 32,
89+
"id": 31,
9090
"kind": "IdentifierExpr",
9191
"layout": [
9292
{
93-
"id": 31,
93+
"id": 30,
9494
"tokenKind": {
9595
"kind": "identifier",
9696
"text": "xx"
@@ -118,7 +118,7 @@
118118
"presence": "Present"
119119
},
120120
{
121-
"id": 37,
121+
"id": 35,
122122
"tokenKind": {
123123
"kind": "eof",
124124
"text": ""

0 commit comments

Comments
 (0)