Skip to content

Commit a46aa28

Browse files
ahoppenJan Svobodarintaro
committed
[syntax-parse] Parse integer literals through the libSyntax tree
This commit re-applies changes that have been applied, reversed, re-applied and re-reversed. Credit for the original implementation goes to Jan Svoboda <[email protected]> for writing the original implementation and Rintaro Ishizaki <[email protected]> who review it and applied some fixes. Co-authored-by: Jan Svoboda <[email protected]> Co-authored-by: Rintaro Ishizaki <[email protected]>
1 parent 4e13315 commit a46aa28

File tree

9 files changed

+277
-2
lines changed

9 files changed

+277
-2
lines changed

‎include/swift/Parse/HiddenLibSyntaxAction.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ class HiddenLibSyntaxAction : public SyntaxParseActions {
108108
/// Returns the node created by explicit syntax action from the specified
109109
/// node that has been created by this action.
110110
OpaqueSyntaxNode getExplicitNodeFor(OpaqueSyntaxNode node);
111+
112+
/// Returns the underlying libSyntax \c SyntaxTreeCreator.
113+
std::shared_ptr<SyntaxTreeCreator> getLibSyntaxAction() {
114+
return LibSyntaxAction;
115+
}
111116
};
112117
} // namespace swift
113118

‎include/swift/Parse/LibSyntaxGenerator.h

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
//===----------- LibSyntaxGenerator.h -------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef SWIFT_PARSE_LIBSYNTAXGENERATOR_H
14+
#define SWIFT_PARSE_LIBSYNTAXGENERATOR_H
15+
16+
#include "swift/Parse/HiddenLibSyntaxAction.h"
17+
#include "swift/Parse/ParsedRawSyntaxNode.h"
18+
#include "swift/Parse/ParsedRawSyntaxRecorder.h"
19+
#include "swift/Parse/SyntaxParsingCache.h"
20+
#include "swift/Syntax/RawSyntax.h"
21+
#include "swift/SyntaxParse/SyntaxTreeCreator.h"
22+
23+
namespace swift {
24+
// TODO: (swift-parse) remove when possible
25+
/// Generates libSyntax nodes either by looking them up using
26+
/// HiddenLibSyntaxAction (based on provided OpaqueSyntaxNode) or by recording
27+
/// them with ParsedRawSyntaxRecorder.
28+
class LibSyntaxGenerator {
29+
std::shared_ptr<HiddenLibSyntaxAction> Actions;
30+
ParsedRawSyntaxRecorder Recorder;
31+
32+
public:
33+
explicit LibSyntaxGenerator(std::shared_ptr<HiddenLibSyntaxAction> spActions)
34+
: Actions(std::move(spActions)), Recorder(Actions->getLibSyntaxAction()) {
35+
}
36+
37+
/// Create a \c TokenSyntax from the raw data and record it in the
38+
/// \c HiddenLibSyntaxAction's \c LibSyntaxAction.
39+
TokenSyntax createToken(ParsedRawSyntaxNode Node) {
40+
assert(Node.isDeferredToken());
41+
42+
auto Kind = Node.getTokenKind();
43+
auto Range = Node.getDeferredTokenRangeWithTrivia();
44+
auto LeadingTriviaPieces = Node.getDeferredLeadingTriviaPieces();
45+
auto TrailingTriviaPieces = Node.getDeferredTrailingTriviaPieces();
46+
47+
auto Recorded = Recorder.recordToken(Kind, Range, LeadingTriviaPieces,
48+
TrailingTriviaPieces);
49+
auto Raw = static_cast<RawSyntax *>(Recorded.getOpaqueNode());
50+
return make<TokenSyntax>(Raw);
51+
}
52+
53+
/// Create a \c SyntaxNode from the raw data and record it in the
54+
/// \c HiddenLibSyntaxAction's \c LibSyntaxAction.
55+
template <typename SyntaxNode>
56+
SyntaxNode createNode(ParsedRawSyntaxNode Node) {
57+
assert(Node.isDeferredLayout());
58+
auto Kind = Node.getKind();
59+
auto Children = Node.getDeferredChildren();
60+
61+
auto Recorded = Recorder.recordRawSyntax(Kind, Children);
62+
RC<RawSyntax> Raw{static_cast<RawSyntax *>(Recorded.takeOpaqueNode())};
63+
Raw->Release(); // -1 since it's transfer of ownership.
64+
return make<SyntaxNode>(Raw);
65+
}
66+
67+
/// Return the libSyntax node stored in the given opaque \c Node.
68+
/// Assumes that \c Node is of type \c HiddenNode.
69+
TokenSyntax getLibSyntaxTokenFor(OpaqueSyntaxNode Node) {
70+
return getLibSyntaxNodeFor<TokenSyntax>(Node);
71+
}
72+
73+
/// Return the libSyntax node stored in the given opaque \c Node.
74+
/// Assumes that \c Node is of type \c HiddenNode.
75+
template <typename SyntaxNode>
76+
SyntaxNode getLibSyntaxNodeFor(OpaqueSyntaxNode Node) {
77+
return make<SyntaxNode>(Actions->getLibSyntaxNodeFor(Node));
78+
}
79+
};
80+
} // namespace swift
81+
82+
#endif // SWIFT_PARSE_LIBSYNTAXGENERATOR_H

‎include/swift/Parse/ParsedSyntaxRecorder.h.gyb

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,29 @@ public:
6565

6666
static Parsed${node.name} makeBlank${node.syntax_kind}(SourceLoc loc,
6767
SyntaxParsingContext &SPCtx);
68+
% elif node.is_unknown():
69+
private:
70+
static Parsed${node.name} record${node.syntax_kind}(
71+
MutableArrayRef<ParsedRawSyntaxNode> elts,
72+
ParsedRawSyntaxRecorder &rec);
73+
static Parsed${node.name} defer${node.syntax_kind}(
74+
MutableArrayRef<ParsedRawSyntaxNode> layout,
75+
SyntaxParsingContext &SPCtx);
76+
public:
77+
static Parsed${node.name} make${node.syntax_kind}(
78+
MutableArrayRef<ParsedSyntax> elts,
79+
SyntaxParsingContext &SPCtx);
6880
% end
6981
% end
7082

7183
#pragma mark - Convenience APIs
7284

85+
public:
86+
static ParsedTokenSyntax makeToken(const Token &Tok,
87+
const ParsedTrivia &LeadingTrivia,
88+
const ParsedTrivia &TrailingTrivia,
89+
SyntaxParsingContext &SPCtx);
90+
7391
/// Records an unlabelled TupleTypeElementSyntax with the provided type and
7492
/// optional trailing comma.
7593
static ParsedTupleTypeElementSyntax

‎include/swift/Parse/Parser.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@
2626
#include "swift/AST/Pattern.h"
2727
#include "swift/AST/Stmt.h"
2828
#include "swift/Basic/OptionSet.h"
29+
#include "swift/Parse/ASTGen.h"
2930
#include "swift/Parse/Lexer.h"
3031
#include "swift/Parse/LocalContext.h"
3132
#include "swift/Parse/PersistentParserState.h"
3233
#include "swift/Parse/Token.h"
34+
#include "swift/Parse/ParsedSyntaxNodes.h"
3335
#include "swift/Parse/ParserPosition.h"
3436
#include "swift/Parse/ParserResult.h"
3537
#include "swift/Parse/SyntaxParsingContext.h"
@@ -405,6 +407,9 @@ class Parser {
405407
/// Has \c AvailabilityMacros been computed?
406408
bool AvailabilityMacrosComputed = false;
407409

410+
/// The AST generator that generates AST nodes from libSyntax nodes.
411+
ASTGen ASTGenerator;
412+
408413
public:
409414
Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
410415
SILParserStateBase *SIL, PersistentParserState *PersistentState,
@@ -892,6 +897,16 @@ class Parser {
892897
}
893898
ParserResult<BraceStmt> parseBraceItemList(Diag<> ID);
894899

900+
//===--------------------------------------------------------------------===//
901+
// MARK: - Primitive Parsing using libSyntax
902+
903+
/// Consume a token and return the corresponding \c ParsedTokenSyntax.
904+
ParsedTokenSyntax consumeTokenSyntax();
905+
ParsedTokenSyntax consumeTokenSyntax(tok K) {
906+
assert(Tok.is(K) && "Consuming wrong token kind");
907+
return consumeTokenSyntax();
908+
}
909+
895910
//===--------------------------------------------------------------------===//
896911
// Decl Parsing
897912

@@ -1616,6 +1631,21 @@ class Parser {
16161631

16171632
void validateCollectionElement(ParserResult<Expr> element);
16181633

1634+
//===--------------------------------------------------------------------===//
1635+
// MARK: - Expression parsing using libSyntax
1636+
1637+
// TODO: (syntax-parse) remove when possible
1638+
/// Parse the upcoming expression of type \c SyntaxNode through libSyntax,
1639+
/// i.e. parse it into a libSyntax node and generate the \c Expr node from the
1640+
/// libSyntax node using \c ASTGen.
1641+
template <typename SyntaxNode> ParserResult<Expr> parseExprAST();
1642+
1643+
// TODO: (syntax-parse) create new result type for ParsedSyntax
1644+
// TODO: (syntax-parse) turn into proper non-templated methods later
1645+
/// Parse the upcoming expression of type \c SyntaxNode into a libSyntax node
1646+
/// and return it.
1647+
template <typename SyntaxNode> ParsedExprSyntax parseExprSyntax();
1648+
16191649
//===--------------------------------------------------------------------===//
16201650
// Statement Parsing
16211651

‎include/swift/Parse/SyntaxParsingContext.h

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@
1616
#include "swift/Basic/Debug.h"
1717
#include "swift/Basic/SourceLoc.h"
1818
#include "swift/Parse/HiddenLibSyntaxAction.h"
19+
#include "swift/Parse/LibSyntaxGenerator.h"
1920
#include "swift/Parse/ParsedRawSyntaxNode.h"
2021
#include "swift/Parse/ParsedRawSyntaxRecorder.h"
22+
#include "swift/Parse/ParsedSyntaxNodes.h"
2123
#include "llvm/ADT/PointerUnion.h"
2224

2325
namespace swift {
@@ -61,6 +63,7 @@ enum class SyntaxNodeCreationKind {
6163

6264
constexpr size_t SyntaxAlignInBits = 3;
6365

66+
// TODO: (syntax-parse) remove when/if possible
6467
/// RAII object which receive RawSyntax parts. On destruction, this constructs
6568
/// a specified syntax node from received parts and propagate it to the parent
6669
/// context.
@@ -97,13 +100,17 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
97100

98101
ParsedRawSyntaxRecorder Recorder;
99102

103+
/// The \c LibSyntaxGenerator that produces libSyntax node from the data
104+
/// recorded in this \c SyntaxParsingContext. See \c topNode() below.
105+
LibSyntaxGenerator LibSyntaxCreator;
106+
100107
llvm::BumpPtrAllocator ScratchAlloc;
101108

102109
RootContextData(SourceFile &SF, DiagnosticEngine &Diags,
103110
SourceManager &SourceMgr, unsigned BufferID,
104111
std::shared_ptr<HiddenLibSyntaxAction> spActions)
105112
: SF(SF), Diags(Diags), SourceMgr(SourceMgr), BufferID(BufferID),
106-
Recorder(std::move(spActions)) {}
113+
Recorder(spActions), LibSyntaxCreator(spActions) {}
107114
};
108115

109116
private:
@@ -251,6 +258,10 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
251258
return getRootData()->Storage;
252259
}
253260

261+
LibSyntaxGenerator &getSyntaxCreator() {
262+
return getRootData()->LibSyntaxCreator;
263+
}
264+
254265
const SyntaxParsingContext *getRoot() const;
255266

256267
ParsedRawSyntaxRecorder &getRecorder() { return getRootData()->Recorder; }
@@ -269,6 +280,20 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
269280
/// Add Syntax to the parts.
270281
void addSyntax(ParsedSyntax Node);
271282

283+
/// Whether the top node on the parsing context's storage stack is of the
284+
/// type \c SyntaxNode.
285+
template <typename SyntaxNode> bool isTopNode() {
286+
auto parts = getParts();
287+
return (!parts.empty() && SyntaxNode::kindof(parts.back().getKind()));
288+
}
289+
290+
/// Creates a parsed libSyntax node from the top node of the parsing context's
291+
/// storage stack. If the node has already been recorded, the data stored in
292+
/// the \c LibSyntaxNode of the corresponding \c HiddenNode is returned.
293+
/// If the node is deferred, the node will be recorded in the
294+
/// \c LibSyntaxAction and returned.
295+
template <typename SyntaxNode> SyntaxNode topNode();
296+
272297
template<typename SyntaxNode>
273298
llvm::Optional<SyntaxNode> popIf() {
274299
auto &Storage = getStorage();
@@ -354,6 +379,27 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
354379
SWIFT_DEBUG_DUMPER(dumpStorage());
355380
};
356381

382+
template <typename SyntaxNode>
383+
inline SyntaxNode SyntaxParsingContext::topNode() {
384+
assert(isTopNode<SyntaxNode>());
385+
ParsedRawSyntaxNode &TopNode = getStorage().back();
386+
if (TopNode.isRecorded()) {
387+
OpaqueSyntaxNode OpaqueNode = TopNode.getOpaqueNode();
388+
return getSyntaxCreator().getLibSyntaxNodeFor<SyntaxNode>(OpaqueNode);
389+
}
390+
return getSyntaxCreator().createNode<SyntaxNode>(TopNode.copyDeferred());
391+
}
392+
393+
template <> inline TokenSyntax SyntaxParsingContext::topNode<TokenSyntax>() {
394+
assert(isTopNode<TokenSyntax>());
395+
ParsedRawSyntaxNode &TopNode = getStorage().back();
396+
if (TopNode.isRecorded()) {
397+
OpaqueSyntaxNode OpaqueNode = TopNode.getOpaqueNode();
398+
return getSyntaxCreator().getLibSyntaxNodeFor<TokenSyntax>(OpaqueNode);
399+
}
400+
return getSyntaxCreator().createToken(TopNode.copyDeferred());
401+
}
402+
357403
/// Ensure that the \c SyntaxParsingContext is enabled while this RAII is alive.
358404
/// If the context was previously enabled, do nothing. Otherwise, enable the
359405
/// context and set its accumulation mode to discard. Once the RAII is destroyed

‎lib/Parse/ParseExpr.cpp

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1409,6 +1409,15 @@ ParserResult<Expr> Parser::parseExprPostfix(Diag<> ID, bool isExprBasic) {
14091409
/// expr-selector
14101410
///
14111411
ParserResult<Expr> Parser::parseExprPrimary(Diag<> ID, bool isExprBasic) {
1412+
// Parsing of expressions through the libSyntax tree
1413+
switch (Tok.getKind()) {
1414+
case tok::integer_literal:
1415+
return parseExprAST<IntegerLiteralExprSyntax>();
1416+
default:
1417+
break;
1418+
}
1419+
1420+
// Direct parsing of tokens to an AST
14121421
SyntaxParsingContext ExprContext(SyntaxContext, SyntaxContextKind::Expr);
14131422
switch (Tok.getKind()) {
14141423
case tok::integer_literal: {
@@ -3702,3 +3711,29 @@ Parser::parsePlatformVersionConstraintSpec() {
37023711
return makeParserResult(new (Context) PlatformVersionConstraintAvailabilitySpec(
37033712
Platform.getValue(), PlatformLoc, Version, RuntimeVersion, VersionRange));
37043713
}
3714+
3715+
//===----------------------------------------------------------------------===//
3716+
// MARK: - Expression parsing using libSyntax
3717+
3718+
template <typename SyntaxNode> ParserResult<Expr> Parser::parseExprAST() {
3719+
EnableContextRAII enableSyntaxParsing(*SyntaxContext);
3720+
3721+
auto Loc = leadingTriviaLoc();
3722+
SyntaxContext->addSyntax(parseExprSyntax<SyntaxNode>());
3723+
// TODO: (syntax-parse) improve this somehow
3724+
if (SyntaxContext->isTopNode<UnknownExprSyntax>()) {
3725+
auto Expr = SyntaxContext->topNode<UnknownExprSyntax>();
3726+
auto ExprAST = ASTGenerator.generate(Expr, Loc);
3727+
return makeParserResult(ExprAST);
3728+
}
3729+
auto Expr = SyntaxContext->topNode<SyntaxNode>();
3730+
auto ExprAST = ASTGenerator.generate(Expr, Loc);
3731+
return makeParserResult(ExprAST);
3732+
}
3733+
3734+
template <>
3735+
ParsedExprSyntax Parser::parseExprSyntax<IntegerLiteralExprSyntax>() {
3736+
auto Token = consumeTokenSyntax(tok::integer_literal);
3737+
return ParsedSyntaxRecorder::makeIntegerLiteralExpr(std::move(Token),
3738+
*SyntaxContext);
3739+
}

‎lib/Parse/ParsedSyntaxRecorder.cpp.gyb

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,9 +163,54 @@ ParsedSyntaxRecorder::makeBlank${node.syntax_kind}(SourceLoc loc,
163163
}
164164
return Parsed${node.name}(std::move(raw));
165165
}
166+
% elif node.is_unknown():
167+
Parsed${node.name}
168+
ParsedSyntaxRecorder::record${node.syntax_kind}(
169+
MutableArrayRef<ParsedRawSyntaxNode> layout,
170+
ParsedRawSyntaxRecorder &rec) {
171+
auto raw = rec.recordRawSyntax(SyntaxKind::${node.syntax_kind}, layout);
172+
return Parsed${node.name}(std::move(raw));
173+
}
174+
175+
Parsed${node.name}
176+
ParsedSyntaxRecorder::defer${node.syntax_kind}(
177+
MutableArrayRef<ParsedRawSyntaxNode> layout,
178+
SyntaxParsingContext &SPCtx) {
179+
auto raw = ParsedRawSyntaxNode::makeDeferred(SyntaxKind::${node.syntax_kind},
180+
layout, SPCtx);
181+
return Parsed${node.name}(std::move(raw));
182+
}
183+
184+
Parsed${node.name}
185+
ParsedSyntaxRecorder::make${node.syntax_kind}(
186+
MutableArrayRef<ParsedSyntax> elements,
187+
SyntaxParsingContext &SPCtx) {
188+
SmallVector<ParsedRawSyntaxNode, 16> layout;
189+
layout.reserve(elements.size());
190+
for (auto &element : elements) {
191+
layout.push_back(element.takeRaw());
192+
}
193+
if (SPCtx.shouldDefer())
194+
return defer${node.syntax_kind}(layout, SPCtx);
195+
return record${node.syntax_kind}(layout, SPCtx.getRecorder());
196+
}
166197
% end
167198
% end
168199

200+
ParsedTokenSyntax
201+
ParsedSyntaxRecorder::makeToken(const Token &Tok,
202+
const ParsedTrivia &LeadingTrivia,
203+
const ParsedTrivia &TrailingTrivia,
204+
SyntaxParsingContext &SPCtx) {
205+
ParsedRawSyntaxNode raw;
206+
if (SPCtx.shouldDefer()) {
207+
raw = ParsedRawSyntaxNode::makeDeferred(Tok, LeadingTrivia, TrailingTrivia, SPCtx);
208+
} else {
209+
raw = SPCtx.getRecorder().recordToken(Tok, LeadingTrivia, TrailingTrivia);
210+
}
211+
return ParsedTokenSyntax(std::move(raw));
212+
}
213+
169214
ParsedTupleTypeElementSyntax
170215
ParsedSyntaxRecorder::makeTupleTypeElement(ParsedTypeSyntax Type,
171216
llvm::Optional<ParsedTokenSyntax> TrailingComma,

0 commit comments

Comments
 (0)