Skip to content

[Parser] Decouple the parser from AST creation (part 2) #26478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Aug 27, 2019
Merged

[Parser] Decouple the parser from AST creation (part 2) #26478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
77924c4
[Parser] Decouple the parser from AST creation (part 2)
Aug 3, 2019
a705127
Use SourceLoc as key in type map
Aug 6, 2019
773cf51
Update ASTGen documentation.
Aug 6, 2019
cc1aa01
Handle type parsing errors properly
Aug 14, 2019
0135503
Consume any token in identifier parsing
Aug 14, 2019
6de2764
Use correct SourceLoc (before trivia) for Any type parsing
Aug 14, 2019
6acfc8b
Allow nullptr as output of ASTGen
Aug 14, 2019
8b5ca75
Fix attribute @ SourceLoc in ASTGen
Aug 14, 2019
9ae61aa
Handle failed tuple parsing
Aug 15, 2019
52d999d
[SyntaxParse] Use ArrayRef to pass list of nodes to ParsedSyntaxResult
rintaro Aug 20, 2019
80e6d27
[SyntaxParse] ParseError is not equivalent to code completion
rintaro Aug 21, 2019
1551db6
[SyntaxParse] Support kindof() in isTopNode()
rintaro Aug 21, 2019
c8ac6b6
[SyntaxParse] Fix memory leak in incomplete composition parsing
rintaro Aug 19, 2019
dc0cf99
[SyntaxParse] Fix memory leak
rintaro Aug 21, 2019
e609a3a
[SyntaxParse] Fix memory leak for invalid generic argument clause
rintaro Aug 21, 2019
a333a5a
[SyntaxParse] Fix memory leak for incomplete function types
rintaro Aug 21, 2019
a5cb907
[SyntaxParse] Fix memory leak for incomplete type identifier
rintaro Aug 22, 2019
3f4a4c0
[SyntaxParse] Don't record tokens while skipping
rintaro Aug 16, 2019
be0ba7d
Skip unnecessary start-of-line check in ASTGen for keywords
Aug 23, 2019
02a323e
Add comments to ASTGen's UnknownTypeSyntax recovery
Aug 23, 2019
ad5b825
[Lexer] Don't lex trailing trivia for eof (including artificial eof)
rintaro Aug 26, 2019
8da82d3
Revert Lexer UTF8 replacement char changes
Aug 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 82 additions & 28 deletions include/swift/Parse/ASTGen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,54 +14,108 @@
#define SWIFT_PARSE_ASTGEN_H

#include "swift/AST/ASTContext.h"
#include "swift/AST/Decl.h"
#include "swift/AST/Expr.h"
#include "swift/Parse/PersistentParserState.h"
#include "swift/Syntax/SyntaxNodes.h"
#include "llvm/ADT/DenseMap.h"

namespace swift {
/// Generates AST nodes from Syntax nodes.
class ASTGen {
ASTContext &Context;
// A stack of source locations of syntax constructs. Allows us to get the
// SourceLoc necessary to create AST nodes for nodes in not-yet-complete
// Syntax tree. The topmost item should always correspond to the token/node
// that has been parsed/transformed most recently.
// todo [gsoc]: remove when possible
llvm::SmallVector<SourceLoc, 16> LocStack;

/// Type cache to prevent multiple transformations of the same syntax node.
llvm::DenseMap<syntax::SyntaxNodeId, TypeRepr *> TypeCache;

PersistentParserState **ParserState;

// FIXME: remove when Syntax can represent all types and ASTGen can handle them
/// Types that cannot be represented by Syntax or generated by ASTGen.
llvm::DenseMap<SourceLoc, TypeRepr *> Types;

public:
explicit ASTGen(ASTContext &Context) : Context(Context) {}

IntegerLiteralExpr *generate(syntax::IntegerLiteralExprSyntax &Expr);
FloatLiteralExpr *generate(syntax::FloatLiteralExprSyntax &Expr);
NilLiteralExpr *generate(syntax::NilLiteralExprSyntax &Expr);
BooleanLiteralExpr *generate(syntax::BooleanLiteralExprSyntax &Expr);
MagicIdentifierLiteralExpr *generate(syntax::PoundFileExprSyntax &Expr);
MagicIdentifierLiteralExpr *generate(syntax::PoundLineExprSyntax &Expr);
MagicIdentifierLiteralExpr *generate(syntax::PoundColumnExprSyntax &Expr);
MagicIdentifierLiteralExpr *generate(syntax::PoundFunctionExprSyntax &Expr);
MagicIdentifierLiteralExpr *generate(syntax::PoundDsohandleExprSyntax &Expr);
Expr *generate(syntax::UnknownExprSyntax &Expr);

/// Stores source location necessary for AST creation.
void pushLoc(SourceLoc Loc);
ASTGen(ASTContext &Context, PersistentParserState **ParserState)
: Context(Context), ParserState(ParserState) {}

SourceLoc generate(syntax::TokenSyntax Tok, SourceLoc &Loc);

Expr *generate(syntax::IntegerLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::FloatLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::NilLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::BooleanLiteralExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundFileExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundLineExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundColumnExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundFunctionExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::PoundDsohandleExprSyntax &Expr, SourceLoc &Loc);
Expr *generate(syntax::UnknownExprSyntax &Expr, SourceLoc &Loc);

TypeRepr *generate(syntax::TypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::SomeTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::CompositionTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::SimpleTypeIdentifierSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::MemberTypeIdentifierSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::DictionaryTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::ArrayTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::TupleTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::AttributedTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::FunctionTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::MetatypeTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::OptionalTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::ImplicitlyUnwrappedOptionalTypeSyntax Type, SourceLoc &Loc);
TypeRepr *generate(syntax::UnknownTypeSyntax Type, SourceLoc &Loc);

TypeRepr *generate(syntax::GenericArgumentSyntax Arg, SourceLoc &Loc);
llvm::SmallVector<TypeRepr *, 4>
generate(syntax::GenericArgumentListSyntax Args, SourceLoc &Loc);

/// Copy a numeric literal value into AST-owned memory, stripping underscores
/// so the semantic part of the value can be parsed by APInt/APFloat parsers.
static StringRef copyAndStripUnderscores(StringRef Orig, ASTContext &Context);

private:
Expr *generateMagicIdentifierLiteralExpression(syntax::TokenSyntax PoundToken,
SourceLoc &Loc);

TupleTypeRepr *generateTuple(syntax::TokenSyntax LParen,
syntax::TupleTypeElementListSyntax Elements,
syntax::TokenSyntax RParen, SourceLoc &Loc,
bool IsFunction = false);

void gatherTypeIdentifierComponents(
syntax::TypeSyntax Component, SourceLoc &Loc,
llvm::SmallVectorImpl<ComponentIdentTypeRepr *> &Components);

template <typename T>
TypeRepr *generateSimpleOrMemberIdentifier(T Type, SourceLoc &Loc);

template <typename T>
ComponentIdentTypeRepr *generateIdentifier(T Type, SourceLoc &Loc);

StringRef copyAndStripUnderscores(StringRef Orig);

SourceLoc topLoc();
static SourceLoc advanceLocBegin(const SourceLoc &Loc,
const syntax::Syntax &Node);
static SourceLoc advanceLocEnd(const SourceLoc &Loc,
const syntax::TokenSyntax &Token);
static SourceLoc advanceLocAfter(const SourceLoc &Loc,
const syntax::Syntax &Node);

static MagicIdentifierLiteralExpr::Kind getMagicIdentifierLiteralKind(tok Kind);

ValueDecl *lookupInScope(DeclName Name);

TypeRepr *cacheType(syntax::TypeSyntax Type, TypeRepr *TypeAST);

TypeRepr *lookupType(syntax::TypeSyntax Type);

public:
TypeRepr *addType(TypeRepr *Type, const SourceLoc &Loc);

MagicIdentifierLiteralExpr *
generateMagicIdentifierLiteralExpr(const syntax::TokenSyntax &PoundToken);
bool hasType(const SourceLoc &Loc) const;

/// Map magic literal tokens such as #file to their MagicIdentifierLiteralExpr
/// kind.
static MagicIdentifierLiteralExpr::Kind
getMagicIdentifierLiteralKind(tok Kind);
TypeRepr *getType(const SourceLoc &Loc) const;
};
} // namespace swift

Expand Down
27 changes: 26 additions & 1 deletion include/swift/Parse/ParsedRawSyntaxNode.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,20 @@ class ParsedRawSyntaxNode {
/// Primary used for a deferred missing token.
bool isMissing() const { return IsMissing; }

CharSourceRange getDeferredRange() const {
switch (DK) {
case DataKind::DeferredLayout:
return getDeferredLayoutRange();
case DataKind::DeferredToken:
return getDeferredTokenRangeWithoutBackticks();
default:
llvm_unreachable("node not deferred");
}
}

// Recorded Data ===========================================================//

CharSourceRange getRange() const {
CharSourceRange getRecordedRange() const {
assert(isRecorded());
return RecordedData.Range;
}
Expand All @@ -149,6 +160,20 @@ class ParsedRawSyntaxNode {

// Deferred Layout Data ====================================================//

CharSourceRange getDeferredLayoutRange() const {
assert(DK == DataKind::DeferredLayout);
assert(!DeferredLayout.Children.empty());
auto getLastNonNullChild = [this]() {
for (auto &&Child : llvm::reverse(getDeferredChildren()))
if (!Child.isNull())
return Child;
llvm_unreachable("layout node without non-null children");
};
auto firstRange = DeferredLayout.Children.front().getDeferredRange();
auto lastRange = getLastNonNullChild().getDeferredRange();
firstRange.widen(lastRange);
return firstRange;
}
ArrayRef<ParsedRawSyntaxNode> getDeferredChildren() const {
assert(DK == DataKind::DeferredLayout);
return DeferredLayout.Children;
Expand Down
Loading