Skip to content

Commit 1ac8da5

Browse files
rintarorintaro
authored
Merge pull request #24785 from jansvoboda11/syntax-raw-string-delimiters
[libSyntax] Represent raw string delimiters
2 parents 924c519 + b0f6168 commit 1ac8da5

File tree

7 files changed

+88
-29
lines changed

7 files changed

+88
-29
lines changed

‎lib/Parse/ParseExpr.cpp

Lines changed: 62 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1862,10 +1862,22 @@ parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,
18621862
SyntaxParsingContext ExprContext(SyntaxContext,
18631863
SyntaxKind::ExpressionSegment);
18641864

1865+
unsigned DelimiterLen = EntireTok.getCustomDelimiterLen();
1866+
bool HasCustomDelimiter = DelimiterLen > 0;
1867+
18651868
// Backslash is part of an expression segment.
1866-
SourceLoc BackSlashLoc = Segment.Loc.getAdvancedLoc(-1);
1869+
SourceLoc BackSlashLoc = Segment.Loc.getAdvancedLoc(-1 - DelimiterLen);
18671870
Token BackSlash(tok::backslash, CharSourceRange(BackSlashLoc, 1).str());
18681871
ExprContext.addToken(BackSlash, EmptyTrivia, EmptyTrivia);
1872+
1873+
// Custom delimiter may be a part of an expression segment.
1874+
if (HasCustomDelimiter) {
1875+
SourceLoc DelimiterLoc = Segment.Loc.getAdvancedLoc(-DelimiterLen);
1876+
Token Delimiter(tok::raw_string_delimiter,
1877+
CharSourceRange(DelimiterLoc, DelimiterLen).str());
1878+
ExprContext.addToken(Delimiter, EmptyTrivia, EmptyTrivia);
1879+
}
1880+
18691881
// Create a temporary lexer that lexes from the body of the string.
18701882
LexerState BeginState =
18711883
L->getStateForBeginningOfTokenLoc(Segment.Loc);
@@ -1948,51 +1960,70 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
19481960
SourceLoc Loc = Tok.getLoc();
19491961
SourceLoc EndLoc = Loc.getAdvancedLoc(Tok.getLength());
19501962

1951-
StringRef OpenQuoteStr, CloseQuoteStr;
1963+
StringRef OpenDelimiterStr, OpenQuoteStr, CloseQuoteStr, CloseDelimiterStr;
1964+
unsigned DelimiterLength = Tok.getCustomDelimiterLen();
1965+
bool HasCustomDelimiter = DelimiterLength > 0;
1966+
unsigned QuoteLength;
19521967
tok QuoteKind;
1953-
std::tie(OpenQuoteStr, CloseQuoteStr, QuoteKind) = Tok.isMultilineString() ?
1954-
std::make_tuple(Tok.getRawText().take_front(3),
1955-
Tok.getRawText().take_back(3),
1956-
tok::multiline_string_quote) :
1957-
std::make_tuple(Tok.getRawText().take_front(1),
1958-
Tok.getRawText().take_back(1),
1959-
tok::string_quote);
1968+
std::tie(QuoteLength, QuoteKind) =
1969+
Tok.isMultilineString() ? std::make_tuple(3, tok::multiline_string_quote)
1970+
: std::make_tuple(1, tok::string_quote);
1971+
unsigned CloseQuoteBegin = Tok.getLength() - DelimiterLength - QuoteLength;
1972+
1973+
OpenDelimiterStr = Tok.getRawText().take_front(DelimiterLength);
1974+
OpenQuoteStr = Tok.getRawText().substr(DelimiterLength, QuoteLength);
1975+
CloseQuoteStr = Tok.getRawText().substr(CloseQuoteBegin, QuoteLength);
1976+
CloseDelimiterStr = Tok.getRawText().take_back(DelimiterLength);
19601977

19611978
// Make unknown tokens to represent the open and close quote.
19621979
Token OpenQuote(QuoteKind, OpenQuoteStr);
19631980
Token CloseQuote(QuoteKind, CloseQuoteStr);
19641981
ParsedTrivia EmptyTrivia;
19651982
ParsedTrivia EntireTrailingTrivia = TrailingTrivia;
19661983

1967-
// Add the open quote to the context; the quote should have the leading trivia
1968-
// of the entire string token and a void trailing trivia.
1969-
SyntaxContext->addToken(OpenQuote, LeadingTrivia, EmptyTrivia);
1970-
1984+
if (HasCustomDelimiter) {
1985+
Token OpenDelimiter(tok::raw_string_delimiter, OpenDelimiterStr);
1986+
// When a custom delimiter is present, it owns the leading trivia.
1987+
SyntaxContext->addToken(OpenDelimiter, LeadingTrivia, EmptyTrivia);
1988+
1989+
SyntaxContext->addToken(OpenQuote, EmptyTrivia, EmptyTrivia);
1990+
} else {
1991+
// Without custom delimiter the quote owns trailing trivia.
1992+
SyntaxContext->addToken(OpenQuote, LeadingTrivia, EmptyTrivia);
1993+
}
1994+
19711995
// The simple case: just a single literal segment.
19721996
if (Segments.size() == 1 &&
19731997
Segments.front().Kind == Lexer::StringSegment::Literal) {
19741998
{
19751999
consumeExtraToken(Tok);
19762000
consumeTokenWithoutFeedingReceiver();
1977-
2001+
19782002
SyntaxParsingContext SegmentsCtx(SyntaxContext,
19792003
SyntaxKind::StringLiteralSegments);
1980-
2004+
19812005
SyntaxParsingContext StrSegContext(SyntaxContext,
19822006
SyntaxKind::StringSegment);
1983-
2007+
19842008
// Make an unknown token to encapsulate the entire string segment and add
19852009
// such token to the context.
19862010
auto Segment = Segments.front();
19872011
Token content(tok::string_segment,
19882012
CharSourceRange(Segment.Loc, Segment.Length).str());
19892013
SyntaxContext->addToken(content, EmptyTrivia, EmptyTrivia);
19902014
}
1991-
1992-
// Add the close quote the context; the quote should have a void leading trivia
1993-
// and the trailing trivia of the entire string token.
1994-
SyntaxContext->addToken(CloseQuote, EmptyTrivia, EntireTrailingTrivia);
1995-
2015+
2016+
if (HasCustomDelimiter) {
2017+
SyntaxContext->addToken(CloseQuote, EmptyTrivia, EmptyTrivia);
2018+
2019+
Token CloseDelimiter(tok::raw_string_delimiter, CloseDelimiterStr);
2020+
// When a custom delimiter is present it owns the trailing trivia.
2021+
SyntaxContext->addToken(CloseDelimiter, EmptyTrivia, EntireTrailingTrivia);
2022+
} else {
2023+
// Without custom delimiter the quote owns trailing trivia.
2024+
SyntaxContext->addToken(CloseQuote, EmptyTrivia, EntireTrailingTrivia);
2025+
}
2026+
19962027
return makeParserResult(
19972028
createStringLiteralExprFromSegment(Context, L, Segments.front(), Loc));
19982029
}
@@ -2048,9 +2079,16 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
20482079
AppendingExpr = new (Context) TapExpr(nullptr, Body);
20492080
}
20502081

2051-
// Add the close quote the context; the quote should have a void leading trivia
2052-
// and the trailing trivia of the entire string token.
2053-
SyntaxContext->addToken(CloseQuote, EmptyTrivia, EntireTrailingTrivia);
2082+
if (HasCustomDelimiter) {
2083+
SyntaxContext->addToken(CloseQuote, EmptyTrivia, EmptyTrivia);
2084+
2085+
Token CloseDelimiter(tok::raw_string_delimiter, CloseDelimiterStr);
2086+
// When a custom delimiter is present it owns the trailing trivia.
2087+
SyntaxContext->addToken(CloseDelimiter, EmptyTrivia, EntireTrailingTrivia);
2088+
} else {
2089+
// Without custom delimiter the quote owns trailing trivia.
2090+
SyntaxContext->addToken(CloseQuote, EmptyTrivia, EntireTrailingTrivia);
2091+
}
20542092

20552093
if (AppendingExpr->getBody()->getNumElements() == 1) {
20562094
Status.setIsParseError();

‎test/Syntax/Outputs/round_trip_parse_gen.swift.withkinds

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -574,5 +574,11 @@ func foo<FunctionSignature><ParameterClause>() </ParameterClause></FunctionSigna
574574

575575
@<SimpleTypeIdentifier>custom_attr</SimpleTypeIdentifier></CustomAttribute><CustomAttribute>
576576
@<SimpleTypeIdentifier>custom</SimpleTypeIdentifier>(<FunctionCallArgument>A: <IdentifierExpr>a</IdentifierExpr>, </FunctionCallArgument><FunctionCallArgument>B: <IdentifierExpr>b</IdentifierExpr>, </FunctionCallArgument><FunctionCallArgument>C:<IdentifierExpr>c</IdentifierExpr></FunctionCallArgument>)</CustomAttribute>
577-
func foo<FunctionSignature><ParameterClause>() </ParameterClause></FunctionSignature><CodeBlock>{}</CodeBlock></FunctionDecl>
578-
577+
func foo<FunctionSignature><ParameterClause>() </ParameterClause></FunctionSignature><CodeBlock>{}</CodeBlock></FunctionDecl><StringLiteralExpr>
578+
579+
"<StringSegment>abc</StringSegment>"</StringLiteralExpr><StringLiteralExpr>
580+
"<StringSegment>abc </StringSegment><ExpressionSegment>\(<FunctionCallArgument><IdentifierExpr>foo</IdentifierExpr></FunctionCallArgument>)</ExpressionSegment><StringSegment></StringSegment>"</StringLiteralExpr><StringLiteralExpr>
581+
#"<StringSegment>abc</StringSegment>"#</StringLiteralExpr><StringLiteralExpr>
582+
#"<StringSegment>abc </StringSegment><ExpressionSegment>\#(<FunctionCallArgument><IdentifierExpr>foo</IdentifierExpr></FunctionCallArgument>)</ExpressionSegment><StringSegment></StringSegment>"#</StringLiteralExpr><StringLiteralExpr>
583+
##"<StringSegment>abc</StringSegment>"##</StringLiteralExpr><StringLiteralExpr>
584+
##"<StringSegment>abc </StringSegment><ExpressionSegment>\##(<FunctionCallArgument><IdentifierExpr>foo</IdentifierExpr></FunctionCallArgument>)</ExpressionSegment><StringSegment></StringSegment>"##</StringLiteralExpr>

‎test/Syntax/Parser/tree.swift.result

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// RUN: %swift-syntax-parser-test %s -dump-tree > %t.result
33
// RUN: diff -u %s.result %t.result
44

5-
|func| </t6><t105>|test|</t105><NULL/><s110><s108><t88>|(|</t88><s174></s174><t89>|)| </t89></s108><NULL/><NULL/></s110><NULL/><s93><t90>|{|</t90><s163><s92><s48><t102>
6-
|"|</t102><s168><s104><t104>|a|</t104></s104><s105><t100>|\|</t100><t88>|(|</t88><s164><s96><NULL/><NULL/><s28><t105>|b|</t105><NULL/></s28><NULL/></s96></s164><t101>|)|</t101></s105><s104><t104>|c|</t104></s104></s168><t102>|"|</t102></s48><NULL/><NULL/></s92></s163><t91>
5+
|func| </t6><t105>|test|</t105><NULL/><s110><s108><t88>|(|</t88><s174></s174><t89>|)| </t89></s108><NULL/><NULL/></s110><NULL/><s93><t90>|{|</t90><s163><s92><s48><NULL/><t102>
6+
|"|</t102><s168><s104><t104>|a|</t104></s104><s105><t100>|\|</t100><NULL/><t88>|(|</t88><s164><s96><NULL/><NULL/><s28><t105>|b|</t105><NULL/></s28><NULL/></s96></s164><t101>|)|</t101></s105><s104><t104>|c|</t104></s104></s168><t102>|"|</t102><NULL/></s48><NULL/><NULL/></s92></s163><t91>
77
|}|</t91></s93></s13><NULL/><NULL/></s92></s163><t0>
88
||</t0></s118>

‎test/Syntax/round_trip_parse_gen.swift

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,3 +576,9 @@ func foo() {}
576576
@custom(A: a, B: b, C:c)
577577
func foo() {}
578578
579+
"abc"
580+
"abc \(foo)"
581+
#"abc"#
582+
#"abc \#(foo)"#
583+
##"abc"##
584+
##"abc \##(foo)"##

‎test/incrParse/Outputs/incrementalTransfer.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@
199199
"id": 73,
200200
"kind": "StringLiteralExpr",
201201
"layout": [
202+
null,
202203
{
203204
"id": 69,
204205
"tokenKind": {
@@ -240,7 +241,8 @@
240241
"leadingTrivia": [],
241242
"trailingTrivia": [],
242243
"presence": "Present"
243-
}
244+
},
245+
null
244246
],
245247
"presence": "Present"
246248
}

‎utils/gyb_syntax_support/ExprNodes.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,8 @@
462462
traits=['Parenthesized'],
463463
children=[
464464
Child('Backslash', kind='BackslashToken'),
465+
Child('Delimiter', kind='RawStringDelimiterToken',
466+
is_optional=True),
465467
Child('LeftParen', kind='LeftParenToken',
466468
classification='StringInterpolationAnchor',
467469
force_classification=True),
@@ -473,6 +475,8 @@
473475
# e.g. "abc \(foo()) def"
474476
Node('StringLiteralExpr', kind='Expr',
475477
children=[
478+
Child('OpenDelimiter', kind='RawStringDelimiterToken',
479+
is_optional=True),
476480
Child('OpenQuote', kind='Token',
477481
token_choices=[
478482
'StringQuoteToken',
@@ -485,6 +489,8 @@
485489
'StringQuoteToken',
486490
'MultilineStringQuoteToken',
487491
]),
492+
Child('CloseDelimiter', kind='RawStringDelimiterToken',
493+
is_optional=True),
488494
]),
489495

490496
# e.g. "\a.b[2].a"

‎utils/gyb_syntax_support/Token.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,7 @@ def macro_name(self):
324324

325325
Misc('ContextualKeyword', 'contextual_keyword', classification='Keyword',
326326
serialization_code=114),
327+
Misc('RawStringDelimiter', 'raw_string_delimiter', serialization_code=119),
327328
Misc('StringSegment', 'string_segment', classification='StringLiteral',
328329
serialization_code=104),
329330
Misc('StringInterpolationAnchor', 'string_interpolation_anchor',

0 commit comments

Comments
 (0)