Skip to content

Commit 94c7d04

Browse files
rintarorintaro
authored
Merge pull request #27090 from rintaro/syntax-backtick-rdar54810608
[Syntax] Abolish 'backtick' trivia
2 parents ce28cde + 5bcd224 commit 94c7d04

File tree

13 files changed

+48
-67
lines changed

13 files changed

+48
-67
lines changed

‎include/swift/Parse/ParsedRawSyntaxNode.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ class ParsedRawSyntaxNode {
141141
case DataKind::DeferredLayout:
142142
return getDeferredLayoutRange();
143143
case DataKind::DeferredToken:
144-
return getDeferredTokenRangeWithoutBackticks();
144+
return getDeferredTokenRange();
145145
default:
146146
llvm_unreachable("node not deferred");
147147
}
@@ -194,7 +194,7 @@ class ParsedRawSyntaxNode {
194194

195195
return CharSourceRange{begin, len};
196196
}
197-
CharSourceRange getDeferredTokenRangeWithoutBackticks() const {
197+
CharSourceRange getDeferredTokenRange() const {
198198
assert(DK == DataKind::DeferredToken);
199199
return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};
200200
}

‎include/swift/Parse/Token.h

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -248,17 +248,6 @@ class Token {
248248
return CharSourceRange(getLoc(), getLength());
249249
}
250250

251-
CharSourceRange getRangeWithoutBackticks() const {
252-
SourceLoc TokLoc = getLoc();
253-
unsigned TokLength = getLength();
254-
if (isEscapedIdentifier()) {
255-
// Adjust to account for the backticks.
256-
TokLoc = TokLoc.getAdvancedLoc(1);
257-
TokLength -= 2;
258-
}
259-
return CharSourceRange(TokLoc, TokLength);
260-
}
261-
262251
bool hasComment() const {
263252
return CommentLength != 0;
264253
}

‎include/swift/Syntax/TokenSyntax.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,15 @@ class TokenSyntax final : public Syntax {
8080
return getRaw()->getTokenText();
8181
}
8282

83+
StringRef getIdentifierText() const {
84+
StringRef text = getText();
85+
if (text.front() == '`') {
86+
assert(text.back() == '`');
87+
return text.slice(1, text.size() - 1);
88+
}
89+
return text;
90+
}
91+
8392
static bool kindof(SyntaxKind Kind) {
8493
return isTokenKind(Kind);
8594
}

‎lib/Parse/ASTGen.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ TupleTypeRepr *ASTGen::generateTuple(TokenSyntax LParen,
164164
ElementAST.NameLoc = generate(*Name, Loc);
165165
ElementAST.Name = Name->getText() == "_"
166166
? Identifier()
167-
: Context.getIdentifier(Name->getText());
167+
: Context.getIdentifier(Name->getIdentifierText());
168168
}
169169
if (auto Colon = Element.getColon())
170170
ElementAST.ColonLoc = generate(*Colon, Loc);
@@ -173,7 +173,7 @@ TupleTypeRepr *ASTGen::generateTuple(TokenSyntax LParen,
173173
ElementAST.SecondName =
174174
SecondName->getText() == "_"
175175
? Identifier()
176-
: Context.getIdentifier(SecondName->getText());
176+
: Context.getIdentifier(SecondName->getIdentifierText());
177177
if (IsFunction) {
178178
// Form the named parameter type representation.
179179
ElementAST.UnderscoreLoc = ElementAST.NameLoc;
@@ -351,7 +351,7 @@ TypeRepr *ASTGen::generateSimpleOrMemberIdentifier(T Type, SourceLoc &Loc) {
351351
template <typename T>
352352
ComponentIdentTypeRepr *ASTGen::generateIdentifier(T Type, SourceLoc &Loc) {
353353
auto IdentifierLoc = advanceLocBegin(Loc, Type.getName());
354-
auto Identifier = Context.getIdentifier(Type.getName().getText());
354+
auto Identifier = Context.getIdentifier(Type.getName().getIdentifierText());
355355
if (auto Clause = Type.getGenericArgumentClause()) {
356356
auto Args = Clause->getArguments();
357357
if (!Args.empty()) {

‎lib/Parse/Lexer.cpp

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -289,17 +289,14 @@ void Lexer::formToken(tok Kind, const char *TokStart) {
289289
return isCommentTriviaKind(Piece.getKind());
290290
});
291291
for (auto End = LeadingTrivia.end(); Iter != End; Iter++) {
292-
if (Iter->getKind() == TriviaKind::Backtick)
293-
// Since Token::getCommentRange() doesn't take backtick into account,
294-
// we cannot include length of backtick.
295-
break;
296292
CommentLength += Iter->getLength();
297293
}
298294
}
299295

300296
StringRef TokenText { TokStart, static_cast<size_t>(CurPtr - TokStart) };
301297

302298
if (TriviaRetention == TriviaRetentionMode::WithTrivia && Kind != tok::eof) {
299+
assert(TrailingTrivia.empty() && "TrailingTrivia is empty here");
303300
lexTrivia(TrailingTrivia, /* IsForTrailingTrivia */ true);
304301
}
305302

@@ -311,10 +308,6 @@ void Lexer::formEscapedIdentifierToken(const char *TokStart) {
311308
assert(TokStart[0] == '`' && "escaped identifier starts with backtick");
312309
assert(CurPtr[-1] == '`' && "escaped identifier ends with backtick");
313310

314-
LeadingTrivia.push_back(TriviaKind::Backtick, 1);
315-
assert(TrailingTrivia.empty() && "TrailingTrivia is empty here");
316-
TrailingTrivia.push_back(TriviaKind::Backtick, 1);
317-
318311
formToken(tok::identifier, TokStart);
319312
// If this token is at ArtificialEOF, it's forced to be tok::eof. Don't mark
320313
// this as escaped-identifier in this case.

‎lib/Parse/ParseType.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1193,7 +1193,7 @@ Parser::TypeResult Parser::parseTypeTupleBody() {
11931193
auto GetNameText = [this](Optional<ParsedTokenSyntax> Name) {
11941194
return !Name ? StringRef()
11951195
: SourceMgr.extractText(
1196-
Name->getRaw().getDeferredTokenRangeWithoutBackticks(),
1196+
Name->getRaw().getDeferredTokenRange(),
11971197
L->getBufferID());
11981198
};
11991199

@@ -1208,7 +1208,7 @@ Parser::TypeResult Parser::parseTypeTupleBody() {
12081208
auto Diag = diagnose(NameLoc, diag::tuple_type_multiple_labels);
12091209
auto Name = Element.getDeferredName();
12101210
auto NameText = SourceMgr.extractText(
1211-
Name->getRaw().getDeferredTokenRangeWithoutBackticks(),
1211+
Name->getRaw().getDeferredTokenRange(),
12121212
L->getBufferID());
12131213
if (NameText == "_") {
12141214
Diag.fixItRemoveChars(NameLoc, TypeLoc);

‎lib/Parse/ParsedRawSyntaxNode.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ ParsedRawSyntaxNode::makeDeferred(Token tok,
3535
const ParsedTrivia &leadingTrivia,
3636
const ParsedTrivia &trailingTrivia,
3737
SyntaxParsingContext &ctx) {
38-
CharSourceRange tokRange = tok.getRangeWithoutBackticks();
38+
CharSourceRange tokRange = tok.getRange();
3939
size_t piecesCount = leadingTrivia.size() + trailingTrivia.size();
4040
ParsedTriviaPiece *piecesPtr = nullptr;
4141
if (piecesCount > 0) {

‎lib/Parse/ParsedRawSyntaxRecorder.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ ParsedRawSyntaxNode
3030
ParsedRawSyntaxRecorder::recordToken(const Token &tok,
3131
const ParsedTrivia &leadingTrivia,
3232
const ParsedTrivia &trailingTrivia) {
33-
return recordToken(tok.getKind(), tok.getRangeWithoutBackticks(),
33+
return recordToken(tok.getKind(), tok.getRange(),
3434
leadingTrivia.Pieces, trailingTrivia.Pieces);
3535
}
3636

@@ -63,7 +63,7 @@ getRecordedNode(const ParsedRawSyntaxNode &node, ParsedRawSyntaxRecorder &rec) {
6363
if (node.isDeferredLayout())
6464
return rec.recordRawSyntax(node.getKind(), node.getDeferredChildren());
6565
assert(node.isDeferredToken());
66-
CharSourceRange tokRange = node.getDeferredTokenRangeWithoutBackticks();
66+
CharSourceRange tokRange = node.getDeferredTokenRange();
6767
tok tokKind = node.getTokenKind();
6868
if (node.isMissing())
6969
return rec.recordMissingToken(tokKind, tokRange.getStart());

‎lib/Parse/Parser.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
319319
/*SplitTokens=*/ArrayRef<Token>(),
320320
[&](const Token &Tok, const ParsedTrivia &LeadingTrivia,
321321
const ParsedTrivia &TrailingTrivia) {
322-
CharSourceRange TokRange = Tok.getRangeWithoutBackticks();
322+
CharSourceRange TokRange = Tok.getRange();
323323
SourceLoc LeadingTriviaLoc =
324324
TokRange.getStart().getAdvancedLoc(-LeadingTrivia.getLength());
325325
SourceLoc TrailingTriviaLoc =
@@ -328,7 +328,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
328328
LeadingTrivia.convertToSyntaxTrivia(LeadingTriviaLoc, SM, BufferID);
329329
Trivia syntaxTrailingTrivia =
330330
TrailingTrivia.convertToSyntaxTrivia(TrailingTriviaLoc, SM, BufferID);
331-
auto Text = OwnedString::makeRefCounted(Tok.getText());
331+
auto Text = OwnedString::makeRefCounted(Tok.getRawText());
332332
auto ThisToken =
333333
RawSyntax::make(Tok.getKind(), Text, syntaxLeadingTrivia.Pieces,
334334
syntaxTrailingTrivia.Pieces,

‎test/Syntax/tokens_escaped_identifier.swift

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,16 @@
22
let /*leading trivia*/ `if` = 3
33
print(/*leading trivia*/ `if` )
44

5-
// CHECK-LABEL: 2:25
5+
// CHECK-LABEL: 2:24
66
// CHECK-NEXT:(Token identifier
77
// CHECK-NEXT: (trivia blockComment /*leading trivia*/)
88
// CHECK-NEXT: (trivia space 1)
9-
// CHECK-NEXT: (trivia backtick 1)
10-
// CHECK-NEXT: (text="if")
11-
// CHECK-NEXT: (trivia backtick 1)
9+
// CHECK-NEXT: (text="`if`")
1210
// CHECK-NEXT: (trivia space 1))
1311

14-
// CHECK-LABEL: 3:27
12+
// CHECK-LABEL: 3:26
1513
// CHECK-NEXT:(Token identifier
1614
// CHECK-NEXT: (trivia blockComment /*leading trivia*/)
1715
// CHECK-NEXT: (trivia space 1)
18-
// CHECK-NEXT: (trivia backtick 1)
19-
// CHECK-NEXT: (text="if")
20-
// CHECK-NEXT: (trivia backtick 1)
16+
// CHECK-NEXT: (text="`if`")
2117
// CHECK-NEXT: (trivia space 1))

‎unittests/Syntax/TriviaTests.cpp

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,6 @@ TEST(TriviaTests, EmptyEquivalence) {
7373
ASSERT_EQ(Trivia() + Trivia(), Trivia());
7474
}
7575

76-
TEST(TriviaTests, Backtick) {
77-
llvm::SmallString<1> Scratch;
78-
llvm::raw_svector_ostream OS(Scratch);
79-
Trivia::backtick().print(OS);
80-
ASSERT_EQ(OS.str().str(), "`");
81-
}
82-
8376
TEST(TriviaTests, PrintingSpaces) {
8477
llvm::SmallString<4> Scratch;
8578
llvm::raw_svector_ostream OS(Scratch);
@@ -155,10 +148,9 @@ TEST(TriviaTests, PrintingCombinations) {
155148
llvm::raw_svector_ostream OS(Scratch);
156149
auto CCCCombo = Trivia::spaces(1) +
157150
Trivia::tabs(1) +
158-
Trivia::newlines(1) +
159-
Trivia::backtick();
151+
Trivia::newlines(1);
160152
CCCCombo.print(OS);
161-
ASSERT_EQ(OS.str().str(), " \t\n`");
153+
ASSERT_EQ(OS.str().str(), " \t\n");
162154
}
163155

164156
{
@@ -167,7 +159,6 @@ TEST(TriviaTests, PrintingCombinations) {
167159
}
168160

169161
TEST(TriviaTests, Contains) {
170-
ASSERT_FALSE(Trivia().contains(TriviaKind::Backtick));
171162
ASSERT_FALSE(Trivia().contains(TriviaKind::BlockComment));
172163
ASSERT_FALSE(Trivia().contains(TriviaKind::DocBlockComment));
173164
ASSERT_FALSE(Trivia().contains(TriviaKind::DocLineComment));
@@ -177,7 +168,6 @@ TEST(TriviaTests, Contains) {
177168
ASSERT_FALSE(Trivia().contains(TriviaKind::Newline));
178169
ASSERT_FALSE(Trivia().contains(TriviaKind::Space));
179170

180-
ASSERT_TRUE(Trivia::backtick().contains(TriviaKind::Backtick));
181171
ASSERT_TRUE(Trivia::blockComment("/**/").contains(TriviaKind::BlockComment));
182172
ASSERT_TRUE(Trivia::docBlockComment("/***/")
183173
.contains(TriviaKind::DocBlockComment));
@@ -188,12 +178,10 @@ TEST(TriviaTests, Contains) {
188178
ASSERT_TRUE(Trivia::newlines(1).contains(TriviaKind::Newline));
189179
ASSERT_TRUE(Trivia::spaces(1).contains(TriviaKind::Space));
190180

191-
auto Combo = Trivia::spaces(1) + Trivia::backtick() + Trivia::newlines(3)
192-
+ Trivia::spaces(1);
181+
auto Combo = Trivia::spaces(1) + Trivia::newlines(3) + Trivia::spaces(1);
193182

194183
ASSERT_TRUE(Combo.contains(TriviaKind::Space));
195184
ASSERT_TRUE(Combo.contains(TriviaKind::Newline));
196-
ASSERT_TRUE(Combo.contains(TriviaKind::Backtick));
197185
ASSERT_FALSE(Combo.contains(TriviaKind::Tab));
198186
ASSERT_FALSE(Combo.contains(TriviaKind::LineComment));
199187
ASSERT_FALSE(Combo.contains(TriviaKind::Formfeed));
@@ -219,23 +207,23 @@ TEST(TriviaTests, push_back) {
219207
llvm::SmallString<3> Scratch;
220208
llvm::raw_svector_ostream OS(Scratch);
221209
Trivia Triv;
222-
Triv.push_back(TriviaPiece::backtick());
223-
Triv.push_back(TriviaPiece::backtick());
224-
Triv.push_back(TriviaPiece::backtick());
210+
Triv.push_back(TriviaPiece::newline());
211+
Triv.push_back(TriviaPiece::newline());
212+
Triv.push_back(TriviaPiece::newline());
225213
Triv.print(OS);
226-
ASSERT_EQ(OS.str().str(), "```");
214+
ASSERT_EQ(OS.str().str(), "\n\n\n");
227215
}
228216

229217
TEST(TriviaTests, push_front) {
230218
llvm::SmallString<3> Scratch;
231219
llvm::raw_svector_ostream OS(Scratch);
232220
Trivia Triv;
233-
Triv.push_back(TriviaPiece::backtick());
221+
Triv.push_back(TriviaPiece::newline());
234222
Triv.push_front(TriviaPiece::spaces(1));
235223
Triv.push_back(TriviaPiece::spaces(1));
236-
Triv.push_front(TriviaPiece::backtick());
224+
Triv.push_front(TriviaPiece::newline());
237225
Triv.print(OS);
238-
ASSERT_EQ(OS.str().str(), "` ` ");
226+
ASSERT_EQ(OS.str().str(), "\n \n ");
239227
}
240228

241229
TEST(TriviaTests, front) {

‎utils/gyb_syntax_support/Trivia.py

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,17 @@ def __init__(self, name, comment, serialization_code, characters=[],
88
self.name = name
99
self.comment = comment
1010
self.serialization_code = serialization_code
11-
self.characters = characters
11+
self.characters = tuple(characters)
1212
self.lower_name = lowercase_first_word(name)
1313
self.is_new_line = is_new_line
1414
self.is_comment = is_comment
1515

1616
# Swift sometimes doesn't support escaped characters like \f or \v;
1717
# we should allow specifying alternatives explicitly.
18-
self.swift_characters = swift_characters if swift_characters else\
19-
characters
18+
if swift_characters:
19+
self.swift_characters = tuple(swift_characters)
20+
else:
21+
self.swift_characters = characters
2022
assert len(self.swift_characters) == len(self.characters)
2123

2224
def characters_len(self):
@@ -43,9 +45,6 @@ def is_collection(self):
4345
Trivia('CarriageReturnLineFeed',
4446
'A newline consists of contiguous \'\\r\' and \'\\n\' characters.',
4547
characters=['\\r', '\\n'], is_new_line=True, serialization_code=6),
46-
Trivia('Backtick',
47-
'A backtick \'`\' character, used to escape identifiers.',
48-
characters=['`'], serialization_code=7),
4948
Trivia('LineComment', 'A developer line comment, starting with \'//\'',
5049
is_comment=True, serialization_code=8),
5150
Trivia('BlockComment',

‎utils/gyb_syntax_support/__init__.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from PatternNodes import PATTERN_NODES # noqa: I201
1414
from StmtNodes import STMT_NODES # noqa: I201
1515
import Token
16+
from Trivia import TRIVIAS # noqa: I201
1617
from TypeNodes import TYPE_NODES # noqa: I201
1718

1819

@@ -161,10 +162,16 @@ def hash_token_syntax(token):
161162
return hash((token.name, token.serialization_code))
162163

163164

165+
def hash_trivia(trivia):
166+
return hash((trivia.name, trivia.serialization_code, trivia.characters))
167+
168+
164169
def calculate_node_hash():
165170
result = 0
166171
for node in SYNTAX_NODES:
167172
result = hash((result, hash_syntax_node(node)))
168173
for token in SYNTAX_TOKENS:
169174
result = hash((result, hash_token_syntax(token)))
175+
for trivia in TRIVIAS:
176+
result = hash((result, hash_trivia(trivia)))
170177
return result

0 commit comments

Comments
 (0)