Skip to content

Commit 1cd3ef3

Browse files
akyrtziakyrtzi
authored
Merge pull request #14552 from rintaro/4.1-lex-escaped-eof
[4.1][Lexer] Don't setEscapedIdentifier(true) for tok::eof at ArtificialEOF
2 parents d82f5c6 + 339b47d commit 1cd3ef3

File tree

6 files changed

+146
-13
lines changed

6 files changed

+146
-13
lines changed

‎include/swift/Parse/Lexer.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,7 @@ class Lexer {
496496
}
497497

498498
void formToken(tok Kind, const char *TokStart, bool MultilineString = false);
499+
void formEscapedIdentifierToken(const char *TokStart);
499500

500501
/// Advance to the end of the line but leave the current buffer pointer
501502
/// at that newline character.

‎lib/Parse/Lexer.cpp

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include <limits>
3535

3636
using namespace swift;
37+
using namespace swift::syntax;
3738

3839
// clang::isIdentifierHead and clang::isIdentifierBody are deliberately not in
3940
// this list as a reminder that they are using C rules for identifiers.
@@ -269,6 +270,23 @@ void Lexer::formToken(tok Kind, const char *TokStart, bool MultilineString) {
269270
NextToken.setToken(Kind, TokenText, CommentLength, MultilineString);
270271
}
271272

273+
void Lexer::formEscapedIdentifierToken(const char *TokStart) {
274+
assert(CurPtr - TokStart >= 3 && "escaped identifier must be longer than or equal 3 bytes");
275+
assert(TokStart[0] == '`' && "escaped identifier starts with backtick");
276+
assert(CurPtr[-1] == '`' && "escaped identifier ends with backtick");
277+
if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
278+
LeadingTrivia.push_back(TriviaPiece::backtick());
279+
assert(TrailingTrivia.size() == 0 && "TrailingTrivia is empty here");
280+
TrailingTrivia.push_back(TriviaPiece::backtick());
281+
}
282+
formToken(tok::identifier, TokStart);
283+
// If this token is at ArtificialEOF, it's forced to be tok::eof. Don't mark
284+
// this as escaped-identifier in this case.
285+
if (NextToken.is(tok::eof))
286+
return;
287+
NextToken.setEscapedIdentifier(true);
288+
}
289+
272290
Lexer::State Lexer::getStateForBeginningOfTokenLoc(SourceLoc Loc) const {
273291
const char *Ptr = getBufferPtrForSourceLoc(Loc);
274292
// Skip whitespace backwards until we hit a newline. This is needed to
@@ -1795,17 +1813,15 @@ void Lexer::lexEscapedIdentifier() {
17951813
// If we have the terminating "`", it's an escaped identifier.
17961814
if (*CurPtr == '`') {
17971815
++CurPtr;
1798-
formToken(tok::identifier, Quote);
1799-
NextToken.setEscapedIdentifier(true);
1816+
formEscapedIdentifierToken(Quote);
18001817
return;
18011818
}
18021819
}
18031820

18041821
// Special case; allow '`$`'.
18051822
if (Quote[1] == '$' && Quote[2] == '`') {
18061823
CurPtr = Quote + 3;
1807-
formToken(tok::identifier, Quote);
1808-
NextToken.setEscapedIdentifier(true);
1824+
formEscapedIdentifierToken(Quote);
18091825
return;
18101826
}
18111827

‎lib/Parse/Parser.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -286,10 +286,6 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
286286
Trivia LeadingTrivia, TrailingTrivia;
287287
do {
288288
L.lex(Tok, LeadingTrivia, TrailingTrivia);
289-
if (Tok.isEscapedIdentifier()) {
290-
LeadingTrivia.push_back(TriviaPiece::backtick());
291-
TrailingTrivia.push_front(TriviaPiece::backtick());
292-
}
293289
auto ThisToken = RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
294290
SourcePresence::Present, LeadingTrivia,
295291
TrailingTrivia);

‎lib/Syntax/SyntaxParsingContext.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,6 @@ void SyntaxParsingContext::addToken(Token &Tok, Trivia &LeadingTrivia,
7777
if (!Enabled)
7878
return;
7979

80-
if (Tok.isEscapedIdentifier()) {
81-
LeadingTrivia.push_back(TriviaPiece::backtick());
82-
TrailingTrivia.push_front(TriviaPiece::backtick());
83-
}
8480
addRawSyntax(RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
8581
SourcePresence::Present, LeadingTrivia,
8682
TrailingTrivia));

‎test/SourceKit/DocumentStructure/Inputs/main.swift

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,3 +107,14 @@ protocol FooProtocol {
107107

108108
// SR-5717
109109
a.b(c: d?.e?.f, h: i)
110+
111+
// SR-6926
112+
/* 👨‍👩‍👧‍👦👨‍👩‍👧‍👦👨‍👩‍👧‍👦 */
113+
`init`(x: Int, y: Int) {}
114+
class C {
115+
/* 👨‍👩‍👧‍👦👨‍👩‍👧‍👦👨‍👩‍👧‍👦 */
116+
`init`(x: Int, y: Int) {}
117+
}
118+
var // comment
119+
`$` = 1
120+
func /* comment */`foo`(x: Int) {}

‎test/SourceKit/DocumentStructure/structure.swift.response

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
key.offset: 0,
3-
key.length: 1670,
3+
key.length: 1970,
44
key.diagnostic_stage: source.diagnostic.stage.swift.parse,
55
key.substructure: [
66
{
@@ -1095,6 +1095,102 @@
10951095
key.bodylength: 1
10961096
}
10971097
]
1098+
},
1099+
{
1100+
key.kind: source.lang.swift.expr.call,
1101+
key.name: "`init`",
1102+
key.offset: 1764,
1103+
key.length: 25,
1104+
key.nameoffset: 1764,
1105+
key.namelength: 6,
1106+
key.bodyoffset: 1771,
1107+
key.bodylength: 17,
1108+
key.substructure: [
1109+
{
1110+
key.kind: source.lang.swift.expr.argument,
1111+
key.name: "x",
1112+
key.offset: 1771,
1113+
key.length: 6,
1114+
key.nameoffset: 1771,
1115+
key.namelength: 1,
1116+
key.bodyoffset: 1774,
1117+
key.bodylength: 3
1118+
},
1119+
{
1120+
key.kind: source.lang.swift.expr.argument,
1121+
key.name: "y",
1122+
key.offset: 1779,
1123+
key.length: 6,
1124+
key.nameoffset: 1779,
1125+
key.namelength: 1,
1126+
key.bodyoffset: 1782,
1127+
key.bodylength: 3
1128+
},
1129+
{
1130+
key.kind: source.lang.swift.expr.argument,
1131+
key.offset: 1787,
1132+
key.length: 2,
1133+
key.nameoffset: 0,
1134+
key.namelength: 0,
1135+
key.bodyoffset: 1787,
1136+
key.bodylength: 2,
1137+
key.substructure: [
1138+
{
1139+
key.kind: source.lang.swift.stmt.brace,
1140+
key.offset: 1787,
1141+
key.length: 2,
1142+
key.nameoffset: 0,
1143+
key.namelength: 0,
1144+
key.bodyoffset: 1788,
1145+
key.bodylength: 0
1146+
}
1147+
]
1148+
}
1149+
]
1150+
},
1151+
{
1152+
key.kind: source.lang.swift.decl.class,
1153+
key.accessibility: source.lang.swift.accessibility.internal,
1154+
key.name: "C",
1155+
key.offset: 1790,
1156+
key.length: 119,
1157+
key.runtime_name: "_TtC13StructureTest1C",
1158+
key.nameoffset: 1796,
1159+
key.namelength: 1,
1160+
key.bodyoffset: 1799,
1161+
key.bodylength: 109
1162+
},
1163+
{
1164+
key.kind: source.lang.swift.decl.var.global,
1165+
key.accessibility: source.lang.swift.accessibility.internal,
1166+
key.setter_accessibility: source.lang.swift.accessibility.internal,
1167+
key.name: "$",
1168+
key.offset: 1910,
1169+
key.length: 24,
1170+
key.nameoffset: 1927,
1171+
key.namelength: 1
1172+
},
1173+
{
1174+
key.kind: source.lang.swift.decl.function.free,
1175+
key.accessibility: source.lang.swift.accessibility.internal,
1176+
key.name: "foo(x:)",
1177+
key.offset: 1935,
1178+
key.length: 34,
1179+
key.nameoffset: 1953,
1180+
key.namelength: 13,
1181+
key.bodyoffset: 1968,
1182+
key.bodylength: 0,
1183+
key.substructure: [
1184+
{
1185+
key.kind: source.lang.swift.decl.var.parameter,
1186+
key.name: "x",
1187+
key.offset: 1959,
1188+
key.length: 6,
1189+
key.typename: "Int",
1190+
key.nameoffset: 1959,
1191+
key.namelength: 1
1192+
}
1193+
]
10981194
}
10991195
],
11001196
key.diagnostics: [
@@ -1119,6 +1215,23 @@
11191215
key.length: 30
11201216
}
11211217
]
1218+
},
1219+
{
1220+
key.line: 116,
1221+
key.column: 1,
1222+
key.filepath: main.swift,
1223+
key.severity: source.diagnostic.severity.error,
1224+
key.description: "expected declaration",
1225+
key.diagnostic_stage: source.diagnostic.stage.swift.parse,
1226+
key.diagnostics: [
1227+
{
1228+
key.line: 114,
1229+
key.column: 7,
1230+
key.filepath: main.swift,
1231+
key.severity: source.diagnostic.severity.note,
1232+
key.description: "in declaration of 'C'"
1233+
}
1234+
]
11221235
}
11231236
]
11241237
}

0 commit comments

Comments
 (0)