Skip to content

Commit 43568e9

Browse files
ahoppenahoppen
authored
Merge pull request #1333 from ahoppen/ahoppen/rawtokenkind-trivial-enum-prep
Preparation to switch RawTokenKind to be a trivial enum again 🚥 #1328
2 parents 32188f6 + 4162710 commit 43568e9

23 files changed

+301
-382
lines changed

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,6 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(
2727
DeclSyntax("static var \(raw: token.swiftKind): TokenSpec { return TokenSpec(.\(raw: token.swiftKind)) }")
2828
}
2929

30-
DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(.keyword(keyword)) }")
30+
DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(keyword) }")
3131
}
3232
}

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokenKindFile.swift

Lines changed: 17 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,23 @@ let tokenKindFile = SourceFileSyntax {
9696
}
9797
}
9898

99+
try! VariableDeclSyntax("public var nameForDiagnostics: String") {
100+
try! SwitchExprSyntax("switch self") {
101+
SwitchCaseSyntax("case .eof:") {
102+
StmtSyntax(#"return "end of file""#)
103+
}
104+
105+
for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
106+
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
107+
StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")
108+
}
109+
}
110+
SwitchCaseSyntax("case .keyword(let keyword):") {
111+
StmtSyntax("return String(syntaxText: keyword.defaultText)")
112+
}
113+
}
114+
}
115+
99116
try VariableDeclSyntax(
100117
"""
101118
/// Returns `true` if the token is a Swift keyword.
@@ -287,49 +304,6 @@ let tokenKindFile = SourceFileSyntax {
287304
}
288305
}
289306

290-
try! VariableDeclSyntax("public var nameForDiagnostics: String") {
291-
try! SwitchExprSyntax("switch self.base") {
292-
SwitchCaseSyntax("case .eof:") {
293-
StmtSyntax(#"return "end of file""#)
294-
}
295-
296-
for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
297-
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
298-
StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")
299-
}
300-
}
301-
SwitchCaseSyntax("case .keyword:") {
302-
StmtSyntax("return String(syntaxText: self.keyword.defaultText)")
303-
}
304-
}
305-
}
306-
307-
try! VariableDeclSyntax(
308-
"""
309-
/// Returns `true` if the token is a Swift keyword.
310-
///
311-
/// Keywords are reserved unconditionally for use by Swift and may not
312-
/// appear as identifiers in any position without being escaped. For example,
313-
/// `class`, `func`, or `import`.
314-
public var isLexerClassifiedKeyword: Bool
315-
"""
316-
) {
317-
try! SwitchExprSyntax("switch self.base") {
318-
SwitchCaseSyntax("case .eof:") {
319-
StmtSyntax("return false")
320-
}
321-
322-
for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
323-
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
324-
StmtSyntax("return \(raw: token.isKeyword)")
325-
}
326-
}
327-
SwitchCaseSyntax("case .keyword:") {
328-
StmtSyntax("return self.keyword.isLexerClassified")
329-
}
330-
}
331-
}
332-
333307
try! VariableDeclSyntax(
334308
"""
335309
/// Returns `true` if the token is a Swift punctuator.

‎Sources/SwiftParser/Attributes.swift

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ extension Parser {
3232
/// Compiler-known attributes that take arguments.
3333
enum DeclarationAttributeWithSpecialSyntax: TokenSpecSet {
3434
case _alignment
35+
case _backDeploy
3536
case _cdecl
3637
case _documentation
3738
case _dynamicReplacement
@@ -66,7 +67,7 @@ extension Parser {
6667
init?(lexeme: Lexer.Lexeme) {
6768
switch lexeme {
6869
case TokenSpec(._alignment): self = ._alignment
69-
case TokenSpec(._backDeploy): self = .backDeployed
70+
case TokenSpec(._backDeploy): self = ._backDeploy
7071
case TokenSpec(._cdecl): self = ._cdecl
7172
case TokenSpec(._documentation): self = ._documentation
7273
case TokenSpec(._dynamicReplacement): self = ._dynamicReplacement
@@ -105,6 +106,7 @@ extension Parser {
105106
var spec: TokenSpec {
106107
switch self {
107108
case ._alignment: return .keyword(._alignment)
109+
case ._backDeploy: return .keyword(._backDeploy)
108110
case ._cdecl: return .keyword(._cdecl)
109111
case ._documentation: return .keyword(._documentation)
110112
case ._dynamicReplacement: return .keyword(._dynamicReplacement)
@@ -223,7 +225,7 @@ extension Parser {
223225
return parseAttribute(argumentMode: .required) { parser in
224226
return .availability(parser.parseAvailabilityArgumentSpecList())
225227
}
226-
case .backDeployed:
228+
case .backDeployed, ._backDeploy:
227229
return parseAttribute(argumentMode: .required) { parser in
228230
return .backDeployedArguments(parser.parseBackDeployedArguments())
229231
}
@@ -313,7 +315,7 @@ extension Parser {
313315
}
314316
case .rethrows:
315317
let (unexpectedBeforeAtSign, atSign) = self.expect(.atSign)
316-
let (unexpectedBeforeAttributeName, attributeName) = self.expect(TokenSpec(.keyword(.rethrows), remapping: .identifier))
318+
let (unexpectedBeforeAttributeName, attributeName) = self.expect(TokenSpec(.rethrows, remapping: .identifier))
317319
return .attribute(
318320
RawAttributeSyntax(
319321
unexpectedBeforeAtSign,
@@ -1156,15 +1158,15 @@ extension Parser.Lookahead {
11561158
// Alternatively, we might have a token that illustrates we're not going to
11571159
// get anything following the attribute, which means the parentheses describe
11581160
// what follows the attribute.
1159-
switch lookahead.currentToken.rawTokenKind {
1160-
case .arrow,
1161-
.keyword(.throw),
1162-
.keyword(.throws),
1163-
.keyword(.rethrows),
1164-
.rightParen,
1165-
.rightBrace,
1166-
.rightSquareBracket,
1167-
.rightAngle:
1161+
switch lookahead.currentToken {
1162+
case TokenSpec(.arrow),
1163+
TokenSpec(.throw),
1164+
TokenSpec(.throws),
1165+
TokenSpec(.rethrows),
1166+
TokenSpec(.rightParen),
1167+
TokenSpec(.rightBrace),
1168+
TokenSpec(.rightSquareBracket),
1169+
TokenSpec(.rightAngle):
11681170
return false
11691171
case _ where lookahead.at(.keyword(.async)):
11701172
return false

‎Sources/SwiftParser/Declarations.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1477,7 +1477,7 @@ extension Parser {
14771477
inMemberDeclList: Bool = false
14781478
) -> RawVariableDeclSyntax {
14791479
let (unexpectedBeforeIntroducer, introducer) = self.eat(handle)
1480-
let hasTryBeforeIntroducer = unexpectedBeforeIntroducer?.containsToken(where: { $0.tokenKind == .keyword(.try) }) ?? false
1480+
let hasTryBeforeIntroducer = unexpectedBeforeIntroducer?.containsToken(where: { TokenSpec(.try) ~= $0 }) ?? false
14811481

14821482
var elements = [RawPatternBindingSyntax]()
14831483
do {

‎Sources/SwiftParser/Expressions.swift

Lines changed: 25 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@ extension Parser {
356356
return parseUnresolvedAsExpr(handle: handle)
357357

358358
case (.async, _)?:
359-
if self.peek().rawTokenKind == .arrow || self.peek().rawTokenKind == .keyword(.throws) {
359+
if self.peek().rawTokenKind == .arrow || TokenSpec(.throws) ~= self.peek() {
360360
fallthrough
361361
} else {
362362
return nil
@@ -494,20 +494,20 @@ extension Parser {
494494
// First check to see if we have the start of a regex literal `/.../`.
495495
// tryLexRegexLiteral(/*forUnappliedOperator*/ false)
496496

497-
switch self.currentToken.rawTokenKind {
498-
case .keyword(.repeat):
497+
switch self.currentToken {
498+
case TokenSpec(.repeat):
499499
// 'repeat' is the start of a pack expansion expression.
500500
return RawExprSyntax(parsePackExpansionExpr(flavor, pattern: pattern))
501501

502502
// Try parse an 'if' or 'switch' as an expression. Note we do this here in
503503
// parseUnaryExpression as we don't allow postfix syntax to hang off such
504504
// expressions to avoid ambiguities such as postfix '.member', which can
505505
// currently be parsed as a static dot member for a result builder.
506-
case .keyword(.switch):
506+
case TokenSpec(.switch):
507507
return RawExprSyntax(
508508
parseSwitchExpression(switchHandle: .constant(.keyword(.switch)))
509509
)
510-
case .keyword(.if):
510+
case TokenSpec(.if):
511511
return RawExprSyntax(
512512
parseIfExpression(ifHandle: .constant(.keyword(.if)))
513513
)
@@ -2063,7 +2063,7 @@ extension Parser.Lookahead {
20632063
}
20642064

20652065
// If this is the start of a switch body, this isn't a trailing closure.
2066-
if self.peek().rawTokenKind == .keyword(.case) {
2066+
if TokenSpec(.case) ~= self.peek() {
20672067
return false
20682068
}
20692069

@@ -2104,23 +2104,23 @@ extension Parser.Lookahead {
21042104
return false
21052105
}
21062106

2107-
switch backtrack.currentToken.rawTokenKind {
2108-
case .leftBrace,
2109-
.keyword(.where),
2110-
.comma:
2107+
switch backtrack.currentToken {
2108+
case TokenSpec(.leftBrace),
2109+
TokenSpec(.where),
2110+
TokenSpec(.comma):
21112111
return true
2112-
case .leftSquareBracket,
2113-
.leftParen,
2114-
.period,
2115-
.keyword(.is),
2116-
.keyword(.as),
2117-
.postfixQuestionMark,
2118-
.infixQuestionMark,
2119-
.exclamationMark,
2120-
.colon,
2121-
.equal,
2122-
.postfixOperator,
2123-
.binaryOperator:
2112+
case TokenSpec(.leftSquareBracket),
2113+
TokenSpec(.leftParen),
2114+
TokenSpec(.period),
2115+
TokenSpec(.is),
2116+
TokenSpec(.as),
2117+
TokenSpec(.postfixQuestionMark),
2118+
TokenSpec(.infixQuestionMark),
2119+
TokenSpec(.exclamationMark),
2120+
TokenSpec(.colon),
2121+
TokenSpec(.equal),
2122+
TokenSpec(.postfixOperator),
2123+
TokenSpec(.binaryOperator):
21242124
return !backtrack.currentToken.isAtStartOfLine
21252125
default:
21262126
return false
@@ -2225,7 +2225,7 @@ extension Parser {
22252225
public mutating func parseSwitchCases(allowStandaloneStmtRecovery: Bool) -> RawSwitchCaseListSyntax {
22262226
var elements = [RawSwitchCaseListSyntax.Element]()
22272227
var elementsProgress = LoopProgressCondition()
2228-
while !self.at(any: [.eof, .rightBrace, .poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword])
2228+
while !self.at(.eof, .rightBrace) && !self.at(.poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword)
22292229
&& elementsProgress.evaluate(currentToken)
22302230
{
22312231
if self.withLookahead({ $0.isAtStartOfSwitchCase(allowRecovery: false) }) {
@@ -2294,7 +2294,7 @@ extension Parser {
22942294
mutating func parseSwitchCaseBody() -> RawCodeBlockItemListSyntax {
22952295
var items = [RawCodeBlockItemSyntax]()
22962296
var loopProgress = LoopProgressCondition()
2297-
while !self.at(any: [.rightBrace, .poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword])
2297+
while !self.at(.rightBrace) && !self.at(.poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword)
22982298
&& !self.withLookahead({ $0.isStartOfConditionalSwitchCases() }),
22992299
let newItem = self.parseCodeBlockItem(),
23002300
loopProgress.evaluate(currentToken)
@@ -2559,14 +2559,7 @@ extension Parser.Lookahead {
25592559
return true
25602560
}
25612561

2562-
if self.peek().rawTokenKind != .identifier,
2563-
self.peek().rawTokenKind != .keyword(.Self),
2564-
self.peek().rawTokenKind != .keyword(.self),
2565-
!self.peek().rawTokenKind.isLexerClassifiedKeyword
2566-
{
2567-
return false
2568-
}
2569-
return true
2562+
return self.peek().isLexerClassifiedKeyword || TokenSpec(.identifier) ~= self.peek()
25702563
}
25712564

25722565
fileprivate func isNextTokenCallPattern() -> Bool {

‎Sources/SwiftParser/Lookahead.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ extension Parser.Lookahead {
254254
// If we have a 'didSet' or a 'willSet' label, disambiguate immediately as
255255
// an accessor block.
256256
let nextToken = self.peek()
257-
if TokenSpec(.keyword(.didSet)) ~= nextToken || TokenSpec(.keyword(.willSet)) ~= nextToken {
257+
if TokenSpec(.didSet) ~= nextToken || TokenSpec(.willSet) ~= nextToken {
258258
return true
259259
}
260260

@@ -286,7 +286,7 @@ extension Parser.Lookahead {
286286
// MARK: Skipping Tokens
287287

288288
extension Parser.Lookahead {
289-
mutating func skipUntil(_ t1: RawTokenKind, _ t2: RawTokenKind) {
289+
mutating func skipUntil(_ t1: TokenSpec, _ t2: TokenSpec) {
290290
return skip(initialState: .skipUntil(t1, t2))
291291
}
292292

@@ -346,7 +346,7 @@ extension Parser.Lookahead {
346346
/// Execute code after skipping bracketed tokens detected from `skipSingle`.
347347
case skipSinglePost(start: BracketedTokens)
348348
/// Skip until either `t1` or `t2`.
349-
case skipUntil(_ t1: RawTokenKind, _ t2: RawTokenKind)
349+
case skipUntil(_ t1: TokenSpec, _ t2: TokenSpec)
350350
}
351351

352352
/// A non-recursie function to skip tokens.

‎Sources/SwiftParser/Modifiers.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ extension Parser {
196196

197197
let unexpectedBeforeDetail: RawUnexpectedNodesSyntax?
198198
let detail: RawTokenSyntax
199-
if let setHandle = canRecoverTo(TokenSpec(.keyword(.set), recoveryPrecedence: .weakBracketClose)) {
199+
if let setHandle = canRecoverTo(TokenSpec(.set, recoveryPrecedence: .weakBracketClose)) {
200200
(unexpectedBeforeDetail, detail) = eat(setHandle)
201201
} else {
202202
unexpectedBeforeDetail = nil

‎Sources/SwiftParser/Names.swift

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ extension Parser {
6868
ident = self.expectIdentifierWithoutRecovery()
6969
} else if flags.contains(.operators), let (_, _) = self.at(anyIn: Operator.self) {
7070
ident = self.consumeAnyToken(remapping: .binaryOperator)
71-
} else if flags.contains(.keywords) && self.currentToken.rawTokenKind.isLexerClassifiedKeyword {
71+
} else if flags.contains(.keywords) && self.currentToken.isLexerClassifiedKeyword {
7272
ident = self.consumeAnyToken(remapping: .identifier)
7373
} else {
7474
ident = self.expectIdentifierWithoutRecovery()
@@ -279,7 +279,10 @@ extension Lexer.Lexeme {
279279
}
280280

281281
var isLexerClassifiedKeyword: Bool {
282-
self.rawTokenKind.isLexerClassifiedKeyword
282+
// Only lexer-classified lexemes have `RawTokenKind` of `keyword.
283+
// Contextual keywords will only be made keywords when a `RawTokenSyntax` is
284+
// constructed from them.
285+
return self.rawTokenKind.base == .keyword
283286
}
284287

285288
func starts(with symbol: SyntaxText) -> Bool {

‎Sources/SwiftParser/Parser.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ extension Parser {
427427
self.missingToken(.identifier, text: nil)
428428
)
429429
} else if keywordRecovery,
430-
(self.currentToken.rawTokenKind.isLexerClassifiedKeyword || self.currentToken.rawTokenKind == .wildcard),
430+
(self.currentToken.isLexerClassifiedKeyword || self.currentToken.rawTokenKind == .wildcard),
431431
!self.currentToken.isAtStartOfLine
432432
{
433433
let keyword = self.consumeAnyToken()

‎Sources/SwiftParser/Patterns.swift

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,13 @@ extension Parser {
5555
case varKeyword
5656

5757
init?(lexeme: Lexer.Lexeme) {
58-
switch lexeme.rawTokenKind {
59-
case .leftParen: self = .leftParen
60-
case .wildcard: self = .wildcard
61-
case .identifier: self = .identifier
62-
case .dollarIdentifier: self = .dollarIdentifier
63-
case .keyword(.let): self = .letKeyword
64-
case .keyword(.var): self = .varKeyword
58+
switch lexeme {
59+
case TokenSpec(.leftParen): self = .leftParen
60+
case TokenSpec(.wildcard): self = .wildcard
61+
case TokenSpec(.identifier): self = .identifier
62+
case TokenSpec(.dollarIdentifier): self = .dollarIdentifier
63+
case TokenSpec(.let): self = .letKeyword
64+
case TokenSpec(.var): self = .varKeyword
6565
default: return nil
6666
}
6767
}
@@ -131,7 +131,7 @@ extension Parser {
131131
)
132132
)
133133
case nil:
134-
if self.currentToken.rawTokenKind.isLexerClassifiedKeyword, !self.currentToken.isAtStartOfLine {
134+
if self.currentToken.isLexerClassifiedKeyword, !self.currentToken.isAtStartOfLine {
135135
// Recover if a keyword was used instead of an identifier
136136
let keyword = self.consumeAnyToken()
137137
return RawPatternSyntax(
@@ -296,12 +296,12 @@ extension Parser.Lookahead {
296296
case leftParen
297297

298298
init?(lexeme: Lexer.Lexeme) {
299-
switch lexeme.rawTokenKind {
300-
case .identifier: self = .identifier
301-
case .wildcard: self = .wildcard
302-
case .keyword(.let): self = .letKeyword
303-
case .keyword(.var): self = .varKeyword
304-
case .leftParen: self = .leftParen
299+
switch lexeme {
300+
case TokenSpec(.identifier): self = .identifier
301+
case TokenSpec(.wildcard): self = .wildcard
302+
case TokenSpec(.let): self = .letKeyword
303+
case TokenSpec(.var): self = .varKeyword
304+
case TokenSpec(.leftParen): self = .leftParen
305305
default: return nil
306306
}
307307
}

0 commit comments

Comments
 (0)