Skip to content

Commit 32188f6

Browse files
ahoppenahoppen
authored
Merge pull request #1328 from ahoppen/ahoppen/consume-rawtokenkindmatch
Use `TokenSpec` (nee `RawTokenKindMatch`) in `consume` functions
2 parents eb16cd3 + b439add commit 32188f6

27 files changed

+1007
-845
lines changed

‎CodeGeneration/Sources/generate-swiftsyntax/GenerateSwiftSyntax.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
6666
// SwiftParser
6767
TemplateSpec(sourceFile: declarationModifierFile, module: swiftParserDir, filename: "DeclarationModifier.swift"),
6868
TemplateSpec(sourceFile: parserEntryFile, module: swiftParserDir, filename: "Parser+Entry.swift"),
69+
TemplateSpec(sourceFile: tokenSpecStaticMembersFile, module: swiftParserDir, filename: "TokenSpecStaticMembers.swift"),
6970
TemplateSpec(sourceFile: typeAttributeFile, module: swiftParserDir, filename: "TypeAttribute.swift"),
7071

7172
// SwiftSyntax

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/DeclarationModifierFile.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@ let declarationModifierFile = SourceFileSyntax {
2424
"""
2525
)
2626

27-
try! EnumDeclSyntax("enum DeclarationModifier: RawTokenKindSubset") {
27+
try! EnumDeclSyntax("enum DeclarationModifier: TokenSpecSet") {
2828
for attribute in DECL_MODIFIER_KINDS {
2929
DeclSyntax("case \(raw: attribute.swiftName)")
3030
}
3131

3232
try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme)") {
3333
try SwitchExprSyntax("switch lexeme") {
3434
for attribute in DECL_MODIFIER_KINDS {
35-
SwitchCaseSyntax("case RawTokenKindMatch(.\(raw: attribute.swiftName)):") {
35+
SwitchCaseSyntax("case TokenSpec(.\(raw: attribute.swiftName)):") {
3636
ExprSyntax("self = .\(raw: attribute.swiftName)")
3737
}
3838
}
@@ -42,7 +42,7 @@ let declarationModifierFile = SourceFileSyntax {
4242
}
4343
}
4444

45-
try VariableDeclSyntax("var rawTokenKind: RawTokenKind") {
45+
try VariableDeclSyntax("var spec: TokenSpec") {
4646
try SwitchExprSyntax("switch self") {
4747
for attribute in DECL_MODIFIER_KINDS {
4848
SwitchCaseSyntax("case .\(raw: attribute.swiftName):") {

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import SwiftSyntax
14+
import SwiftSyntaxBuilder
15+
import SyntaxSupport
16+
import Utils
17+
18+
let tokenSpecStaticMembersFile = SourceFileSyntax(
19+
leadingTrivia: .docLineComment(generateCopyrightHeader(for: "generate-swiftparser"))
20+
) {
21+
DeclSyntax("import SwiftSyntax")
22+
23+
try! ExtensionDeclSyntax("extension TokenSpec") {
24+
DeclSyntax("static var eof: TokenSpec { return TokenSpec(.eof) }")
25+
26+
for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
27+
DeclSyntax("static var \(raw: token.swiftKind): TokenSpec { return TokenSpec(.\(raw: token.swiftKind)) }")
28+
}
29+
30+
DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(.keyword(keyword)) }")
31+
}
32+
}

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TypeAttributeFile.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,15 @@ let typeAttributeFile = SourceFileSyntax {
2525
)
2626

2727
try! ExtensionDeclSyntax("extension Parser") {
28-
try EnumDeclSyntax("enum TypeAttribute: RawTokenKindSubset") {
28+
try EnumDeclSyntax("enum TypeAttribute: TokenSpecSet") {
2929
for attribute in TYPE_ATTR_KINDS {
3030
DeclSyntax("case \(raw: attribute.name)")
3131
}
3232

3333
try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme)") {
3434
SwitchExprSyntax(switchKeyword: .keyword(.switch), expression: ExprSyntax("lexeme")) {
3535
for attribute in TYPE_ATTR_KINDS {
36-
SwitchCaseSyntax("case RawTokenKindMatch(.\(raw: attribute.name)):") {
36+
SwitchCaseSyntax("case TokenSpec(.\(raw: attribute.name)):") {
3737
ExprSyntax("self = .\(raw: attribute.swiftName)")
3838
}
3939
}
@@ -43,7 +43,7 @@ let typeAttributeFile = SourceFileSyntax {
4343
}
4444
}
4545

46-
try VariableDeclSyntax("var rawTokenKind: RawTokenKind") {
46+
try VariableDeclSyntax("var spec: TokenSpec") {
4747
SwitchExprSyntax(switchKeyword: .keyword(.switch), expression: ExprSyntax("self")) {
4848
for attribute in TYPE_ATTR_KINDS {
4949
SwitchCaseSyntax("case .\(raw: attribute.swiftName):") {

‎Sources/SwiftParser/Attributes.swift

Lines changed: 63 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ extension Parser {
3030

3131
extension Parser {
3232
/// Compiler-known attributes that take arguments.
33-
enum DeclarationAttributeWithSpecialSyntax: RawTokenKindSubset {
33+
enum DeclarationAttributeWithSpecialSyntax: TokenSpecSet {
3434
case _alignment
3535
case _cdecl
3636
case _documentation
@@ -65,44 +65,44 @@ extension Parser {
6565

6666
init?(lexeme: Lexer.Lexeme) {
6767
switch lexeme {
68-
case RawTokenKindMatch(._alignment): self = ._alignment
69-
case RawTokenKindMatch(._backDeploy): self = .backDeployed
70-
case RawTokenKindMatch(._cdecl): self = ._cdecl
71-
case RawTokenKindMatch(._documentation): self = ._documentation
72-
case RawTokenKindMatch(._dynamicReplacement): self = ._dynamicReplacement
73-
case RawTokenKindMatch(._effects): self = ._effects
74-
case RawTokenKindMatch(._expose): self = ._expose
75-
case RawTokenKindMatch(._implements): self = ._implements
76-
case RawTokenKindMatch(._nonSendable): self = ._nonSendable
77-
case RawTokenKindMatch(._objcImplementation): self = ._objcImplementation
78-
case RawTokenKindMatch(._objcRuntimeName): self = ._objcRuntimeName
79-
case RawTokenKindMatch(._optimize): self = ._optimize
80-
case RawTokenKindMatch(._originallyDefinedIn): self = ._originallyDefinedIn
81-
case RawTokenKindMatch(._private): self = ._private
82-
case RawTokenKindMatch(._projectedValueProperty): self = ._projectedValueProperty
83-
case RawTokenKindMatch(._semantics): self = ._semantics
84-
case RawTokenKindMatch(._silgen_name): self = ._silgen_name
85-
case RawTokenKindMatch(._specialize): self = ._specialize
86-
case RawTokenKindMatch(._spi): self = ._spi
87-
case RawTokenKindMatch(._spi_available): self = ._spi_available
88-
case RawTokenKindMatch(._swift_native_objc_runtime_base): self = ._swift_native_objc_runtime_base
89-
case RawTokenKindMatch(._typeEraser): self = ._typeEraser
90-
case RawTokenKindMatch(._unavailableFromAsync): self = ._unavailableFromAsync
91-
case RawTokenKindMatch(.`rethrows`): self = .rethrows
92-
case RawTokenKindMatch(.available): self = .available
93-
case RawTokenKindMatch(.backDeployed): self = .backDeployed
94-
case RawTokenKindMatch(.derivative): self = .derivative
95-
case RawTokenKindMatch(.differentiable): self = .differentiable
96-
case RawTokenKindMatch(.exclusivity): self = .exclusivity
97-
case RawTokenKindMatch(.inline): self = .inline
98-
case RawTokenKindMatch(.objc): self = .objc
99-
case RawTokenKindMatch(.transpose): self = .transpose
68+
case TokenSpec(._alignment): self = ._alignment
69+
case TokenSpec(._backDeploy): self = .backDeployed
70+
case TokenSpec(._cdecl): self = ._cdecl
71+
case TokenSpec(._documentation): self = ._documentation
72+
case TokenSpec(._dynamicReplacement): self = ._dynamicReplacement
73+
case TokenSpec(._effects): self = ._effects
74+
case TokenSpec(._expose): self = ._expose
75+
case TokenSpec(._implements): self = ._implements
76+
case TokenSpec(._nonSendable): self = ._nonSendable
77+
case TokenSpec(._objcImplementation): self = ._objcImplementation
78+
case TokenSpec(._objcRuntimeName): self = ._objcRuntimeName
79+
case TokenSpec(._optimize): self = ._optimize
80+
case TokenSpec(._originallyDefinedIn): self = ._originallyDefinedIn
81+
case TokenSpec(._private): self = ._private
82+
case TokenSpec(._projectedValueProperty): self = ._projectedValueProperty
83+
case TokenSpec(._semantics): self = ._semantics
84+
case TokenSpec(._silgen_name): self = ._silgen_name
85+
case TokenSpec(._specialize): self = ._specialize
86+
case TokenSpec(._spi): self = ._spi
87+
case TokenSpec(._spi_available): self = ._spi_available
88+
case TokenSpec(._swift_native_objc_runtime_base): self = ._swift_native_objc_runtime_base
89+
case TokenSpec(._typeEraser): self = ._typeEraser
90+
case TokenSpec(._unavailableFromAsync): self = ._unavailableFromAsync
91+
case TokenSpec(.`rethrows`): self = .rethrows
92+
case TokenSpec(.available): self = .available
93+
case TokenSpec(.backDeployed): self = .backDeployed
94+
case TokenSpec(.derivative): self = .derivative
95+
case TokenSpec(.differentiable): self = .differentiable
96+
case TokenSpec(.exclusivity): self = .exclusivity
97+
case TokenSpec(.inline): self = .inline
98+
case TokenSpec(.objc): self = .objc
99+
case TokenSpec(.transpose): self = .transpose
100100
default:
101101
return nil
102102
}
103103
}
104104

105-
var rawTokenKind: RawTokenKind {
105+
var spec: TokenSpec {
106106
switch self {
107107
case ._alignment: return .keyword(._alignment)
108108
case ._cdecl: return .keyword(._cdecl)
@@ -171,7 +171,7 @@ extension Parser {
171171
case .required:
172172
shouldParseArgument = true
173173
case .customAttribute:
174-
shouldParseArgument = self.withLookahead { $0.isCustomAttributeArgument() } && self.at(.leftParen, allowTokenAtStartOfLine: false)
174+
shouldParseArgument = self.withLookahead { $0.isCustomAttributeArgument() } && self.at(TokenSpec(.leftParen, allowAtStartOfLine: false))
175175
case .optional:
176176
shouldParseArgument = self.at(.leftParen)
177177
}
@@ -313,7 +313,7 @@ extension Parser {
313313
}
314314
case .rethrows:
315315
let (unexpectedBeforeAtSign, atSign) = self.expect(.atSign)
316-
let (unexpectedBeforeAttributeName, attributeName) = self.expect(.keyword(.rethrows), remapping: .identifier)
316+
let (unexpectedBeforeAttributeName, attributeName) = self.expect(TokenSpec(.keyword(.rethrows), remapping: .identifier))
317317
return .attribute(
318318
RawAttributeSyntax(
319319
unexpectedBeforeAtSign,
@@ -358,21 +358,21 @@ extension Parser {
358358
)
359359
}
360360

361-
enum DifferentiabilityKind: RawTokenKindSubset {
361+
enum DifferentiabilityKind: TokenSpecSet {
362362
case reverse
363363
case linear
364364
case forward
365365

366366
init?(lexeme: Lexer.Lexeme) {
367367
switch lexeme {
368-
case RawTokenKindMatch(.reverse): self = .reverse
369-
case RawTokenKindMatch(._linear): self = .linear
370-
case RawTokenKindMatch(._forward): self = .forward
368+
case TokenSpec(.reverse): self = .reverse
369+
case TokenSpec(._linear): self = .linear
370+
case TokenSpec(._forward): self = .forward
371371
default: return nil
372372
}
373373
}
374374

375-
var rawTokenKind: RawTokenKind {
375+
var spec: TokenSpec {
376376
switch self {
377377
case .reverse: return .keyword(.reverse)
378378
case .linear: return .keyword(._linear)
@@ -469,21 +469,21 @@ extension Parser {
469469
}
470470

471471
mutating func parseDifferentiabilityParameter() -> RawDifferentiabilityParamSyntax? {
472-
enum ExpectedTokenKind: RawTokenKindSubset {
472+
enum ExpectedTokenKind: TokenSpecSet {
473473
case identifier
474474
case integerLiteral
475475
case selfKeyword
476476

477477
init?(lexeme: Lexer.Lexeme) {
478478
switch lexeme {
479-
case RawTokenKindMatch(.identifier): self = .identifier
480-
case RawTokenKindMatch(.integerLiteral): self = .integerLiteral
481-
case RawTokenKindMatch(.self): self = .selfKeyword
479+
case TokenSpec(.identifier): self = .identifier
480+
case TokenSpec(.integerLiteral): self = .integerLiteral
481+
case TokenSpec(.self): self = .selfKeyword
482482
default: return nil
483483
}
484484
}
485485

486-
var rawTokenKind: RawTokenKind {
486+
var spec: TokenSpec {
487487
switch self {
488488
case .identifier: return .identifier
489489
case .integerLiteral: return .integerLiteral
@@ -649,7 +649,7 @@ extension Parser {
649649
}
650650

651651
extension Parser {
652-
enum SpecializeParameter: RawTokenKindSubset {
652+
enum SpecializeParameter: TokenSpecSet {
653653
case target
654654
case availability
655655
case exported
@@ -660,18 +660,18 @@ extension Parser {
660660

661661
init?(lexeme: Lexer.Lexeme) {
662662
switch lexeme {
663-
case RawTokenKindMatch(.target): self = .target
664-
case RawTokenKindMatch(.availability): self = .availability
665-
case RawTokenKindMatch(.exported): self = .exported
666-
case RawTokenKindMatch(.kind): self = .kind
667-
case RawTokenKindMatch(.spi): self = .spi
668-
case RawTokenKindMatch(.spiModule): self = .spiModule
669-
case RawTokenKindMatch(.available): self = .available
663+
case TokenSpec(.target): self = .target
664+
case TokenSpec(.availability): self = .availability
665+
case TokenSpec(.exported): self = .exported
666+
case TokenSpec(.kind): self = .kind
667+
case TokenSpec(.spi): self = .spi
668+
case TokenSpec(.spiModule): self = .spiModule
669+
case TokenSpec(.available): self = .available
670670
default: return nil
671671
}
672672
}
673673

674-
var rawTokenKind: RawTokenKind {
674+
var spec: TokenSpec {
675675
switch self {
676676
case .target: return .keyword(.target)
677677
case .availability: return .keyword(.availability)
@@ -1082,14 +1082,14 @@ extension Parser {
10821082
let value: RawDocumentationAttributeArgumentSyntax.Value
10831083
switch label.tokenText {
10841084
case "visibility":
1085-
enum AccessLevelModifier: RawTokenKindSubset {
1085+
enum AccessLevelModifier: TokenSpecSet {
10861086
case `private`
10871087
case `fileprivate`
10881088
case `internal`
10891089
case `public`
10901090
case `open`
10911091

1092-
var rawTokenKind: RawTokenKind {
1092+
var spec: TokenSpec {
10931093
switch self {
10941094
case .private: return .keyword(.private)
10951095
case .fileprivate: return .keyword(.fileprivate)
@@ -1101,11 +1101,11 @@ extension Parser {
11011101

11021102
init?(lexeme: Lexer.Lexeme) {
11031103
switch lexeme {
1104-
case RawTokenKindMatch(.private): self = .private
1105-
case RawTokenKindMatch(.fileprivate): self = .fileprivate
1106-
case RawTokenKindMatch(.internal): self = .internal
1107-
case RawTokenKindMatch(.public): self = .public
1108-
case RawTokenKindMatch(.open): self = .open
1104+
case TokenSpec(.private): self = .private
1105+
case TokenSpec(.fileprivate): self = .fileprivate
1106+
case TokenSpec(.internal): self = .internal
1107+
case TokenSpec(.public): self = .public
1108+
case TokenSpec(.open): self = .open
11091109
default: return nil
11101110
}
11111111
}
@@ -1180,7 +1180,7 @@ extension Parser.Lookahead {
11801180
return false
11811181
}
11821182

1183-
if self.at(.leftParen, allowTokenAtStartOfLine: false) && self.withLookahead({ $0.isCustomAttributeArgument() }) {
1183+
if self.at(TokenSpec(.leftParen, allowAtStartOfLine: false)) && self.withLookahead({ $0.isCustomAttributeArgument() }) {
11841184
self.skipSingle()
11851185
}
11861186

‎Sources/SwiftParser/Availability.swift

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ extension Parser {
5454
return RawAvailabilitySpecListSyntax(elements: elements, arena: self.arena)
5555
}
5656

57-
enum AvailabilityArgumentKind: RawTokenKindSubset {
57+
enum AvailabilityArgumentKind: TokenSpecSet {
5858
case message
5959
case renamed
6060
case introduced
@@ -67,20 +67,20 @@ extension Parser {
6767

6868
init?(lexeme: Lexer.Lexeme) {
6969
switch lexeme {
70-
case RawTokenKindMatch(.message): self = .message
71-
case RawTokenKindMatch(.renamed): self = .renamed
72-
case RawTokenKindMatch(.introduced): self = .introduced
73-
case RawTokenKindMatch(.deprecated): self = .deprecated
74-
case RawTokenKindMatch(.obsoleted): self = .obsoleted
75-
case RawTokenKindMatch(.unavailable): self = .unavailable
76-
case RawTokenKindMatch(.noasync): self = .noasync
77-
case RawTokenKindMatch(.binaryOperator) where lexeme.tokenText == "*": self = .star
78-
case RawTokenKindMatch(.identifier): self = .identifier
70+
case TokenSpec(.message): self = .message
71+
case TokenSpec(.renamed): self = .renamed
72+
case TokenSpec(.introduced): self = .introduced
73+
case TokenSpec(.deprecated): self = .deprecated
74+
case TokenSpec(.obsoleted): self = .obsoleted
75+
case TokenSpec(.unavailable): self = .unavailable
76+
case TokenSpec(.noasync): self = .noasync
77+
case TokenSpec(.binaryOperator) where lexeme.tokenText == "*": self = .star
78+
case TokenSpec(.identifier): self = .identifier
7979
default: return nil
8080
}
8181
}
8282

83-
var rawTokenKind: RawTokenKind {
83+
var spec: TokenSpec {
8484
switch self {
8585
case .message: return .keyword(.message)
8686
case .renamed: return .keyword(.renamed)

‎Sources/SwiftParser/CMakeLists.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ add_swift_host_library(SwiftParser
2020
Nominals.swift
2121
Parser.swift
2222
Patterns.swift
23-
RawTokenKindMatch.swift
24-
RawTokenKindSubset.swift
23+
TokenSpec.swift
24+
TokenSpecSet.swift
2525
Recovery.swift
2626
Specifiers.swift
2727
Statements.swift
@@ -35,6 +35,7 @@ add_swift_host_library(SwiftParser
3535

3636
generated/DeclarationModifier.swift
3737
generated/Parser+Entry.swift
38+
generated/TokenSpecStaticMembers.swift
3839
generated/TypeAttribute.swift
3940

4041
Lexer/Cursor.swift

0 commit comments

Comments
 (0)