Skip to content

Commit f275db8

Browse files
ahoppenahoppen
committed
Explicitly model all supported contextual keywords in the contextualKeyword RawTokenKind
The main change here is that the `contextualKeyword` `TokenKind` and `RawTokenKind` no longer have a string as an associated value but an enum that enumerates all known contextual keywords. In follow-up commits, this allows us to rename `contextualKeyword` to `keyword` and merge all the other known keywords into the `Keyword` enum that currently enumerates all contextual keywords. That way we can remove the distinction between contextual keywords and keywords we currently have. I’m also thinking that we can use this additional structure to guarantee a little more type safety in the parser, for example by generating `RawTokenKindSubset`s for children that have `text_choices` set, but I still need to look into that.
1 parent 8877afe commit f275db8

File tree

59 files changed

+2918
-1812
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2918
-1812
lines changed

‎CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift.gyb

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public class TokenSpec {
2929
public let isKeyword: Bool
3030
public let requiresLeadingSpace: Bool
3131
public let requiresTrailingSpace: Bool
32+
public let associatedValueClass: String?
3233

3334
public var swiftKind: String {
3435
let name = lowercaseFirstWord(name: self.name)
@@ -49,7 +50,8 @@ public class TokenSpec {
4950
classification: String = "None",
5051
isKeyword: Bool = false,
5152
requiresLeadingSpace: Bool = false,
52-
requiresTrailingSpace: Bool = false
53+
requiresTrailingSpace: Bool = false,
54+
associatedValueClass: String? = nil
5355
) {
5456
self.name = name
5557
self.kind = kind
@@ -64,6 +66,7 @@ public class TokenSpec {
6466
self.isKeyword = isKeyword
6567
self.requiresLeadingSpace = requiresLeadingSpace
6668
self.requiresTrailingSpace = requiresTrailingSpace
69+
self.associatedValueClass = associatedValueClass
6770
}
6871
}
6972

@@ -243,6 +246,9 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
243246
% if token.requires_trailing_space:
244247
% parameters += ['requiresTrailingSpace: true']
245248
% end
249+
% end
250+
% if token.associated_value_class:
251+
% parameters += [f'associatedValueClass: "{token.associated_value_class}"']
246252
% end
247253
${class_name}Spec(${", ".join(parameters)}),
248254
% end

‎CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public class TokenSpec {
2323
public let isKeyword: Bool
2424
public let requiresLeadingSpace: Bool
2525
public let requiresTrailingSpace: Bool
26+
public let associatedValueClass: String?
2627

2728
public var swiftKind: String {
2829
let name = lowercaseFirstWord(name: self.name)
@@ -43,7 +44,8 @@ public class TokenSpec {
4344
classification: String = "None",
4445
isKeyword: Bool = false,
4546
requiresLeadingSpace: Bool = false,
46-
requiresTrailingSpace: Bool = false
47+
requiresTrailingSpace: Bool = false,
48+
associatedValueClass: String? = nil
4749
) {
4850
self.name = name
4951
self.kind = kind
@@ -58,6 +60,7 @@ public class TokenSpec {
5860
self.isKeyword = isKeyword
5961
self.requiresLeadingSpace = requiresLeadingSpace
6062
self.requiresTrailingSpace = requiresTrailingSpace
63+
self.associatedValueClass = associatedValueClass
6164
}
6265
}
6366

@@ -315,7 +318,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
315318
MiscSpec(name: "PostfixOperator", kind: "oper_postfix", nameForDiagnostics: "postfix operator", classification: "OperatorIdentifier"),
316319
MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator", classification: "OperatorIdentifier"),
317320
MiscSpec(name: "DollarIdentifier", kind: "dollarident", nameForDiagnostics: "dollar identifier", classification: "DollarIdentifier"),
318-
MiscSpec(name: "ContextualKeyword", kind: "contextual_keyword", nameForDiagnostics: "keyword", classification: "Keyword"),
321+
MiscSpec(name: "ContextualKeyword", kind: "contextual_keyword", nameForDiagnostics: "keyword", classification: "Keyword", associatedValueClass: "Keyword"),
319322
MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"),
320323
MiscSpec(name: "StringSegment", kind: "string_segment", nameForDiagnostics: "string segment", classification: "StringLiteral"),
321324
MiscSpec(name: "StringInterpolationAnchor", kind: "string_interpolation_anchor", nameForDiagnostics: "string interpolation anchor", text: ")", classification: "StringInterpolationAnchor"),

‎CodeGeneration/Sources/generate-swiftbasicformat/BasicFormatFile.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ let basicFormatFile = SourceFile {
159159
}
160160
}
161161
}
162-
SwitchCase(#"case .contextualKeyword("async"):"#) {
162+
SwitchCase("case .contextualKeyword(.async):") {
163163
ReturnStmt("return true")
164164
}
165165
SwitchCase("default:") {

‎CodeGeneration/Sources/generate-swiftparser/templates/DeclarationAttributeFile.swift

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,49 @@ let declarationAttributeFile = SourceFile {
2525
)
2626

2727
ExtensionDecl("extension Parser") {
28-
EnumDecl("enum DeclarationAttribute: SyntaxText, ContextualKeywords") {
28+
EnumDecl("enum DeclarationAttribute: RawTokenKindSubset") {
2929
for attribute in DECL_ATTR_KINDS {
30-
EnumCaseDecl("case \(raw: attribute.swiftName) = \"\(raw: attribute.name)\"")
30+
EnumCaseDecl("case \(raw: attribute.swiftName)")
31+
}
32+
EnumCaseDecl("case _spi_available")
33+
34+
InitializerDecl("init?(lexeme: Lexer.Lexeme)") {
35+
SwitchStmt(switchKeyword: .switch, expression: Expr("lexeme")) {
36+
for attribute in DECL_ATTR_KINDS {
37+
SwitchCase("case RawTokenKindMatch(.\(raw: attribute.name)):") {
38+
SequenceExpr("self = .\(raw: attribute.swiftName)")
39+
}
40+
}
41+
SwitchCase("case RawTokenKindMatch(.rethrowsKeyword):") {
42+
SequenceExpr("self = .atRethrows")
43+
}
44+
SwitchCase("case RawTokenKindMatch(._spi_available):") {
45+
SequenceExpr("self = ._spi_available")
46+
}
47+
SwitchCase("default:") {
48+
ReturnStmt("return nil")
49+
}
50+
}
51+
}
52+
53+
VariableDecl(
54+
name: IdentifierPattern("rawTokenKind"),
55+
type: TypeAnnotation(
56+
colon: .colon,
57+
type: SimpleTypeIdentifier("RawTokenKind")
58+
)
59+
) {
60+
SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
61+
for attribute in DECL_ATTR_KINDS {
62+
SwitchCase("case .\(raw: attribute.swiftName):") {
63+
ReturnStmt("return .contextualKeyword(.\(raw: attribute.name))")
64+
}
65+
}
66+
SwitchCase("case ._spi_available:") {
67+
ReturnStmt("return .contextualKeyword(._spi_available)")
68+
}
69+
}
3170
}
32-
EnumCaseDecl(#"case _spi_available = "_spi_available""#)
3371
}
3472
}
3573
}

‎CodeGeneration/Sources/generate-swiftparser/templates/DeclarationModifierFile.swift

Lines changed: 7 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,20 +24,17 @@ let declarationModifierFile = SourceFile {
2424
"""
2525
)
2626

27-
EnumDecl("enum DeclarationModifier: SyntaxText, ContextualKeywords, RawTokenKindSubset") {
27+
EnumDecl("enum DeclarationModifier: RawTokenKindSubset") {
2828
for attribute in DECL_MODIFIER_KINDS {
29-
EnumCaseDecl("case \(raw: attribute.swiftName) = \"\(raw: attribute.name)\"")
29+
EnumCaseDecl("case \(raw: attribute.swiftName)")
3030
}
3131
InitializerDecl("init?(lexeme: Lexer.Lexeme)") {
32-
SwitchStmt(switchKeyword: .switch, expression: Expr("lexeme.tokenKind")) {
33-
for attribute in DECL_MODIFIER_KINDS where attribute.swiftName.hasSuffix("Keyword") {
34-
SwitchCase("case .\(raw: attribute.swiftName):") {
32+
SwitchStmt(expression: Expr("lexeme")) {
33+
for attribute in DECL_MODIFIER_KINDS {
34+
SwitchCase("case RawTokenKindMatch(.\(raw: attribute.swiftName)):") {
3535
SequenceExpr("self = .\(raw: attribute.swiftName)")
3636
}
3737
}
38-
SwitchCase("case .identifier:") {
39-
FunctionCallExpr("self.init(rawValue: lexeme.tokenText)")
40-
}
4138
SwitchCase("default:") {
4239
ReturnStmt("return nil")
4340
}
@@ -51,38 +48,17 @@ let declarationModifierFile = SourceFile {
5148
type: SimpleTypeIdentifier("RawTokenKind")
5249
)
5350
) {
54-
SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
51+
SwitchStmt(expression: Expr("self")) {
5552
for attribute in DECL_MODIFIER_KINDS {
5653
SwitchCase("case .\(raw: attribute.swiftName):") {
5754
if attribute.swiftName.hasSuffix("Keyword") {
5855
ReturnStmt("return .\(raw: attribute.swiftName)")
5956
} else {
60-
ReturnStmt("return .identifier")
57+
ReturnStmt("return .contextualKeyword(.\(raw: attribute.swiftName))")
6158
}
6259
}
6360
}
6461
}
6562
}
66-
67-
68-
VariableDecl(
69-
name: IdentifierPattern("contextualKeyword"),
70-
type: TypeAnnotation(
71-
colon: .colon,
72-
type: OptionalType("SyntaxText?")
73-
)
74-
) {
75-
SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
76-
for attribute in DECL_MODIFIER_KINDS where !attribute.swiftName.hasSuffix("Keyword") {
77-
SwitchCase("case .\(raw: attribute.swiftName):") {
78-
ReturnStmt("return \"\(raw: attribute.name)\"")
79-
}
80-
}
81-
82-
SwitchCase("default:") {
83-
ReturnStmt("return nil")
84-
}
85-
}
86-
}
8763
}
8864
}

‎CodeGeneration/Sources/generate-swiftparser/templates/TypeAttributeFile.swift

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,38 @@ let typeAttributeFile = SourceFile {
2525
)
2626

2727
ExtensionDecl("extension Parser") {
28-
EnumDecl("enum TypeAttribute: SyntaxText, ContextualKeywords") {
28+
EnumDecl("enum TypeAttribute: RawTokenKindSubset") {
2929
for attribute in TYPE_ATTR_KINDS {
30-
EnumCaseDecl("case \(raw: attribute.name) = \"\(raw: attribute.name)\"")
30+
EnumCaseDecl("case \(raw: attribute.name)")
31+
}
32+
33+
InitializerDecl("init?(lexeme: Lexer.Lexeme)") {
34+
SwitchStmt(switchKeyword: .switch, expression: Expr("lexeme")) {
35+
for attribute in TYPE_ATTR_KINDS {
36+
SwitchCase("case RawTokenKindMatch(.\(raw: attribute.name)):") {
37+
SequenceExpr("self = .\(raw: attribute.swiftName)")
38+
}
39+
}
40+
SwitchCase("default:") {
41+
ReturnStmt("return nil")
42+
}
43+
}
44+
}
45+
46+
VariableDecl(
47+
name: IdentifierPattern("rawTokenKind"),
48+
type: TypeAnnotation(
49+
colon: .colon,
50+
type: SimpleTypeIdentifier("RawTokenKind")
51+
)
52+
) {
53+
SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
54+
for attribute in TYPE_ATTR_KINDS {
55+
SwitchCase("case .\(raw: attribute.swiftName):") {
56+
ReturnStmt("return .contextualKeyword(.\(raw: attribute.name))")
57+
}
58+
}
59+
}
3160
}
3261
}
3362
}

‎CodeGeneration/Sources/generate-swiftsyntaxbuilder/BuildableNodesFile.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ private func createConvenienceInitializer(node: Node) -> InitializerDecl? {
112112
} else if let token = child.type.token, token.text == nil {
113113
// Allow initializing identifiers and other tokens without default text with a String
114114
shouldCreateInitializer = true
115-
let paramType = child.type.optionalWrapped(type: Type("String"))
115+
let paramType = child.type.optionalWrapped(type: "\(raw: token.associatedValueClass ?? "String")" as TypeSyntax)
116116
let tokenExpr = MemberAccessExpr("Token.\(raw: token.swiftKind.withFirstCharacterLowercased.backticked)")
117117
if child.type.isOptional {
118118
produceExpr = Expr(FunctionCallExpr("\(raw: child.swiftName).map { \(tokenExpr)($0) }"))

‎CodeGeneration/Sources/generate-swiftsyntaxbuilder/TokenFile.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ let tokenFile = SourceFile {
5252
VariableDecl("""
5353
/// The `open` contextual token
5454
static var open: TokenSyntax {
55-
return .contextualKeyword("open").withTrailingTrivia(.space)
55+
return .contextualKeyword(.open).withTrailingTrivia(.space)
5656
}
5757
"""
5858
)

‎Sources/SwiftBasicFormat/generated/BasicFormat.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ open class BasicFormat: SyntaxRewriter {
309309
return true
310310
case .spacedBinaryOperator:
311311
return true
312-
case .contextualKeyword("async"):
312+
case .contextualKeyword(.async):
313313
return true
314314
default:
315315
return false

‎Sources/SwiftOperators/PrecedenceGroup.swift

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,13 @@ public struct PrecedenceRelation {
4343
public enum Kind {
4444
case higherThan
4545
case lowerThan
46+
47+
var keyword: Keyword {
48+
switch self {
49+
case .higherThan: return .higherThan
50+
case .lowerThan: return .lowerThan
51+
}
52+
}
4653
}
4754

4855
/// The relationship to the other group.

‎Sources/SwiftOperators/SyntaxSynthesis.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ extension PrecedenceRelation {
4949
) -> PrecedenceGroupRelationSyntax {
5050
PrecedenceGroupRelationSyntax(
5151
higherThanOrLowerThan: .contextualKeyword(
52-
"\(kind)",
52+
kind.keyword,
5353
leadingTrivia: [.newlines(1), .spaces(indentation)]
5454
),
5555
colon: .colonToken(),

0 commit comments

Comments
 (0)