Merge 'yield' to new keyword infrastructure

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/gyb_generated/StmtNodes.swift

@@ -311,9 +311,12 @@ public let STMT_NODES: [Node] = [
        kind: "Stmt",
        children: [
          Child(name: "YieldKeyword",
-               kind: "YieldToken",
+               kind: "KeywordToken",
                tokenChoices: [
-                 "Yield"
+                 "Keyword"
+               ],
+               textChoices: [
+                 "yield"
                ]),
          Child(name: "Yields",
                kind: "Syntax",

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

@@ -267,7 +267,6 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   MiscSpec(name: "Keyword", kind: "keyword", nameForDiagnostics: "keyword", classification: "Keyword", associatedValueClass: "Keyword"),
   MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"),
   MiscSpec(name: "StringSegment", kind: "string_segment", nameForDiagnostics: "string segment", classification: "StringLiteral"),
-  MiscSpec(name: "Yield", kind: "kw_yield", nameForDiagnostics: "yield", text: "yield"),
 ]
 
 public let SYNTAX_TOKEN_MAP = Dictionary(uniqueKeysWithValues: SYNTAX_TOKENS.map { ("\($0.name)Token", $0) })

Sources/IDEUtils/generated/SyntaxClassification.swift

@@ -266,8 +266,6 @@ extension RawTokenKind {
       return .none
     case .stringSegment: 
       return .stringLiteral
-    case .yield: 
-      return .none
     case .eof: 
       return .none
     }

Sources/SwiftParser/RawTokenKindSubset.swift

@@ -111,7 +111,6 @@ enum CanBeStatementStart: RawTokenKindSubset {
   case throwKeyword
   case whileKeyword
   case yield
-  case yieldAsIdentifier
 
   init?(lexeme: Lexer.Lexeme) {
     switch lexeme {
@@ -129,8 +128,7 @@ enum CanBeStatementStart: RawTokenKindSubset {
     case RawTokenKindMatch(.switch): self = .switchKeyword
     case RawTokenKindMatch(.throw): self = .throwKeyword
     case RawTokenKindMatch(.while): self = .whileKeyword
-    case RawTokenKindMatch(RawTokenKind.yield): self = .yield
-    case RawTokenKindMatch(Keyword.yield): self = .yieldAsIdentifier
+    case RawTokenKindMatch(.yield): self = .yield
     default: return nil
     }
   }
@@ -151,15 +149,7 @@ enum CanBeStatementStart: RawTokenKindSubset {
     case .switchKeyword: return .keyword(.switch)
     case .throwKeyword: return .keyword(.throw)
     case .whileKeyword: return .keyword(.while)
-    case .yield: return .yield
-    case .yieldAsIdentifier: return .keyword(.yield)
-    }
-  }
-
-  var precedence: TokenPrecedence? {
-    switch self {
-    case .yieldAsIdentifier: return .stmtKeyword
-    default: return nil
+    case .yield: return .keyword(.yield)
     }
   }
 }

Sources/SwiftParser/Statements.swift

@@ -118,8 +118,7 @@ extension Parser {
       return label(self.parseDoStatement(doHandle: handle), with: optLabel)
     case (.poundAssertKeyword, let handle)?:
       return label(self.parsePoundAssertStatement(poundAssertHandle: handle), with: optLabel)
-    case (.yieldAsIdentifier, let handle)?,
-      (.yield, let handle)?:
+    case (.yield, let handle)?:
       return label(self.parseYieldStatement(yieldHandle: handle), with: optLabel)
     case nil:
       let missingStmt = RawStmtSyntax(RawMissingStmtSyntax(arena: self.arena))
@@ -1257,7 +1256,6 @@ extension Parser.Lookahead {
       .continueKeyword?,
       .fallthroughKeyword?,
       .switchKeyword?,
-      .yield?,
       .poundAssertKeyword?:
       return true
     case .repeatKeyword?:
@@ -1266,7 +1264,7 @@ extension Parser.Lookahead {
       // FIXME: 'repeat' followed by '{' could be a pack expansion
       // with a closure pattern.
       return self.peek().rawTokenKind == .leftBrace
-    case .yieldAsIdentifier?:
+    case .yield?:
       switch self.peek().rawTokenKind {
       case .prefixAmpersand:
         // "yield &" always denotes a yield statement.

Sources/SwiftParser/TokenPrecedence.swift

@@ -159,7 +159,7 @@ public enum TokenPrecedence: Comparable {
       // Secondary parts of control-flow constructs
       .keyword(.case), .keyword(.catch), .keyword(.default), .keyword(.else),
       // Return-like statements
-      .keyword(.break), .keyword(.continue), .keyword(.fallthrough), .keyword(.return), .keyword(.throw), .yield,
+      .keyword(.break), .keyword(.continue), .keyword(.fallthrough), .keyword(.return), .keyword(.throw), .keyword(.yield),
       // #error, #warning and #assert are statement-like
       .poundErrorKeyword, .poundWarningKeyword, .poundAssertKeyword:
       self = .stmtKeyword

Sources/SwiftSyntax/generated/TokenKind.swift

@@ -131,8 +131,6 @@ public enum TokenKind: Hashable {
 
   case stringSegment(String)
 
-  case yield
-  
   /// Initializes a keyword token kind from its string representation. If the
   /// given string is not a keyword, this function returns `nil`.
   public init?(keyword: String) {
@@ -320,8 +318,6 @@ public enum TokenKind: Hashable {
       return text
     case .stringSegment(let text): 
       return text
-    case .yield: 
-      return #"yield"#
     case .eof: 
       return ""
     }
@@ -433,8 +429,6 @@ public enum TokenKind: Hashable {
       return #"#_hasSymbol"#
     case .keyword(let assoc): 
       return assoc.defaultText
-    case .yield: 
-      return #"yield"#
     case .eof: 
       return ""
     default: 
@@ -575,8 +569,6 @@ public enum TokenKind: Hashable {
       return false
     case .stringSegment: 
       return false
-    case .yield: 
-      return false
     case .keyword(let keyword): 
       return keyword.isLexerClassified
     }
@@ -717,8 +709,6 @@ public enum TokenKind: Hashable {
       return false
     case .stringSegment: 
       return false
-    case .yield: 
-      return false
     }
   }
 }
@@ -854,8 +844,6 @@ extension TokenKind: Equatable {
       return lhsText == rhsText
     case (.stringSegment(let lhsText), .stringSegment(let rhsText)): 
       return lhsText == rhsText
-    case (.yield, .yield): 
-      return true
     default: 
       return false
     }
@@ -993,8 +981,6 @@ public enum RawTokenKind: Equatable, Hashable {
 
   case stringSegment
 
-  case yield
-  
   @_spi(RawSyntax) 
   public var defaultText: SyntaxText? {
     switch self {
@@ -1102,8 +1088,6 @@ public enum RawTokenKind: Equatable, Hashable {
       return #"#_hasSymbol"#
     case .keyword(let assoc): 
       return assoc.defaultText
-    case .yield: 
-      return #"yield"#
     default: 
       return nil
     }
@@ -1237,8 +1221,6 @@ public enum RawTokenKind: Equatable, Hashable {
       return #"raw string delimiter"#
     case .stringSegment: 
       return #"string segment"#
-    case .yield: 
-      return #"yield"#
     case .keyword(let keyword): 
       return String(syntaxText: keyword.defaultText)
     }
@@ -1377,8 +1359,6 @@ public enum RawTokenKind: Equatable, Hashable {
       return false
     case .stringSegment: 
       return false
-    case .yield: 
-      return false
     case .keyword(let keyword): 
       return keyword.isLexerClassified
     }
@@ -1519,8 +1499,6 @@ public enum RawTokenKind: Equatable, Hashable {
       return false
     case .stringSegment: 
       return false
-    case .yield: 
-      return false
     }
   }
 
@@ -1825,9 +1803,6 @@ extension TokenKind {
       return .rawStringDelimiter(text)
     case .stringSegment: 
       return .stringSegment(text)
-    case .yield: 
-      assert(text.isEmpty || rawKind.defaultText.map(String.init ) == text)
-      return .yield
     }
   }
 
@@ -1964,8 +1939,6 @@ extension TokenKind {
       return (.rawStringDelimiter, str)
     case .stringSegment(let str): 
       return (.stringSegment, str)
-    case .yield: 
-      return (.yield, nil)
     }
   }
 }

Sources/SwiftSyntax/generated/Tokens.swift

@@ -847,19 +847,6 @@ extension TokenSyntax {
     )
   }
 
-  public static func yieldToken(
-    leadingTrivia: Trivia = [], 
-    trailingTrivia: Trivia = [], 
-    presence: SourcePresence = .present
-  ) -> TokenSyntax {
-    return TokenSyntax(
-      .yield, 
-      leadingTrivia: leadingTrivia, 
-      trailingTrivia: trailingTrivia, 
-      presence: presence
-    )
-  }
-  
   public static func eof(
     leadingTrivia: Trivia = [], 
     presence: SourcePresence = .present

Sources/SwiftSyntax/gyb_generated/SyntaxFactory.swift

@@ -6200,7 +6200,7 @@ public enum SyntaxFactory {
       let data = SyntaxData.forRoot(RawSyntax.makeLayout(kind: .yieldStmt,
         from: [
         nil,
-        RawSyntax.makeMissingToken(kind: TokenKind.yield, arena: arena),
+        RawSyntax.makeMissingToken(kind: TokenKind.keyword(.yield), arena: arena),
         nil,
         RawSyntax.makeEmptyLayout(kind: SyntaxKind.missing, arena: arena),
         nil,
@@ -9077,15 +9077,6 @@ public enum SyntaxFactory {
                      leadingTrivia: leadingTrivia,
                      trailingTrivia: trailingTrivia)
   }
-  @available(*, deprecated, message: "Use TokenSyntax.yieldToken instead")
-  public static func makeYieldToken(
-    leadingTrivia: Trivia = [],
-    trailingTrivia: Trivia = []
-  ) -> TokenSyntax {
-    return makeToken(.yield, presence: .present,
-                     leadingTrivia: leadingTrivia,
-                     trailingTrivia: trailingTrivia)
-  }
 
 /// MARK: Convenience APIs
 

Sources/SwiftSyntax/gyb_generated/syntax_nodes/SyntaxStmtNodes.swift

@@ -3066,7 +3066,7 @@ public struct YieldStmtSyntax: StmtSyntaxProtocol, SyntaxHashable {
   public init(
     leadingTrivia: Trivia? = nil,
     _ unexpectedBeforeYieldKeyword: UnexpectedNodesSyntax? = nil,
-    yieldKeyword: TokenSyntax = .yieldToken(),
+    yieldKeyword: TokenSyntax = .keyword(.yield),
     _ unexpectedBetweenYieldKeywordAndYields: UnexpectedNodesSyntax? = nil,
     yields: Yields,
     _ unexpectedAfterYields: UnexpectedNodesSyntax? = nil,

Sources/SwiftSyntaxBuilder/generated/Token.swift

@@ -266,11 +266,6 @@ public extension TokenSyntax {
     return .poundHasSymbolKeyword()
   }
 
-  /// The `yield` token
-  static var `yield`: TokenSyntax {
-    return .yieldToken()
-  }
-  
   /// The `eof` token
   static var eof: TokenSyntax {
     return .eof()

gyb_syntax_support/StmtNodes.py

@@ -132,7 +132,7 @@
     # yield-stmt -> 'yield' '('? expr-list? ')'?
     Node('YieldStmt', name_for_diagnostics="'yield' statement", kind='Stmt',
          children=[
-             Child('YieldKeyword', kind='YieldToken'),
+             Child('YieldKeyword', kind='KeywordToken', text_choices=['yield']),
              Child('Yields', kind='Syntax',
                    node_choices=[
                        Child('YieldList', kind='YieldList'),

gyb_syntax_support/Token.py

@@ -300,8 +300,6 @@ def macro_name(self):
          name_for_diagnostics='raw string delimiter'),
     Misc('StringSegment', 'string_segment', name_for_diagnostics='string segment',
          classification='StringLiteral'),
-    Misc('Yield', 'kw_yield', name_for_diagnostics='yield',
-         text='yield'),
 ]
 
 SYNTAX_TOKEN_MAP = {token.name + 'Token': token for token in SYNTAX_TOKENS}