Rename regex and string delimiter TokenKinds

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift

@@ -740,7 +740,7 @@ public let EXPR_NODES: [Node] = [
       Child(
         name: "Pounds",
         deprecatedName: "Delimiter",
-        kind: .token(choices: [.token(tokenKind: "RawStringDelimiterToken")]),
+        kind: .token(choices: [.token(tokenKind: "RawStringPoundDelimiterToken")]),
         isOptional: true
       ),
       Child(
@@ -1419,7 +1419,7 @@ public let EXPR_NODES: [Node] = [
     children: [
       Child(
         name: "OpeningPounds",
-        kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]),
+        kind: .token(choices: [.token(tokenKind: "RegexPoundDelimiterToken")]),
         isOptional: true
       ),
       Child(
@@ -1439,7 +1439,7 @@ public let EXPR_NODES: [Node] = [
       ),
       Child(
         name: "ClosingPounds",
-        kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]),
+        kind: .token(choices: [.token(tokenKind: "RegexPoundDelimiterToken")]),
         isOptional: true
       ),
     ]
@@ -1484,7 +1484,7 @@ public let EXPR_NODES: [Node] = [
       Child(
         name: "OpeningPounds",
         deprecatedName: "OpenDelimiter",
-        kind: .token(choices: [.token(tokenKind: "RawStringDelimiterToken")]),
+        kind: .token(choices: [.token(tokenKind: "RawStringPoundDelimiterToken")]),
         isOptional: true
       ),
       Child(
@@ -1504,7 +1504,7 @@ public let EXPR_NODES: [Node] = [
       Child(
         name: "ClosingPounds",
         deprecatedName: "CloseDelimiter",
-        kind: .token(choices: [.token(tokenKind: "RawStringDelimiterToken")]),
+        kind: .token(choices: [.token(tokenKind: "RawStringPoundDelimiterToken")]),
         isOptional: true
       ),
     ]

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -122,7 +122,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   MiscSpec(name: "EndOfFile", nameForDiagnostics: "end of file", text: ""),
   PunctuatorSpec(name: "Equal", text: "="),
   PunctuatorSpec(name: "ExclamationMark", text: "!"),
-  MiscSpec(name: "ExtendedRegexDelimiter", nameForDiagnostics: "extended delimiter"),
+  MiscSpec(name: "RegexPoundDelimiter", nameForDiagnostics: "extended delimiter"),
   LiteralSpec(name: "FloatingLiteral", nameForDiagnostics: "floating literal"),
   MiscSpec(name: "Identifier", nameForDiagnostics: "identifier"),
   PunctuatorSpec(name: "InfixQuestionMark", text: "?"),
@@ -146,7 +146,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PoundConfigSpec(name: "PoundUnavailable", text: "#unavailable"),
   PunctuatorSpec(name: "PrefixAmpersand", text: "&"),
   MiscSpec(name: "PrefixOperator", nameForDiagnostics: "prefix operator"),
-  MiscSpec(name: "RawStringDelimiter", nameForDiagnostics: "raw string delimiter"),
+  MiscSpec(name: "RawStringPoundDelimiter", nameForDiagnostics: "raw string delimiter"),
   MiscSpec(name: "RegexLiteralPattern", nameForDiagnostics: "regex pattern"),
   PunctuatorSpec(name: "RegexSlash", text: "/"),
   PunctuatorSpec(name: "RightAngle", text: ">"),

CodeGeneration/Tests/ValidateSyntaxNodes/ValidateSyntaxNodes.swift

@@ -265,7 +265,7 @@ class ValidateSyntaxNodes: XCTestCase {
         ),
         ValidationFailure(
           node: .regexLiteralExpr,
-          message: "child 'OpeningPounds' has a token as its only token choice and should thus be named 'ExtendedRegexDelimiter'"
+          message: "child 'OpeningPounds' has a token as its only token choice and should thus be named 'RegexPoundDelimiter'"
         ),
         ValidationFailure(
           node: .regexLiteralExpr,
@@ -278,22 +278,22 @@ class ValidateSyntaxNodes: XCTestCase {
         ),
         ValidationFailure(
           node: .regexLiteralExpr,
-          message: "child 'ClosingPounds' has a token as its only token choice and should thus be named 'ExtendedRegexDelimiter'"
+          message: "child 'ClosingPounds' has a token as its only token choice and should thus be named 'RegexPoundDelimiter'"
             // There are the opening and closing ExtendedRegexDelimiter in the node
         ),
         // We should explicitly mention token here because it’s not obvious that the end of a file is represented by a token
         ValidationFailure(node: .sourceFile, message: "child 'EndOfFileToken' has a token as its only token choice and should thus be named 'EndOfFile'"),
         ValidationFailure(
           node: .stringLiteralExpr,
-          message: "child 'OpeningPounds' has a token as its only token choice and should thus be named 'RawStringDelimiter'"
+          message: "child 'OpeningPounds' has a token as its only token choice and should thus be named 'RawStringPoundDelimiter'"
         ),
         ValidationFailure(
           node: .stringLiteralExpr,
-          message: "child 'ClosingPounds' has a token as its only token choice and should thus be named 'RawStringDelimiter'"
+          message: "child 'ClosingPounds' has a token as its only token choice and should thus be named 'RawStringPoundDelimiter'"
         ),
         ValidationFailure(
           node: .exprSegment,
-          message: "child 'Pounds' has a token as its only token choice and should thus be named 'RawStringDelimiter'"
+          message: "child 'Pounds' has a token as its only token choice and should thus be named 'RawStringPoundDelimiter'"
         ),
 
         // MARK: Tokens that contain underscores

Sources/SwiftBasicFormat/BasicFormat.swift

@@ -259,8 +259,8 @@ open class BasicFormat: SyntaxRewriter {
       (.endOfFile, _),
       (.exclamationMark, .leftParen),  // myOptionalClosure!()
       (.exclamationMark, .period),  // myOptionalBar!.foo()
-      (.extendedRegexDelimiter, .leftParen),  // opening extended regex delimiter should never be separate by a space
-      (.extendedRegexDelimiter, .regexSlash),  // opening extended regex delimiter should never be separate by a space
+      (.regexPoundDelimiter, .leftParen),  // opening extended regex delimiter should never be separate by a space
+      (.regexPoundDelimiter, .regexSlash),  // opening extended regex delimiter should never be separate by a space
       (.identifier, .leftAngle),  // MyType<Int>
       (.identifier, .leftParen),  // foo()
       (.identifier, .leftSquare),  // myArray[1]
@@ -277,7 +277,7 @@ open class BasicFormat: SyntaxRewriter {
       (.leftBrace, .rightBrace),  // {}
       (.leftParen, _),
       (.leftSquare, _),
-      (.multilineStringQuote, .rawStringDelimiter),  // closing raw string delimiter should never be separate by a space
+      (.multilineStringQuote, .rawStringPoundDelimiter),  // closing raw string delimiter should never be separate by a space
       (.period, _),
       (.postfixQuestionMark, .leftAngle),  // init?<T>()
       (.postfixQuestionMark, .leftParen),  // init?() or myOptionalClosure?()
@@ -286,19 +286,19 @@ open class BasicFormat: SyntaxRewriter {
       (.poundUnavailable, .leftParen),  // #unavailable(...)
       (.prefixAmpersand, _),
       (.prefixOperator, _),
-      (.rawStringDelimiter, .leftParen),  // opening raw string delimiter should never be separate by a space
-      (.rawStringDelimiter, .multilineStringQuote),  // opening raw string delimiter should never be separate by a space
-      (.rawStringDelimiter, .singleQuote),  // opening raw string delimiter should never be separate by a space
-      (.rawStringDelimiter, .stringQuote),  // opening raw string delimiter should never be separate by a space
+      (.rawStringPoundDelimiter, .leftParen),  // opening raw string delimiter should never be separate by a space
+      (.rawStringPoundDelimiter, .multilineStringQuote),  // opening raw string delimiter should never be separate by a space
+      (.rawStringPoundDelimiter, .singleQuote),  // opening raw string delimiter should never be separate by a space
+      (.rawStringPoundDelimiter, .stringQuote),  // opening raw string delimiter should never be separate by a space
       (.regexLiteralPattern, _),
-      (.regexSlash, .extendedRegexDelimiter),  // closing extended regex delimiter should never be separate by a space
+      (.regexSlash, .regexPoundDelimiter),  // closing extended regex delimiter should never be separate by a space
       (.rightAngle, .leftParen),  // func foo<T>(x: T)
       (.rightBrace, .leftParen),  // { return 1 }()
       (.rightParen, .leftParen),  // returnsClosure()()
       (.rightParen, .period),  // foo().bar
       (.rightSquare, .period),  // myArray[1].someProperty
-      (.singleQuote, .rawStringDelimiter),  // closing raw string delimiter should never be separate by a space
-      (.stringQuote, .rawStringDelimiter),  // closing raw string delimiter should never be separate by a space
+      (.singleQuote, .rawStringPoundDelimiter),  // closing raw string delimiter should never be separate by a space
+      (.stringQuote, .rawStringPoundDelimiter),  // closing raw string delimiter should never be separate by a space
       (.stringSegment, _),
       (_, .comma),
       (_, .ellipsis),

Sources/SwiftIDEUtils/SyntaxClassification.swift

@@ -126,8 +126,6 @@ extension RawTokenKind {
       return .none
     case .exclamationMark:
       return .none
-    case .extendedRegexDelimiter:
-      return .regexLiteral
     case .floatingLiteral:
       return .floatingLiteral
     case .identifier:
@@ -174,10 +172,12 @@ extension RawTokenKind {
       return .none
     case .prefixOperator:
       return .operatorIdentifier
-    case .rawStringDelimiter:
+    case .rawStringPoundDelimiter:
       return .none
     case .regexLiteralPattern:
       return .regexLiteral
+    case .regexPoundDelimiter:
+      return .regexLiteral
     case .regexSlash:
       return .regexLiteral
     case .rightAngle:

Sources/SwiftParser/Expressions.swift

@@ -1384,7 +1384,7 @@ extension Parser {
   /// The broad structure of the regular expression is validated by the lexer.
   mutating func parseRegexLiteral() -> RawRegexLiteralExprSyntax {
     // See if we have an opening set of pounds.
-    let openPounds = self.consume(if: .extendedRegexDelimiter)
+    let openPounds = self.consume(if: .regexPoundDelimiter)
 
     // Parse the opening slash.
     let (unexpectedBeforeSlash, openSlash) = self.expect(.regexSlash)
@@ -1401,7 +1401,7 @@ extension Parser {
     let closeSlash = self.expectWithoutRecoveryOrLeadingTrivia(.regexSlash)
 
     // Finally, parse a closing set of pounds.
-    let (unexpectedBeforeClosePounds, closePounds) = parsePoundDelimiter(.extendedRegexDelimiter, matching: openPounds)
+    let (unexpectedBeforeClosePounds, closePounds) = parsePoundDelimiter(.regexPoundDelimiter, matching: openPounds)
 
     return RawRegexLiteralExprSyntax(
       openingPounds: openPounds,

Sources/SwiftParser/Lexer/Cursor.swift

@@ -907,7 +907,7 @@ extension Lexer.Cursor {
     case UInt8(ascii: "#"):
       // Try lex a raw string literal.
       if let delimiterLength = self.advanceIfOpeningRawStringDelimiter() {
-        return Lexer.Result(.rawStringDelimiter, stateTransition: .push(newState: .afterRawStringDelimiter(delimiterLength: delimiterLength)))
+        return Lexer.Result(.rawStringPoundDelimiter, stateTransition: .push(newState: .afterRawStringDelimiter(delimiterLength: delimiterLength)))
       }
 
       // Try lex a regex literal.
@@ -1021,7 +1021,7 @@ extension Lexer.Cursor {
     switch self.peek() {
     case UInt8(ascii: "#"):
       self.advance(while: { $0 == Unicode.Scalar("#") })
-      return Lexer.Result(.rawStringDelimiter, stateTransition: .pop)
+      return Lexer.Result(.rawStringPoundDelimiter, stateTransition: .pop)
     case nil:
       return Lexer.Result(.endOfFile)
     default:
@@ -1040,7 +1040,7 @@ extension Lexer.Cursor {
       /// would have returned false in `lexInStringLiteral` and w we wouldn't have
       /// transitioned to the `afterBackslashOfStringInterpolation` state.
       self.advance(while: { $0 == Unicode.Scalar("#") })
-      return Lexer.Result(.rawStringDelimiter)
+      return Lexer.Result(.rawStringPoundDelimiter)
     case UInt8(ascii: "("):
       _ = self.advance()
       return Lexer.Result(.leftParen, stateTransition: .replace(newState: .inStringInterpolation(stringLiteralKind: stringLiteralKind, parenCount: 0)))

Sources/SwiftParser/Lexer/RegexLiteralLexer.swift

@@ -359,7 +359,7 @@ extension RegexLiteralLexemes.Element {
   var tokenKind: RawTokenKind {
     switch kind {
     case .openingPounds, .closingPounds:
-      return .extendedRegexDelimiter
+      return .regexPoundDelimiter
     case .openingSlash, .closingSlash:
       return .regexSlash
     case .pattern:
@@ -652,7 +652,7 @@ extension Lexer.Cursor {
       return true
 
     // Bits of string/regex grammar, we can't start lexing a regex literal here.
-    case .extendedRegexDelimiter, .regexSlash, .regexLiteralPattern, .rawStringDelimiter, .stringQuote, .stringSegment, .multilineStringQuote, .singleQuote:
+    case .regexPoundDelimiter, .regexSlash, .regexLiteralPattern, .rawStringPoundDelimiter, .stringQuote, .stringSegment, .multilineStringQuote, .singleQuote:
       return false
 
     // Allow unknown for better recovery.

Sources/SwiftParser/StringLiterals.swift

@@ -470,7 +470,7 @@ extension Parser {
   /// Parse a string literal expression.
   mutating func parseStringLiteral() -> RawStringLiteralExprSyntax {
     /// Parse opening raw string delimiter if exist.
-    let openDelimiter = self.consume(if: .rawStringDelimiter)
+    let openDelimiter = self.consume(if: .rawStringPoundDelimiter)
 
     /// Try to parse @ in order to recover from Objective-C style literals
     let unexpectedAtSign = self.consume(if: .atSign)
@@ -495,7 +495,7 @@ extension Parser {
       if let stringSegment = self.consume(if: .stringSegment, TokenSpec(.identifier, remapping: .stringSegment)) {
         segments.append(.stringSegment(RawStringSegmentSyntax(content: stringSegment, arena: self.arena)))
       } else if let backslash = self.consume(if: .backslash) {
-        let (unexpectedBeforeDelimiter, delimiter) = self.parsePoundDelimiter(.rawStringDelimiter, matching: openDelimiter)
+        let (unexpectedBeforeDelimiter, delimiter) = self.parsePoundDelimiter(.rawStringPoundDelimiter, matching: openDelimiter)
         let leftParen = self.expectWithoutRecoveryOrLeadingTrivia(.leftParen)
         let expressions = RawTupleExprElementListSyntax(elements: self.parseArgumentListElements(pattern: .none), arena: self.arena)
 
@@ -561,7 +561,7 @@ extension Parser {
       closeQuote = self.expectWithoutRecoveryOrLeadingTrivia(TokenSpec(openQuote.tokenKind))
     }
 
-    let (unexpectedBeforeCloseDelimiter, closeDelimiter) = self.parsePoundDelimiter(.rawStringDelimiter, matching: openDelimiter)
+    let (unexpectedBeforeCloseDelimiter, closeDelimiter) = self.parsePoundDelimiter(.rawStringPoundDelimiter, matching: openDelimiter)
 
     if openQuote.tokenKind == .multilineStringQuote, !openQuote.isMissing, !closeQuote.isMissing {
       let postProcessed = postProcessMultilineStringLiteral(

Sources/SwiftParser/TokenPrecedence.swift

@@ -141,8 +141,8 @@ enum TokenPrecedence: Comparable {
       self = .weakBracketed(closingDelimiter: .rightSquare)
     case .leftAngle:
       self = .weakBracketed(closingDelimiter: .rightAngle)
-    case .multilineStringQuote, .rawStringDelimiter, .singleQuote, .stringQuote,
-      .regexSlash, .extendedRegexDelimiter:
+    case .multilineStringQuote, .rawStringPoundDelimiter, .singleQuote, .stringQuote,
+      .regexSlash, .regexPoundDelimiter:
       self = .weakBracketed(closingDelimiter: tokenKind)
     case  // Chaining punctuators
     .infixQuestionMark, .period, .postfixQuestionMark, .exclamationMark,

Sources/SwiftParser/TokenSpec.swift

@@ -172,15 +172,15 @@ public struct TokenSpec {
     switch rawTokenKind {
     case .binaryOperator: return .binaryOperator("+")
     case .dollarIdentifier: return .dollarIdentifier("$0")
-    case .extendedRegexDelimiter: return .extendedRegexDelimiter("#")
     case .floatingLiteral: return .floatingLiteral("1.0")
     case .identifier: return .identifier("myIdent")
     case .integerLiteral: return .integerLiteral("1")
     case .keyword: return .keyword(keyword!)
     case .postfixOperator: return .postfixOperator("++")
     case .prefixOperator: return .prefixOperator("!")
-    case .rawStringDelimiter: return .rawStringDelimiter("#")
+    case .rawStringPoundDelimiter: return .rawStringPoundDelimiter("#")
     case .regexLiteralPattern: return .regexLiteralPattern(".*")
+    case .regexPoundDelimiter: return .regexPoundDelimiter("#")
     case .stringSegment: return .stringSegment("abc")
     default: return TokenKind.fromRaw(kind: rawTokenKind, text: "")
     }

Sources/SwiftParser/TokenSpecSet.swift

@@ -817,12 +817,12 @@ enum PrimaryExpressionStart: TokenSpecSet {
     case TokenSpec(.poundAvailable): self = .poundAvailable
     case TokenSpec(.poundUnavailable): self = .poundUnavailable
     case TokenSpec(.regexSlash): self = .regexSlash
-    case TokenSpec(.extendedRegexDelimiter): self = .extendedRegexDelimiter
+    case TokenSpec(.regexPoundDelimiter): self = .extendedRegexDelimiter
     case TokenSpec(.self): self = .self
     case TokenSpec(.super): self = .super
     case TokenSpec(.true): self = .true
     case TokenSpec(.wildcard): self = .wildcard
-    case TokenSpec(.rawStringDelimiter): self = .rawStringDelimiter
+    case TokenSpec(.rawStringPoundDelimiter): self = .rawStringDelimiter
     case TokenSpec(.stringQuote): self = .stringQuote
     case TokenSpec(.multilineStringQuote): self = .multilineStringQuote
     case TokenSpec(.singleQuote): self = .singleQuote
@@ -850,12 +850,12 @@ enum PrimaryExpressionStart: TokenSpecSet {
     case .poundAvailable: return .poundAvailable
     case .poundUnavailable: return .poundUnavailable
     case .regexSlash: return .regexSlash
-    case .extendedRegexDelimiter: return .extendedRegexDelimiter
+    case .extendedRegexDelimiter: return .regexPoundDelimiter
     case .self: return .keyword(.self)
     case .super: return .keyword(.super)
     case .true: return .keyword(.true)
     case .wildcard: return .wildcard
-    case .rawStringDelimiter: return .rawStringDelimiter
+    case .rawStringDelimiter: return .rawStringPoundDelimiter
     case .stringQuote: return .stringQuote
     case .multilineStringQuote: return .multilineStringQuote
     case .singleQuote: return .singleQuote

Sources/SwiftParser/generated/TokenSpecStaticMembers.swift

@@ -63,8 +63,8 @@ extension TokenSpec {
     return TokenSpec(.exclamationMark)
   }
 
-  static var extendedRegexDelimiter: TokenSpec {
-    return TokenSpec(.extendedRegexDelimiter)
+  static var regexPoundDelimiter: TokenSpec {
+    return TokenSpec(.regexPoundDelimiter)
   }
 
   static var floatingLiteral: TokenSpec {
@@ -155,8 +155,8 @@ extension TokenSpec {
     return TokenSpec(.prefixOperator)
   }
 
-  static var rawStringDelimiter: TokenSpec {
-    return TokenSpec(.rawStringDelimiter)
+  static var rawStringPoundDelimiter: TokenSpec {
+    return TokenSpec(.rawStringPoundDelimiter)
   }
 
   static var regexLiteralPattern: TokenSpec {

Sources/SwiftParserDiagnostics/MissingNodesError.swift

@@ -197,9 +197,9 @@ fileprivate extension TokenKind {
       return .leftParen
     case .rightSquare:
       return .leftSquare
-    case .stringQuote, .multilineStringQuote, .rawStringDelimiter:
+    case .stringQuote, .multilineStringQuote, .rawStringPoundDelimiter:
       return self
-    case .regexSlash, .extendedRegexDelimiter:
+    case .regexSlash, .regexPoundDelimiter:
       return self
     default:
       return nil

Sources/SwiftParserDiagnostics/MissingTokenError.swift

@@ -36,7 +36,7 @@ extension ParseDiagnosticsGenerator {
       handled = handleInvalidMultilineStringQuote(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer)
     case (.period, .period):
       handled = handleInvalidPeriod(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer)
-    case (.rawStringDelimiter, .rawStringDelimiter), (.extendedRegexDelimiter, .extendedRegexDelimiter):
+    case (.rawStringPoundDelimiter, .rawStringPoundDelimiter), (.regexPoundDelimiter, .regexPoundDelimiter):
       handled = handleInvalidPoundDelimiter(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer)
     default:
       handled = false

Sources/SwiftParserDiagnostics/generated/TokenNameForDiagnostics.swift

@@ -41,7 +41,7 @@ extension TokenKind {
       return #"="#
     case .exclamationMark:
       return #"!"#
-    case .extendedRegexDelimiter:
+    case .regexPoundDelimiter:
       return #"extended delimiter"#
     case .floatingLiteral:
       return #"floating literal"#
@@ -87,7 +87,7 @@ extension TokenKind {
       return #"&"#
     case .prefixOperator:
       return #"prefix operator"#
-    case .rawStringDelimiter:
+    case .rawStringPoundDelimiter:
       return #"raw string delimiter"#
     case .regexLiteralPattern:
       return #"regex pattern"#