Produce separate tokens for raw string delimiters and string quotes in the lexer

The eventual goal of this change is that we no longer need to re-lex string literals from the parser to separate them into their components. Instead, the lexer should just produce the lexemes that will later be put into the syntax tree as tokens.

The downside of this is that the lexer now needs to carry state and know whether it is lexing a string literal. On the upside, the string literal parser could be significantly simplified and the diagnostics got better without any further changes.

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AttributeNodes.swift

@@ -83,7 +83,6 @@ public let ATTRIBUTE_NODES: [Node] = [
                        kind: "Token",
                        tokenChoices: [
                          "Identifier",
-                         "StringLiteral",
                          "IntegerLiteral"
                        ]),
                  Child(name: "StringExpr",
@@ -252,10 +251,7 @@ public let ATTRIBUTE_NODES: [Node] = [
                kind: "Syntax",
                nodeChoices: [
                  Child(name: "String",
-                       kind: "StringLiteralToken",
-                       tokenChoices: [
-                         "StringLiteral"
-                       ]),
+                       kind: "StringLiteralExpr"),
                  Child(name: "Declname",
                        kind: "DeclName")
                ])
@@ -578,11 +574,8 @@ public let ATTRIBUTE_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "MangledName",
-               kind: "StringLiteralToken",
-               description: "The mangled name of a declaration.",
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               kind: "StringLiteralExpr",
+               description: "The mangled name of a declaration."),
          Child(name: "Comma",
                kind: "CommaToken",
                tokenChoices: [
@@ -636,11 +629,8 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "CTypeString",
-               kind: "StringLiteralToken",
-               isOptional: true,
-               tokenChoices: [
-                 "StringLiteral"
-               ])
+               kind: "StringLiteralExpr",
+               isOptional: true)
        ]),
 
   Node(name: "ConventionWitnessMethodAttributeArguments",

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AvailabilityNodes.swift

@@ -71,10 +71,7 @@ public let AVAILABILITY_NODES: [Node] = [
                description: "The value of this labeled argument",
                nodeChoices: [
                  Child(name: "String",
-                       kind: "StringLiteralToken",
-                       tokenChoices: [
-                         "StringLiteral"
-                       ]),
+                       kind: "StringLiteralExpr"),
                  Child(name: "Version",
                        kind: "VersionTuple")
                ])

CodeGeneration/Sources/SyntaxSupport/gyb_generated/DeclNodes.swift

@@ -318,10 +318,7 @@ public let DECL_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "FileName",
-               kind: "StringLiteralToken",
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               kind: "StringLiteralExpr"),
          Child(name: "Comma",
                kind: "CommaToken",
                tokenChoices: [

CodeGeneration/Sources/SyntaxSupport/gyb_generated/StmtNodes.swift

@@ -685,12 +685,9 @@ public let STMT_NODES: [Node] = [
                  "Comma"
                ]),
          Child(name: "Message",
-               kind: "StringLiteralToken",
+               kind: "StringLiteralExpr",
                description: "The assertion message.",
-               isOptional: true,
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               isOptional: true),
          Child(name: "RightParen",
                kind: "RightParenToken",
                tokenChoices: [

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

@@ -309,7 +309,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PoundConfigSpec(name: "PoundHasSymbol", kind: "pound__hasSymbol", text: "#_hasSymbol"),
   LiteralSpec(name: "IntegerLiteral", kind: "integer_literal", nameForDiagnostics: "integer literal", classification: "IntegerLiteral"),
   LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"),
-  LiteralSpec(name: "StringLiteral", kind: "string_literal", nameForDiagnostics: "string literal", classification: "StringLiteral"),
+  LiteralSpec(name: "StringLiteralContents", kind: "string_literal", nameForDiagnostics: "string literal", classification: "StringLiteral"),
   LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"),
   MiscSpec(name: "Unknown", kind: "unknown", nameForDiagnostics: "token"),
   MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"),

Sources/IDEUtils/generated/SyntaxClassification.swift

@@ -350,7 +350,7 @@ extension RawTokenKind {
       return .integerLiteral
     case .floatingLiteral: 
       return .floatingLiteral
-    case .stringLiteral: 
+    case .stringLiteralContents: 
       return .stringLiteral
     case .regexLiteral: 
       return .none

Sources/SwiftParser/Attributes.swift

@@ -764,7 +764,7 @@ extension Parser {
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
     let (unexpectedBeforeLabel, label) = self.expectIdentifier(keywordRecovery: true)
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
-    let filename = self.consumeAnyToken()
+    let filename = self.parseStringLiteral()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawAttributeSyntax(
       unexpectedBeforeAtSign,
@@ -900,11 +900,10 @@ extension Parser {
 
 extension Parser {
   mutating func parseOpaqueReturnTypeOfAttributeArguments() -> RawOpaqueReturnTypeOfAttributeArgumentsSyntax {
-    let (unexpectedBeforeString, mangledName) = self.expect(.stringLiteral)
+    let mangledName = self.parseStringLiteral()
     let (unexpectedBeforeComma, comma) = self.expect(.comma)
     let (unexpectedBeforeOrdinal, ordinal) = self.expect(.integerLiteral)
     return RawOpaqueReturnTypeOfAttributeArgumentsSyntax(
-      unexpectedBeforeString,
       mangledName: mangledName,
       unexpectedBeforeComma,
       comma: comma,
@@ -959,20 +958,18 @@ extension Parser {
       let cTypeLabel: RawTokenSyntax?
       let unexpectedBeforeColon: RawUnexpectedNodesSyntax?
       let colon: RawTokenSyntax?
-      let unexpectedBeforeCTypeString: RawUnexpectedNodesSyntax?
-      let cTypeString: RawTokenSyntax?
+      let cTypeString: RawStringLiteralExprSyntax?
       if self.at(.comma) {
         (unexpectedBeforeComma, comma) = self.expect(.comma)
         cTypeLabel = self.consumeAnyToken()
         (unexpectedBeforeColon, colon) = self.expect(.colon)
-        (unexpectedBeforeCTypeString, cTypeString) = self.expect(.stringLiteral)
+        cTypeString = self.parseStringLiteral()
       } else {
         unexpectedBeforeComma = nil
         comma = nil
         cTypeLabel = nil
         unexpectedBeforeColon = nil
         colon = nil
-        unexpectedBeforeCTypeString = nil
         cTypeString = nil
       }
       return .conventionArguments(
@@ -983,7 +980,6 @@ extension Parser {
           cTypeLabel: cTypeLabel,
           unexpectedBeforeColon,
           colon: colon,
-          unexpectedBeforeCTypeString,
           cTypeString: cTypeString,
           arena: self.arena
         )

Sources/SwiftParser/Availability.swift

@@ -137,7 +137,7 @@ extension Parser {
           let argumentLabel = self.eat(handle)
           let (unexpectedBeforeColon, colon) = self.expect(.colon)
           // FIXME: Make sure this is a string literal with no interpolation.
-          let stringValue = self.consumeAnyToken()
+          let stringValue = self.parseStringLiteral()
 
           entry = .availabilityLabeledArgument(
             RawAvailabilityLabeledArgumentSyntax(

Sources/SwiftParser/Declarations.swift

@@ -1241,7 +1241,7 @@ extension Parser {
     /// consumes the entire regex literal, we're done.
     return self.currentToken.tokenText.withBuffer {
       (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
-      var cursor = Lexer.Cursor(input: buffer, previous: 0)
+      var cursor = Lexer.Cursor(input: buffer, previous: 0, state: .normal)
       guard buffer[0] == UInt8(ascii: "/") else { return false }
       switch cursor.lexOperatorIdentifier(cursor, cursor).tokenKind {
       case .unknown:
@@ -2124,19 +2124,7 @@ extension Parser {
     }
 
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
-    let stringLiteral: RawStringLiteralExprSyntax
-    if self.at(.stringLiteral) {
-      stringLiteral = self.parseStringLiteral()
-    } else {
-      stringLiteral = RawStringLiteralExprSyntax(
-        openDelimiter: nil,
-        openQuote: RawTokenSyntax(missing: .stringQuote, arena: self.arena),
-        segments: RawStringLiteralSegmentsSyntax(elements: [], arena: self.arena),
-        closeQuote: RawTokenSyntax(missing: .stringQuote, arena: self.arena),
-        closeDelimiter: nil,
-        arena: self.arena
-      )
-    }
+    let stringLiteral = self.parseStringLiteral()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
 
     switch directive {

Sources/SwiftParser/Directives.swift

@@ -154,7 +154,7 @@ extension Parser {
     if !self.at(.rightParen) {
       let (unexpectedBeforeFile, file) = self.expectIdentifier()
       let (unexpectedBeforeFileColon, fileColon) = self.expect(.colon)
-      let (unexpectedBeforeFileName, fileName) = self.expect(.stringLiteral)
+      let fileName = self.parseStringLiteral()
       let (unexpectedBeforeComma, comma) = self.expect(.comma)
 
       let (unexpectedBeforeLine, line) = self.expectIdentifier()
@@ -166,7 +166,6 @@ extension Parser {
         fileArgLabel: file,
         unexpectedBeforeFileColon,
         fileArgColon: fileColon,
-        unexpectedBeforeFileName,
         fileName: fileName,
         unexpectedBeforeComma,
         comma: comma,

Sources/SwiftParser/Expressions.swift

@@ -1136,7 +1136,7 @@ extension Parser {
           arena: self.arena
         )
       )
-    case (.stringLiteral, _)?:
+    case (.stringLiteralContents, _)?, (.rawStringDelimiter, _)?, (.stringQuote, _)?, (.multilineStringQuote, _)?, (.singleQuote, _)?:
       return RawExprSyntax(self.parseStringLiteral())
     case (.regexLiteral, _)?:
       return RawExprSyntax(self.parseRegexLiteral())
@@ -1428,83 +1428,62 @@ extension Parser {
   ///     escaped-newline → escape-sequence inline-spaces? line-break
   @_spi(RawSyntax)
   public mutating func parseStringLiteral() -> RawStringLiteralExprSyntax {
-    var text = self.currentToken.wholeText[self.currentToken.textRange]
-
     /// Parse opening raw string delimiter if exist.
-    let openDelimiter = self.parseStringLiteralDelimiter(at: .leading, text: text)
-    if let openDelimiter = openDelimiter {
-      text = text.dropFirst(openDelimiter.tokenText.count)
-    }
+    let openDelimiter = self.consume(if: .rawStringDelimiter)
 
     /// Parse open quote.
-    let openQuote =
-      self.parseStringLiteralQuote(
-        at: openDelimiter != nil ? .leadingRaw : .leading,
-        text: text,
-        wantsMultiline: self.currentToken.isMultilineStringLiteral
-      ) ?? RawTokenSyntax(missing: .stringQuote, arena: arena)
-    if !openQuote.isMissing {
-      text = text.dropFirst(openQuote.tokenText.count)
-    }
+    let (unexpectedBeforeOpenQuote, openQuote) = self.expectAny([.stringQuote, .multilineStringQuote, .singleQuote], default: .stringQuote)
 
     /// Parse segments.
-    let (segments, closeStart) = self.parseStringLiteralSegments(
-      text,
-      openQuote,
-      openDelimiter?.tokenText ?? ""
-    )
-    text = text[closeStart...]
-
-    /// Parse close quote.
-    let closeQuote =
-      self.parseStringLiteralQuote(
-        at: openDelimiter != nil ? .trailingRaw : .trailing,
+    let text = self.currentToken.wholeText[...]
+    let segments: RawStringLiteralSegmentsSyntax
+    if self.at(.stringLiteralContents) {
+      (segments, _) = self.parseStringLiteralSegments(
         text: text,
-        wantsMultiline: self.currentToken.isMultilineStringLiteral
-      ) ?? RawTokenSyntax(missing: openQuote.tokenKind, arena: arena)
-    if !closeQuote.isMissing {
-      text = text.dropFirst(closeQuote.tokenText.count)
-    }
-    /// Parse closing raw string delimiter if exist.
-    let closeDelimiter: RawTokenSyntax?
-    if let delimiter = self.parseStringLiteralDelimiter(
-      at: .trailing,
-      text: text
-    ) {
-      closeDelimiter = delimiter
-    } else if let openDelimiter = openDelimiter {
-      closeDelimiter = RawTokenSyntax(
-        missing: .rawStringDelimiter,
-        text: openDelimiter.tokenText,
-        arena: arena
+        allowsMultiline: openQuote.tokenKind == .multilineStringQuote,
+        delimiter: openDelimiter?.tokenText ?? ""
       )
+      /// Discard the raw string literal token and create the structed string
+      /// literal expression.
+      /// FIXME: We should not instantiate `RawTokenSyntax` and discard it here.
+      _ = self.consumeAnyToken()
     } else {
-      closeDelimiter = nil
-    }
-    assert(
-      (openDelimiter == nil) == (closeDelimiter == nil),
-      "existence of open/close delimiter should match"
-    )
-    if let closeDelimiter = closeDelimiter, !closeDelimiter.isMissing {
-      text = text.dropFirst(closeDelimiter.byteLength)
+      segments = RawStringLiteralSegmentsSyntax(elements: [], arena: self.arena)
     }
 
-    assert(
-      text.isEmpty,
-      "string literal parsing should consume all the literal text"
-    )
+    /// Parse close quote.
+    let (unexpectedBeforeCloseQuote, closeQuote) = self.expect(openQuote.tokenKind)
 
-    /// Discard the raw string literal token and create the structed string
-    /// literal expression.
-    /// FIXME: We should not instantiate `RawTokenSyntax` and discard it here.
-    _ = self.consumeAnyToken()
+    /// Parse closing raw string delimiter if exist.
+    var unexpectedCloseDelimiter: RawTokenSyntax? = nil
+    var closeDelimiter = self.consume(if: .rawStringDelimiter)
+
+    switch (openDelimiter, closeDelimiter) {
+    case (nil, nil):
+      break  // good, no raw delimiters on either side
+    case (let open?, nil):
+      closeDelimiter = missingToken(.rawStringDelimiter, text: open.tokenText)
+    case (nil, .some):
+      unexpectedCloseDelimiter = closeDelimiter
+      closeDelimiter = nil
+    case (let open?, let close?):
+      if open.tokenText == close.tokenText {
+        break  // good, same delimiter on both sides
+      } else {
+        unexpectedCloseDelimiter = closeDelimiter
+        closeDelimiter = missingToken(.rawStringDelimiter, text: open.tokenText)
+      }
+    }
 
     /// Construct the literal expression.
     return RawStringLiteralExprSyntax(
       openDelimiter: openDelimiter,
+      unexpectedBeforeOpenQuote,
       openQuote: openQuote,
       segments: segments,
+      unexpectedBeforeCloseQuote,
       closeQuote: closeQuote,
+      RawUnexpectedNodesSyntax([unexpectedCloseDelimiter], arena: self.arena),
       closeDelimiter: closeDelimiter,
       arena: self.arena
     )
@@ -1679,12 +1658,10 @@ extension Parser {
   ///   - closer: opening quote token.
   ///   - delimiter: opening custom string delimiter or empty string.
   mutating func parseStringLiteralSegments(
-    _ text: Slice<SyntaxText>,
-    _ closer: RawTokenSyntax,
-    _ delimiter: SyntaxText
+    text: Slice<SyntaxText>,
+    allowsMultiline: Bool,
+    delimiter: SyntaxText
   ) -> (RawStringLiteralSegmentsSyntax, SyntaxText.Index) {
-    let allowsMultiline = closer.tokenKind == .multilineStringQuote
-
     var segments = [RawStringLiteralSegmentsSyntax.Element]()
     var segment = text
     var stringLiteralSegmentStart = segment.startIndex
@@ -1728,7 +1705,10 @@ extension Parser {
       let contentSize = content.withBuffer { buf in
         Lexer.lexToEndOfInterpolatedExpression(buf, allowsMultiline)
       }
-      let contentEnd = text.index(contentStart, offsetBy: contentSize)
+      var contentEnd = text.index(contentStart, offsetBy: contentSize)
+      if contentEnd == text.endIndex {
+        contentEnd = text.index(before: text.endIndex)
+      }
 
       do {
         // `\`
@@ -1809,13 +1789,7 @@ extension Parser {
       // trim `##`.
       segment = text[stringLiteralSegmentStart..<text.index(segment.endIndex, offsetBy: -delimiter.count)]
 
-      if (SyntaxText(rebasing: segment).hasSuffix(closer.tokenText)) {
-        // trim `"`.
-        segment = text[stringLiteralSegmentStart..<text.index(segment.endIndex, offsetBy: -closer.tokenText.count)]
-      } else {
-        // If `"` is not found, eat the rest.
-        segment = text[stringLiteralSegmentStart...]
-      }
+      segment = text[stringLiteralSegmentStart...]
     }
 
     assert(segments.count % 2 == 0)