Produce separate tokens for raw string delimiters and string quotes in the lexer

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AttributeNodes.swift

@@ -45,6 +45,8 @@ public let ATTRIBUTE_NODES: [Node] = [
                        kind: "TupleExprElementList"),
                  Child(name: "Token",
                        kind: "Token"),
+                 Child(name: "String",
+                       kind: "StringLiteralExpr"),
                  Child(name: "Availability",
                        kind: "AvailabilitySpecList"),
                  Child(name: "SpecializeArguments",
@@ -513,11 +515,8 @@ public let ATTRIBUTE_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "MangledName",
-               kind: "StringLiteralToken",
-               description: "The mangled name of a declaration.",
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               kind: "StringLiteralExpr",
+               description: "The mangled name of a declaration."),
          Child(name: "Comma",
                kind: "CommaToken",
                tokenChoices: [
@@ -571,11 +570,8 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "CTypeString",
-               kind: "StringLiteralToken",
-               isOptional: true,
-               tokenChoices: [
-                 "StringLiteral"
-               ])
+               kind: "StringLiteralExpr",
+               isOptional: true)
        ]),
 
   Node(name: "ConventionWitnessMethodAttributeArguments",
@@ -614,11 +610,8 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Comma"
                ]),
          Child(name: "CxxName",
-               kind: "StringLiteralToken",
-               isOptional: true,
-               tokenChoices: [
-                 "StringLiteral"
-               ])
+               kind: "StringLiteralExpr",
+               isOptional: true)
        ]),
 
   Node(name: "OriginallyDefinedInArguments",
@@ -640,10 +633,7 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "ModuleName",
-               kind: "StringLiteralToken",
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               kind: "StringLiteralExpr"),
          Child(name: "Comma",
                kind: "CommaToken",
                tokenChoices: [
@@ -673,10 +663,7 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "Filename",
-               kind: "StringLiteralToken",
-               tokenChoices: [
-                 "StringLiteral"
-               ])
+               kind: "StringLiteralExpr")
        ]),
 
   Node(name: "DynamicReplacementArguments",
@@ -720,10 +707,7 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "Message",
-               kind: "StringLiteralToken",
-               tokenChoices: [
-                 "StringLiteral"
-               ])
+               kind: "StringLiteralExpr")
        ]),
 
   Node(name: "EffectsArguments",
@@ -754,11 +738,16 @@ public let ATTRIBUTE_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "Value",
-               kind: "Token",
-               tokenChoices: [
-                 "Identifier",
-                 "Keyword",
-                 "StringLiteral"
+               kind: "Syntax",
+               nodeChoices: [
+                 Child(name: "Token",
+                       kind: "Token",
+                       tokenChoices: [
+                         "Identifier",
+                         "Keyword"
+                       ]),
+                 Child(name: "String",
+                       kind: "StringLiteralExpr")
                ]),
          Child(name: "TrailingComma",
                kind: "CommaToken",

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AvailabilityNodes.swift

@@ -69,10 +69,7 @@ public let AVAILABILITY_NODES: [Node] = [
                description: "The value of this labeled argument",
                nodeChoices: [
                  Child(name: "String",
-                       kind: "StringLiteralToken",
-                       tokenChoices: [
-                         "StringLiteral"
-                       ]),
+                       kind: "StringLiteralExpr"),
                  Child(name: "Version",
                        kind: "VersionTuple")
                ])

CodeGeneration/Sources/SyntaxSupport/gyb_generated/DeclNodes.swift

@@ -327,10 +327,7 @@ public let DECL_NODES: [Node] = [
                  "Colon"
                ]),
          Child(name: "FileName",
-               kind: "StringLiteralToken",
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               kind: "StringLiteralExpr"),
          Child(name: "Comma",
                kind: "CommaToken",
                tokenChoices: [

CodeGeneration/Sources/SyntaxSupport/gyb_generated/StmtNodes.swift

@@ -720,12 +720,9 @@ public let STMT_NODES: [Node] = [
                  "Comma"
                ]),
          Child(name: "Message",
-               kind: "StringLiteralToken",
+               kind: "StringLiteralExpr",
                description: "The assertion message.",
-               isOptional: true,
-               tokenChoices: [
-                 "StringLiteral"
-               ]),
+               isOptional: true),
          Child(name: "RightParen",
                kind: "RightParenToken",
                tokenChoices: [

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

@@ -221,7 +221,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PoundConfigSpec(name: "PoundHasSymbol", kind: "pound__hasSymbol", text: "#_hasSymbol"),
   LiteralSpec(name: "IntegerLiteral", kind: "integer_literal", nameForDiagnostics: "integer literal", classification: "IntegerLiteral"),
   LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"),
-  LiteralSpec(name: "StringLiteral", kind: "string_literal", nameForDiagnostics: "string literal", classification: "StringLiteral"),
+  LiteralSpec(name: "StringLiteralContents", kind: "string_literal", nameForDiagnostics: "string literal", classification: "StringLiteral"),
   LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"),
   MiscSpec(name: "Unknown", kind: "unknown", nameForDiagnostics: "token"),
   MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"),

Sources/IDEUtils/generated/SyntaxClassification.swift

@@ -242,7 +242,7 @@ extension RawTokenKind {
       return .integerLiteral
     case .floatingLiteral: 
       return .floatingLiteral
-    case .stringLiteral: 
+    case .stringLiteralContents: 
       return .stringLiteral
     case .regexLiteral: 
       return .none

Sources/SwiftParser/Attributes.swift

@@ -284,19 +284,15 @@ extension Parser {
       }
     case ._cdecl, ._silgen_name:
       return parseAttribute(argumentMode: .required) { parser in
-        return .token(parser.consume(if: .stringLiteral) ?? parser.missingToken(.stringLiteral))
+        return .string(parser.parseStringLiteral())
       }
     case ._implements:
       return parseAttribute(argumentMode: .required) { parser in
         return .implementsArguments(parser.parseImplementsAttributeArguments())
       }
     case ._semantics:
       return parseAttribute(argumentMode: .required) { parser in
-        if let value = parser.consume(if: .stringLiteral) {
-          return .token(value)
-        } else {
-          return .token(parser.missingToken(.stringLiteral))
-        }
+        return .string(parser.parseStringLiteral())
       }
     case ._backDeploy:
       return parseAttribute(argumentMode: .required) { parser in
@@ -854,11 +850,10 @@ extension Parser {
 
 extension Parser {
   mutating func parseOpaqueReturnTypeOfAttributeArguments() -> RawOpaqueReturnTypeOfAttributeArgumentsSyntax {
-    let (unexpectedBeforeString, mangledName) = self.expect(.stringLiteral)
+    let mangledName = self.parseStringLiteral()
     let (unexpectedBeforeComma, comma) = self.expect(.comma)
     let (unexpectedBeforeOrdinal, ordinal) = self.expect(.integerLiteral)
     return RawOpaqueReturnTypeOfAttributeArgumentsSyntax(
-      unexpectedBeforeString,
       mangledName: mangledName,
       unexpectedBeforeComma,
       comma: comma,
@@ -890,20 +885,18 @@ extension Parser {
       let cTypeLabel: RawTokenSyntax?
       let unexpectedBeforeColon: RawUnexpectedNodesSyntax?
       let colon: RawTokenSyntax?
-      let unexpectedBeforeCTypeString: RawUnexpectedNodesSyntax?
-      let cTypeString: RawTokenSyntax?
+      let cTypeString: RawStringLiteralExprSyntax?
       if self.at(.comma) {
         (unexpectedBeforeComma, comma) = self.expect(.comma)
         cTypeLabel = self.consumeAnyToken()
         (unexpectedBeforeColon, colon) = self.expect(.colon)
-        (unexpectedBeforeCTypeString, cTypeString) = self.expect(.stringLiteral)
+        cTypeString = self.parseStringLiteral()
       } else {
         unexpectedBeforeComma = nil
         comma = nil
         cTypeLabel = nil
         unexpectedBeforeColon = nil
         colon = nil
-        unexpectedBeforeCTypeString = nil
         cTypeString = nil
       }
       return .conventionArguments(
@@ -914,7 +907,6 @@ extension Parser {
           cTypeLabel: cTypeLabel,
           unexpectedBeforeColon,
           colon: colon,
-          unexpectedBeforeCTypeString,
           cTypeString: cTypeString,
           arena: self.arena
         )
@@ -961,22 +953,19 @@ extension Parser {
     }
     let unexpectedBeforeComma: RawUnexpectedNodesSyntax?
     let comma: RawTokenSyntax?
-    let unexpectedBeforeCxxName: RawUnexpectedNodesSyntax?
-    let cxxName: RawTokenSyntax?
+    let cxxName: RawStringLiteralExprSyntax?
     if self.at(.comma) {
       (unexpectedBeforeComma, comma) = self.expect(.comma)
-      (unexpectedBeforeCxxName, cxxName) = self.expect(.stringLiteral)
+      cxxName = self.parseStringLiteral()
     } else {
       unexpectedBeforeComma = nil
       comma = nil
-      unexpectedBeforeCxxName = nil
       cxxName = nil
     }
     return RawExposeAttributeArgumentsSyntax(
       language: language,
       unexpectedBeforeComma,
       comma: comma,
-      unexpectedBeforeCxxName,
       cxxName: cxxName,
       arena: self.arena
     )
@@ -987,7 +976,7 @@ extension Parser {
   mutating func parseOriginallyDefinedInArguments() -> RawOriginallyDefinedInArgumentsSyntax {
     let (unexpectedBeforeModuleLabel, moduleLabel) = self.expect(.keyword(.module), remapping: .identifier)
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
-    let (unexpectedBeforeModuleName, moduleName) = self.expect(.stringLiteral)
+    let moduleName = self.parseStringLiteral()
     let (unexpectedBeforeComma, comma) = self.expect(.comma)
 
     var platforms: [RawAvailabilityVersionRestrictionListEntrySyntax] = []
@@ -1009,7 +998,6 @@ extension Parser {
       moduleLabel: moduleLabel,
       unexpectedBeforeColon,
       colon: colon,
-      unexpectedBeforeModuleName,
       moduleName: moduleName,
       unexpectedBeforeComma,
       comma: comma,
@@ -1023,13 +1011,12 @@ extension Parser {
   mutating func parseUnderscorePrivateAttributeArguments() -> RawUnderscorePrivateAttributeArgumentsSyntax {
     let (unexpectedBeforeLabel, label) = self.expect(.keyword(.sourceFile), remapping: .identifier)
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
-    let (unexpectedBeforeFilename, filename) = self.expect(.stringLiteral)
+    let filename = self.parseStringLiteral()
     return RawUnderscorePrivateAttributeArgumentsSyntax(
       unexpectedBeforeLabel,
       sourceFileLabel: label,
       unexpectedBeforeColon,
       colon: colon,
-      unexpectedBeforeFilename,
       filename: filename,
       arena: self.arena
     )
@@ -1066,13 +1053,12 @@ extension Parser {
   mutating func parseUnavailableFromAsyncArguments() -> RawUnavailableFromAsyncArgumentsSyntax {
     let (unexpectedBeforeLabel, label) = self.expect(.keyword(.message), remapping: .identifier)
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
-    let (unexpectedBeforeMessage, message) = self.expect(.stringLiteral)
+    let message = self.parseStringLiteral()
     return RawUnavailableFromAsyncArgumentsSyntax(
       unexpectedBeforeLabel,
       messageLabel: label,
       unexpectedBeforeColon,
       colon: colon,
-      unexpectedBeforeMessage,
       message: message,
       arena: self.arena
     )
@@ -1088,15 +1074,22 @@ extension Parser {
       let (unexpectedBeforeLabel, label) = self.expectAny([.keyword(.visibility), .keyword(.metadata)], default: .keyword(.visibility))
       let (unexpectedBeforeColon, colon) = self.expect(.colon)
       let unexpectedBeforeValue: RawUnexpectedNodesSyntax?
-      let value: RawTokenSyntax
+      let value: RawDocumentationAttributeArgumentSyntax.Value
       switch label.tokenText {
       case "visibility":
-        (unexpectedBeforeValue, value) = self.expectAny([.keyword(.open), .keyword(.public), .keyword(.internal), .keyword(.fileprivate), .keyword(.private)], default: .keyword(.internal))
+        let (unexpected, token) = self.expectAny([.keyword(.open), .keyword(.public), .keyword(.internal), .keyword(.fileprivate), .keyword(.private)], default: .keyword(.internal))
+        unexpectedBeforeValue = unexpected
+        value = .token(token)
       case "metadata":
-        (unexpectedBeforeValue, value) = self.expectAny([.stringLiteral, .identifier], default: .stringLiteral)
+        unexpectedBeforeValue = nil
+        if let identifier = self.consume(if: .identifier) {
+          value = .token(identifier)
+        } else {
+          value = .string(self.parseStringLiteral())
+        }
       default:
         unexpectedBeforeValue = nil
-        value = missingToken(.identifier)
+        value = .token(missingToken(.identifier))
       }
       keepGoing = self.consume(if: .comma)
       arguments.append(

Sources/SwiftParser/Availability.swift

@@ -108,7 +108,7 @@ extension Parser {
         let argumentLabel = self.eat(handle)
         let (unexpectedBeforeColon, colon) = self.expect(.colon)
         // FIXME: Make sure this is a string literal with no interpolation.
-        let stringValue = self.consumeAnyToken()
+        let stringValue = self.parseStringLiteral()
 
         entry = .availabilityLabeledArgument(
           RawAvailabilityLabeledArgumentSyntax(

Sources/SwiftParser/Declarations.swift

@@ -1251,7 +1251,7 @@ extension Parser {
     /// consumes the entire regex literal, we're done.
     return self.currentToken.tokenText.withBuffer {
       (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
-      var cursor = Lexer.Cursor(input: buffer, previous: 0)
+      var cursor = Lexer.Cursor(input: buffer, previous: 0, state: .normal)
       guard buffer[0] == UInt8(ascii: "/") else { return false }
       switch cursor.lexOperatorIdentifier(cursor, cursor).tokenKind {
       case .unknown:
@@ -2151,19 +2151,7 @@ extension Parser {
     }
 
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
-    let stringLiteral: RawStringLiteralExprSyntax
-    if self.at(.stringLiteral) {
-      stringLiteral = self.parseStringLiteral()
-    } else {
-      stringLiteral = RawStringLiteralExprSyntax(
-        openDelimiter: nil,
-        openQuote: RawTokenSyntax(missing: .stringQuote, arena: self.arena),
-        segments: RawStringLiteralSegmentsSyntax(elements: [], arena: self.arena),
-        closeQuote: RawTokenSyntax(missing: .stringQuote, arena: self.arena),
-        closeDelimiter: nil,
-        arena: self.arena
-      )
-    }
+    let stringLiteral = self.parseStringLiteral()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
 
     switch directive {

Sources/SwiftParser/Directives.swift

@@ -154,7 +154,7 @@ extension Parser {
     if !self.at(.rightParen) {
       let (unexpectedBeforeFile, file) = self.expectIdentifier()
       let (unexpectedBeforeFileColon, fileColon) = self.expect(.colon)
-      let (unexpectedBeforeFileName, fileName) = self.expect(.stringLiteral)
+      let fileName = self.parseStringLiteral()
       let (unexpectedBeforeComma, comma) = self.expect(.comma)
 
       let (unexpectedBeforeLine, line) = self.expectIdentifier()
@@ -166,7 +166,6 @@ extension Parser {
         fileArgLabel: file,
         unexpectedBeforeFileColon,
         fileArgColon: fileColon,
-        unexpectedBeforeFileName,
         fileName: fileName,
         unexpectedBeforeComma,
         comma: comma,