Merge pull request #1250 from ahoppen/ahoppen/dont-relex-string-literals

Lex string literal segments in the lexer and compose them in the parser

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

@@ -221,7 +221,6 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PoundConfigSpec(name: "PoundHasSymbol", kind: "pound__hasSymbol", text: "#_hasSymbol"),
   LiteralSpec(name: "IntegerLiteral", kind: "integer_literal", nameForDiagnostics: "integer literal", classification: "IntegerLiteral"),
   LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"),
-  LiteralSpec(name: "StringLiteralContents", kind: "string_literal", nameForDiagnostics: "string literal", classification: "StringLiteral"),
   LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"),
   MiscSpec(name: "Unknown", kind: "unknown", nameForDiagnostics: "token"),
   MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"),

Sources/IDEUtils/generated/SyntaxClassification.swift

@@ -242,8 +242,6 @@ extension RawTokenKind {
       return .integerLiteral
     case .floatingLiteral: 
       return .floatingLiteral
-    case .stringLiteralContents: 
-      return .stringLiteral
     case .regexLiteral: 
       return .none
     case .unknown: 

Sources/SwiftParser/CMakeLists.txt

@@ -24,6 +24,7 @@ add_swift_host_library(SwiftParser
   RawTokenKindSubset.swift
   Recovery.swift
   Statements.swift
+  StringLiterals.swift
   SyntaxUtils.swift
   TokenConsumer.swift
   TokenPrecedence.swift

Sources/SwiftParser/Declarations.swift

@@ -1251,7 +1251,7 @@ extension Parser {
     /// consumes the entire regex literal, we're done.
     return self.currentToken.tokenText.withBuffer {
       (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
-      var cursor = Lexer.Cursor(input: buffer, previous: 0, state: .normal)
+      var cursor = Lexer.Cursor(input: buffer, previous: 0)
       guard buffer[0] == UInt8(ascii: "/") else { return false }
       switch cursor.lexOperatorIdentifier(sourceBufferStart: cursor).tokenKind {
       case .unknown: