Refactor the lexer to store lexer errors in the tokens instead of having a diagnosticsHandler

Instead of marking a token as having an error and relying on re-lexing to emit a diagnostic, store the type of error and the offset at which it occurred in the token itself.

This is a more sound design IMO and will continue to work even if we introduce state into the lexer (e.g. whether we are currently in a string literal or inside a conflict marker range).

Author: ahoppen

Package.swift

@@ -97,7 +97,7 @@ let package = Package(
     ),
     .target(
       name: "SwiftParser",
-      dependencies: ["SwiftDiagnostics", "SwiftSyntax"],
+      dependencies: ["SwiftSyntax"],
       exclude: [
         "CMakeLists.txt",
         "README.md",

Sources/SwiftParser/Declarations.swift

@@ -1222,9 +1222,8 @@ extension Parser {
       (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
       var cursor = Lexer.Cursor(input: buffer, previous: 0)
       guard buffer[0] == UInt8(ascii: "/") else { return false }
-
-      switch (cursor.lexOperatorIdentifier(cursor, cursor)) {
-      case (.unknown, _):
+      switch cursor.lexOperatorIdentifier(cursor, cursor).tokenKind {
+      case .unknown:
         return false
 
       default:

Sources/SwiftParser/Expressions.swift

@@ -1584,7 +1584,7 @@ extension Parser {
       wholeText: wholeText,
       textRange: textRange,
       presence: .present,
-      hasLexerError: false,
+      lexerError: nil,
       arena: self.arena
     )
   }