Merge pull request #1227 from ahoppen/ahoppen/refactor-lexer

Refactor the lexer to make it easier to understand, maintain and more swifty

Author: ahoppen

Sources/SwiftParser/CMakeLists.txt

@@ -13,7 +13,6 @@ add_swift_host_library(SwiftParser
   Declarations.swift
   Directives.swift
   Expressions.swift
-  Lexer.swift
   Lookahead.swift
   LoopProgressCondition.swift
   Modifiers.swift
@@ -34,7 +33,14 @@ add_swift_host_library(SwiftParser
 
   generated/DeclarationModifier.swift
   generated/Parser+Entry.swift
-  generated/TypeAttribute.swift)
+  generated/TypeAttribute.swift
+
+  Lexer/Cursor.swift
+  Lexer/Lexeme.swift
+  Lexer/LexemeSequence.swift
+  Lexer/Lexer.swift
+  Lexer/UnicodeScalarExtensions.swift
+)
 
 target_link_libraries(SwiftParser PUBLIC
   SwiftSyntax

Sources/SwiftParser/Declarations.swift

@@ -1253,7 +1253,7 @@ extension Parser {
       (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
       var cursor = Lexer.Cursor(input: buffer, previous: 0, state: .normal)
       guard buffer[0] == UInt8(ascii: "/") else { return false }
-      switch cursor.lexOperatorIdentifier(cursor, cursor).tokenKind {
+      switch cursor.lexOperatorIdentifier(sourceBufferStart: cursor).tokenKind {
       case .unknown:
         return false