Parse 0.2 after … as float literal, not member access

Deciding whether `0.2` should be lexed as a float literal or a member access is a little more difficult than just looking at the previous character because `0.2` might be preceeded by an operator like `…` or `.^.`, in which case it should be lexed as a float literal and not a member access.

We might be able to do some disambiguation magic on whether the character before the period is also an operator continuation point but that seems fairly brittle to me. The sanest way of doing this, is to store the previously lexed token’s kind in the cursor and checking that.

I measured and did not see a performance regregssion when parsing MovieSwiftUI.

rdar://103273988

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -201,6 +201,8 @@ extension Lexer {
   struct Cursor {
     var input: UnsafeBufferPointer<UInt8>
     var previous: UInt8
+    /// If we have already lexed a token, the kind of the previously lexed token
+    var previousTokenKind: RawTokenBaseKind?
     private var stateStack: StateStack = StateStack()
 
     init(input: UnsafeBufferPointer<UInt8>, previous: UInt8) {
@@ -335,6 +337,7 @@ extension Lexer.Cursor {
       flags.insert(.isAtStartOfLine)
     }
 
+    self.previousTokenKind = result.tokenKind.base
     let error = result.error.map { error in
       return LexerError(error.kind, byteOffset: cursor.distance(to: error.position))
     }
@@ -676,6 +679,9 @@ extension Lexer.Cursor {
   }
 
   /// Rever the lexer by `offset` bytes. This should only be used by `resetForSplit`.
+  /// This must not back up by more bytes than the last token because that would
+  /// require us to also update `previousTokenKind`, which we don't do in this
+  /// function
   mutating func backUp(by offset: Int) {
     assert(!self.isAtStartOfFile)
     self.previous = self.input.baseAddress!.advanced(by: -(offset + 1)).pointee
@@ -1224,11 +1230,16 @@ extension Lexer.Cursor {
 
     // TODO: This can probably be unified with lexHexNumber somehow
 
-    // Lex things like 4.x as '4' followed by a tok::period.
     if self.is(at: ".") {
-      // NextToken is the soon to be previous token
-      // Therefore: x.0.1 is sub-tuple access, not x.float_literal
-      if let peeked = self.peek(at: 1), !Unicode.Scalar(peeked).isDigit || tokenStart.previous == UInt8(ascii: ".") {
+      if self.peek(at: 1) == nil {
+        // If there are no more digits following the '.', we don't have a float
+        // literal.
+        return Lexer.Result(.integerLiteral)
+      } else if let peeked = self.peek(at: 1), !Unicode.Scalar(peeked).isDigit {
+        // ".a" is a member access and certainly not a float literal
+        return Lexer.Result(.integerLiteral)
+      } else if self.previousTokenKind == .period {
+        // Lex x.0.1 is sub-tuple access, not x.float_literal.
         return Lexer.Result(.integerLiteral)
       }
     } else if self.isAtEndOfFile || self.is(notAt: "e", "E") {

Tests/SwiftParserTest/LexerTests.swift

@@ -1002,4 +1002,38 @@ public class LexerTests: XCTestCase {
       ]
     )
   }
+
+  func testMultiDigitTupleAccess() {
+    AssertLexemes(
+      "x.13.1",
+      lexemes: [
+        LexemeSpec(.identifier, text: "x"),
+        LexemeSpec(.period, text: "."),
+        LexemeSpec(.integerLiteral, text: "13"),
+        LexemeSpec(.period, text: "."),
+        LexemeSpec(.integerLiteral, text: "1"),
+      ]
+    )
+  }
+
+  func testFloatingPointNumberAfterRangeOperator() {
+    AssertLexemes(
+      "0.1...0.2",
+      lexemes: [
+        LexemeSpec(.floatingLiteral, text: "0.1"),
+        LexemeSpec(.binaryOperator, text: "..."),
+        LexemeSpec(.floatingLiteral, text: "0.2"),
+      ]
+    )
+  }
+
+  func testUnterminatedFloatLiteral() {
+    AssertLexemes(
+      "0.",
+      lexemes: [
+        LexemeSpec(.integerLiteral, text: "0"),
+        LexemeSpec(.period, text: "."),
+      ]
+    )
+  }
 }