Merge pull request #790 from CodaFi/lex-luthor

Newlines Prevent Bare Slash Regex Lexing

Author: CodaFi

Sources/SwiftParser/Lexer.swift

@@ -2029,7 +2029,7 @@ extension Lexer.Cursor {
     // }
     //
     if poundCount == 0 && !Tmp.isAtEndOfFile &&
-        (Tmp.peek() == UInt8(ascii: " ") || Tmp.peek() == UInt8(ascii: "\t")) {
+        (Tmp.peek() == UInt8(ascii: " ") || Tmp.peek() == UInt8(ascii: "\n") || Tmp.peek() == UInt8(ascii: "\t")) {
       return nil
     }
 

Tests/SwiftParserTest/LexerTests.swift

@@ -485,28 +485,58 @@ public class LexerTests: XCTestCase {
   }
 
   func testNotARegex() {
-    var data =
-    """
-    min(reduced.count / 2, chunkSize / 2)
-    """
-    let lexemes = data.withUTF8 { buf in
-      Lexer.lex(buf)
+    do {
+      var data =
+      """
+      min(reduced.count / 2, chunkSize / 2)
+      """
+      let lexemes = data.withUTF8 { buf in
+        Lexer.lex(buf)
+      }
+      AssertEqualTokens(lexemes, [
+        lexeme(.identifier, "min"),
+        lexeme(.leftParen, "("),
+        lexeme(.identifier, "reduced"),
+        lexeme(.period, "."),
+        lexeme(.identifier, "count ", trailing: 1),
+        lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
+        lexeme(.integerLiteral, "2"),
+        lexeme(.comma, ", ", trailing: 1),
+        lexeme(.identifier, "chunkSize ", trailing: 1),
+        lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
+        lexeme(.integerLiteral, "2"),
+        lexeme(.rightParen, ")"),
+        lexeme(.eof, ""),
+      ])
+    }
+
+    do {
+      var data =
+      """
+      var x: Int {
+        return 0 /
+               x
+      }
+
+      ///
+      """
+      let lexemes = data.withUTF8 { buf in
+        Lexer.lex(buf)
+      }
+      AssertEqualTokens(lexemes, [
+        lexeme(.varKeyword, "var ", trailing: 1),
+        lexeme(.identifier, "x"),
+        lexeme(.colon, ": ", trailing: 1),
+        lexeme(.identifier, "Int ", trailing: 1),
+        lexeme(.leftBrace, "{"),
+        lexeme(.returnKeyword, "\n  return ", leading: 3, trailing: 1),
+        lexeme(.integerLiteral, "0 ", trailing: 1),
+        lexeme(.spacedBinaryOperator, "/", trailing: 0),
+        lexeme(.identifier, "\n         x", leading: 10),
+        lexeme(.rightBrace, "\n}", leading: 1),
+        lexeme(.eof, "\n\n///", leading: 5),
+      ])
     }
-    AssertEqualTokens(lexemes, [
-      lexeme(.identifier, "min"),
-      lexeme(.leftParen, "("),
-      lexeme(.identifier, "reduced"),
-      lexeme(.period, "."),
-      lexeme(.identifier, "count ", trailing: 1),
-      lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
-      lexeme(.integerLiteral, "2"),
-      lexeme(.comma, ", ", trailing: 1),
-      lexeme(.identifier, "chunkSize ", trailing: 1),
-      lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
-      lexeme(.integerLiteral, "2"),
-      lexeme(.rightParen, ")"),
-      lexeme(.eof, ""),
-    ])
   }
 
   func testUnicodeReplcementsInStream() {