Skip to content

Commit 6c4145a

Browse files
committed
Newlines Prevent Bare Slash Regex Lexing
Break bare slash regex parsing at newline boundaries as well. The lexing test here was trying to pair the / operator with the comment operator several lines below, which was catching the brace in the fray.
1 parent 4b057ee commit 6c4145a

File tree

2 files changed

+52
-22
lines changed

2 files changed

+52
-22
lines changed

Sources/SwiftParser/Lexer.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2029,7 +2029,7 @@ extension Lexer.Cursor {
20292029
// }
20302030
//
20312031
if poundCount == 0 && !Tmp.isAtEndOfFile &&
2032-
(Tmp.peek() == UInt8(ascii: " ") || Tmp.peek() == UInt8(ascii: "\t")) {
2032+
(Tmp.peek() == UInt8(ascii: " ") || Tmp.peek() == UInt8(ascii: "\n") || Tmp.peek() == UInt8(ascii: "\t")) {
20332033
return nil
20342034
}
20352035

Tests/SwiftParserTest/LexerTests.swift

Lines changed: 51 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -485,28 +485,58 @@ public class LexerTests: XCTestCase {
485485
}
486486

487487
func testNotARegex() {
488-
var data =
489-
"""
490-
min(reduced.count / 2, chunkSize / 2)
491-
"""
492-
let lexemes = data.withUTF8 { buf in
493-
Lexer.lex(buf)
488+
do {
489+
var data =
490+
"""
491+
min(reduced.count / 2, chunkSize / 2)
492+
"""
493+
let lexemes = data.withUTF8 { buf in
494+
Lexer.lex(buf)
495+
}
496+
AssertEqualTokens(lexemes, [
497+
lexeme(.identifier, "min"),
498+
lexeme(.leftParen, "("),
499+
lexeme(.identifier, "reduced"),
500+
lexeme(.period, "."),
501+
lexeme(.identifier, "count ", trailing: 1),
502+
lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
503+
lexeme(.integerLiteral, "2"),
504+
lexeme(.comma, ", ", trailing: 1),
505+
lexeme(.identifier, "chunkSize ", trailing: 1),
506+
lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
507+
lexeme(.integerLiteral, "2"),
508+
lexeme(.rightParen, ")"),
509+
lexeme(.eof, ""),
510+
])
511+
}
512+
513+
do {
514+
var data =
515+
"""
516+
var x: Int {
517+
return 0 /
518+
x
519+
}
520+
521+
///
522+
"""
523+
let lexemes = data.withUTF8 { buf in
524+
Lexer.lex(buf)
525+
}
526+
AssertEqualTokens(lexemes, [
527+
lexeme(.varKeyword, "var ", trailing: 1),
528+
lexeme(.identifier, "x"),
529+
lexeme(.colon, ": ", trailing: 1),
530+
lexeme(.identifier, "Int ", trailing: 1),
531+
lexeme(.leftBrace, "{"),
532+
lexeme(.returnKeyword, "\n return ", leading: 3, trailing: 1),
533+
lexeme(.integerLiteral, "0 ", trailing: 1),
534+
lexeme(.spacedBinaryOperator, "/", trailing: 0),
535+
lexeme(.identifier, "\n x", leading: 10),
536+
lexeme(.rightBrace, "\n}", leading: 1),
537+
lexeme(.eof, "\n\n///", leading: 5),
538+
])
494539
}
495-
AssertEqualTokens(lexemes, [
496-
lexeme(.identifier, "min"),
497-
lexeme(.leftParen, "("),
498-
lexeme(.identifier, "reduced"),
499-
lexeme(.period, "."),
500-
lexeme(.identifier, "count ", trailing: 1),
501-
lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
502-
lexeme(.integerLiteral, "2"),
503-
lexeme(.comma, ", ", trailing: 1),
504-
lexeme(.identifier, "chunkSize ", trailing: 1),
505-
lexeme(.spacedBinaryOperator, "/ ", trailing: 1),
506-
lexeme(.integerLiteral, "2"),
507-
lexeme(.rightParen, ")"),
508-
lexeme(.eof, ""),
509-
])
510540
}
511541

512542
func testUnicodeReplcementsInStream() {

0 commit comments

Comments
 (0)