Skip to content

Commit cb0c874

Browse files
committed
Fix lexer to balk at unrecognized characters, e.g. '@'
1 parent 2fa0a04 commit cb0c874

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

Tools/cases_generator/lexer.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ def choice(*opts):
112112
COMMENT = 'COMMENT'
113113

114114
newline = r"\n"
115-
matcher = re.compile(choice(id_re, number_re, str_re, char, newline, macro, comment_re, *operators.values()))
115+
invalid = r"\S" # A single non-space character that's not caught by any of the other patterns
116+
matcher = re.compile(choice(id_re, number_re, str_re, char, newline, macro, comment_re, *operators.values(), invalid))
116117
letter = re.compile(r'[a-zA-Z_]')
117118

118119
kwds = (
@@ -177,7 +178,6 @@ def __repr__(self):
177178

178179
def tokenize(src, line=1, filename=None):
179180
linestart = -1
180-
# TODO: finditer() skips over unrecognized characters, e.g. '@'
181181
for m in matcher.finditer(src):
182182
start, end = m.span()
183183
text = m.group(0)

0 commit comments

Comments
 (0)