Skip to content

Commit c471ca4

Browse files
albertjanserhiy-storchaka
authored andcommitted
bpo-30377: Simplify handling of COMMENT and NL in tokenize.py (#1607)
1 parent a17a2f5 commit c471ca4

File tree

2 files changed

+12
-12
lines changed

2 files changed

+12
-12
lines changed

Lib/test/test_tokenize.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,20 +39,22 @@ def test_basic(self):
3939
""")
4040
self.check_tokenize("if False:\n"
4141
" # NL\n"
42+
" \n"
4243
" True = False # NEWLINE\n", """\
4344
NAME 'if' (1, 0) (1, 2)
4445
NAME 'False' (1, 3) (1, 8)
4546
OP ':' (1, 8) (1, 9)
4647
NEWLINE '\\n' (1, 9) (1, 10)
4748
COMMENT '# NL' (2, 4) (2, 8)
4849
NL '\\n' (2, 8) (2, 9)
49-
INDENT ' ' (3, 0) (3, 4)
50-
NAME 'True' (3, 4) (3, 8)
51-
OP '=' (3, 9) (3, 10)
52-
NAME 'False' (3, 11) (3, 16)
53-
COMMENT '# NEWLINE' (3, 17) (3, 26)
54-
NEWLINE '\\n' (3, 26) (3, 27)
55-
DEDENT '' (4, 0) (4, 0)
50+
NL '\\n' (3, 4) (3, 5)
51+
INDENT ' ' (4, 0) (4, 4)
52+
NAME 'True' (4, 4) (4, 8)
53+
OP '=' (4, 9) (4, 10)
54+
NAME 'False' (4, 11) (4, 16)
55+
COMMENT '# NEWLINE' (4, 17) (4, 26)
56+
NEWLINE '\\n' (4, 26) (4, 27)
57+
DEDENT '' (5, 0) (5, 0)
5658
""")
5759
indent_error_file = b"""\
5860
def k(x):

Lib/tokenize.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -560,13 +560,11 @@ def _tokenize(readline, encoding):
560560
if line[pos] in '#\r\n': # skip comments or blank lines
561561
if line[pos] == '#':
562562
comment_token = line[pos:].rstrip('\r\n')
563-
nl_pos = pos + len(comment_token)
564563
yield TokenInfo(COMMENT, comment_token,
565564
(lnum, pos), (lnum, pos + len(comment_token)), line)
566-
yield TokenInfo(NL, line[nl_pos:],
567-
(lnum, nl_pos), (lnum, len(line)), line)
568-
else:
569-
yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:],
565+
pos += len(comment_token)
566+
567+
yield TokenInfo(NL, line[pos:],
570568
(lnum, pos), (lnum, len(line)), line)
571569
continue
572570

0 commit comments

Comments
 (0)