Skip to content

Commit c3f167d

Browse files
authored
bpo-42864: Simplify the tokenizer exceptions after generic SyntaxError (GH-24273)
Automerge-Triggered-By: GH:pablogsal
1 parent 75e59a9 commit c3f167d

File tree

1 file changed

+3
-10
lines changed

1 file changed

+3
-10
lines changed

Parser/pegen.c

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1177,14 +1177,9 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
11771177
return 0;
11781178
}
11791179

1180-
11811180
Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
11821181
Py_ssize_t current_err_line = current_token->lineno;
11831182

1184-
// Save the tokenizer state to restore them later in case we found nothing
1185-
struct tok_state saved_tok;
1186-
memcpy(&saved_tok, p->tok, sizeof(struct tok_state));
1187-
11881183
for (;;) {
11891184
const char *start;
11901185
const char *end;
@@ -1206,8 +1201,6 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
12061201
break;
12071202
}
12081203

1209-
// Restore the tokenizer state
1210-
memcpy(p->tok, &saved_tok, sizeof(struct tok_state));
12111204
return 0;
12121205
}
12131206

@@ -1239,10 +1232,10 @@ _PyPegen_run_parser(Parser *p)
12391232
RAISE_INDENTATION_ERROR("unexpected unindent");
12401233
}
12411234
else {
1242-
if (_PyPegen_check_tokenizer_errors(p)) {
1243-
return NULL;
1244-
}
12451235
RAISE_SYNTAX_ERROR("invalid syntax");
1236+
// _PyPegen_check_tokenizer_errors will override the existing
1237+
// generic SyntaxError we just raised if errors are found.
1238+
_PyPegen_check_tokenizer_errors(p);
12461239
}
12471240
}
12481241
return NULL;

0 commit comments

Comments
 (0)