bpo-42827: Fix crash on SyntaxError in multiline expressions

When trying to extract the error line for the error message there
are two distinct cases:
1. The input comes from a file, which means that we can extract the
   error line by using `PyErr_ProgramTextObject` and which we already
   do.
2. The input does not come from a file, at which point we need to get
   the source code from the tokenizer:
   * If the tokenizer's current line number is the same with the line
     of the error, we get the line from `tok->buf` and we're ready.
   * Else, we can extract the error line from the source code in the
     following two ways:
     * If the input comes from a string we have all the input
       in `tok->str` and we can extract the error line from it.
     * If the input comes from stdin, i.e. the interactive prompt, we
       do not have access to the previous line. That's why a new
       field `tok->stdin_content` is added which holds the whole input for the
       current (multiline) statement or expression. We can then extract the
       error line from `tok->stdin_content` like we do in the string case above.

Author: lysnikolaou

Lib/test/test_exceptions.py

@@ -209,6 +209,9 @@ def testSyntaxErrorOffset(self):
         check('x = "a', 1, 7)
         check('lambda x: x = 2', 1, 1)
         check('f{a + b + c}', 1, 2)
+        check('[file for str(file) in []\n])', 1, 11)
+        check('[\nfile\nfor str(file)\nin\n[]\n]', 3, 5)
+        check('[file for\n str(file) in []]', 2, 2)
 
         # Errors thrown by compile.c
         check('class foo:return 1', 1, 11)

Parser/pegen.c

@@ -380,6 +380,21 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
     return NULL;
 }
 
+PyObject *
+get_error_line(Parser *p, int lineno)
+{
+    char *cur_line = p->tok->fp == NULL ? p->tok->str : p->tok->stdin_content;
+    for (int i = 0; i < lineno - 1; i++) {
+        cur_line = strchr(cur_line, '\n') + 1;
+    }
+
+    char *next_newline;
+    if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
+        return PyUnicode_DecodeUTF8(cur_line, strlen(cur_line), "replace");
+    }
+    return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
+}
+
 void *
 _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
                                     Py_ssize_t lineno, Py_ssize_t col_offset,
@@ -416,8 +431,13 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     }
 
     if (!error_line) {
-        Py_ssize_t size = p->tok->inp - p->tok->buf;
-        error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
+        if (p->tok->lineno == lineno) {
+            Py_ssize_t size = p->tok->inp - p->tok->buf;
+            error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
+        }
+        else {
+            error_line = get_error_line(p, lineno);
+        }
         if (!error_line) {
             goto error;
         }

Parser/tokenizer.c

@@ -81,6 +81,7 @@ tok_new(void)
     tok->decoding_readline = NULL;
     tok->decoding_buffer = NULL;
     tok->type_comments = 0;
+    tok->stdin_content = NULL;
 
     tok->async_hacks = 0;
     tok->async_def = 0;
@@ -856,6 +857,25 @@ tok_nextc(struct tok_state *tok)
                 if (translated == NULL)
                     return EOF;
                 newtok = translated;
+                if (tok->stdin_content == NULL) {
+                    tok->stdin_content = PyMem_Malloc(strlen(translated) + 1);
+                    if (tok->stdin_content == NULL) {
+                        tok->done = E_NOMEM;
+                        return EOF;
+                    }
+                    strcpy(tok->stdin_content, translated);
+                    tok->stdin_content[strlen(translated)] = 0;
+                }
+                else {
+                    char *new_str = PyMem_Malloc(strlen(tok->stdin_content) + strlen(translated) + 1);
+                    if (new_str == NULL) {
+                        tok->done = E_NOMEM;
+                        return EOF;
+                    }
+                    sprintf(new_str, "%s%s", tok->stdin_content, translated);
+                    PyMem_Free(tok->stdin_content);
+                    tok->stdin_content = new_str;
+                }
             }
             if (tok->encoding && newtok && *newtok) {
                 /* Recode to UTF-8 */

Parser/tokenizer.h

@@ -37,6 +37,7 @@ struct tok_state {
     int atbol;          /* Nonzero if at begin of new line */
     int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
     const char *prompt, *nextprompt;          /* For interactive prompting */
+    char *stdin_content;
     int lineno;         /* Current line number */
     int first_lineno;   /* First line of a single line or multi line string
                            expression (cf. issue 16806) */