bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)

(cherry picked from commit 5ec91f7)

Co-authored-by: Pablo Galindo <[email protected]>

Author: miss-islington

Lib/test/test_repl.py

@@ -58,5 +58,41 @@ def test_no_memory(self):
         # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
         self.assertIn(p.returncode, (1, 120))
 
+    @cpython_only
+    def test_multiline_string_parsing(self):
+        # bpo-39209: Multiline string tokens need to be handled in the tokenizer
+        # in two places: the interactive path and the non-interactive path.
+        user_input = '''\
+        x = """<?xml version="1.0" encoding="iso-8859-1"?>
+        <test>
+            <Users>
+                <fun25>
+                    <limits>
+                        <total>0KiB</total>
+                        <kbps>0</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </limits>
+                    <usages>
+                        <total>16738211KiB</total>
+                        <kbps>237.15</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </usages>
+                    <time_to_refresh>never</time_to_refresh>
+                    <limit_exceeded_URL>none</limit_exceeded_URL>
+                </fun25>
+            </Users>
+        </test>"""
+        '''
+        user_input = dedent(user_input)
+        user_input = user_input.encode()
+        p = spawn_repl()
+        with SuppressCrashReport():
+            p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst

@@ -0,0 +1,2 @@
+Correctly handle multi-line tokens in interactive mode. Patch by Pablo
+Galindo.

Parser/tokenizer.c

@@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok)
                 size_t start = tok->start - tok->buf;
                 size_t oldlen = tok->cur - tok->buf;
                 size_t newlen = oldlen + strlen(newtok);
+                Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
                 char *buf = tok->buf;
                 buf = (char *)PyMem_REALLOC(buf, newlen+1);
                 tok->lineno++;
@@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok)
                 }
                 tok->buf = buf;
                 tok->cur = tok->buf + oldlen;
+                tok->multi_line_start = tok->buf + cur_multi_line_start;
                 tok->line_start = tok->cur;
                 strcpy(tok->buf + oldlen, newtok);
                 PyMem_FREE(newtok);