Fix test_tokenize

Author: lysnikolaou

Include/internal/pycore_token.h

@@ -91,8 +91,7 @@ extern "C" {
                                  (x) == INDENT    || \
                                  (x) == DEDENT)
 #define ISSTRINGLIT(x)          ((x) == STRING           || \
-                                 (x) == FSTRING_MIDDLE   || \
-                                 (x) == FSTRING_END)
+                                 (x) == FSTRING_MIDDLE)
 
 
 // Symbols exported for test_peg_generator

Lib/test/test_tokenize.py

@@ -1938,7 +1938,8 @@ def test_string(self):
 
         self.check_tokenize('f"abc"', """\
     FSTRING_START 'f"'          (1, 0) (1, 2)
-    FSTRING_END 'abc'         (1, 2) (1, 5)
+    FSTRING_MIDDLE 'abc'         (1, 2) (1, 5)
+    FSTRING_END '"'           (1, 5) (1, 6)
     """)
 
         self.check_tokenize('fR"a{b}c"', """\
@@ -1947,24 +1948,28 @@ def test_string(self):
     LBRACE     '{'           (1, 4) (1, 5)
     NAME       'b'           (1, 5) (1, 6)
     RBRACE     '}'           (1, 6) (1, 7)
-    FSTRING_END 'c'           (1, 7) (1, 8)
+    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
+    FSTRING_END '"'           (1, 8) (1, 9)
     """)
 
         self.check_tokenize('f"""abc"""', """\
     FSTRING_START 'f\"""'        (1, 0) (1, 4)
-    FSTRING_END 'abc'         (1, 4) (1, 7)
+    FSTRING_MIDDLE 'abc'         (1, 4) (1, 7)
+    FSTRING_END '\"""'         (1, 7) (1, 10)
     """)
 
         self.check_tokenize(r'f"abc\
 def"', """\
     FSTRING_START \'f"\'          (1, 0) (1, 2)
-    FSTRING_END 'abc\\\\\\ndef'  (1, 2) (2, 3)
+    FSTRING_MIDDLE 'abc\\\\\\ndef'  (1, 2) (2, 3)
+    FSTRING_END '"'           (2, 3) (2, 4)
     """)
 
         self.check_tokenize(r'Rf"abc\
 def"', """\
     FSTRING_START 'Rf"'         (1, 0) (1, 3)
-    FSTRING_END 'abc\\\\\\ndef'  (1, 3) (2, 3)
+    FSTRING_MIDDLE 'abc\\\\\\ndef'  (1, 3) (2, 3)
+    FSTRING_END '"'           (2, 3) (2, 4)
     """)
 
     def test_function(self):