bpo-36878: Track extra text added to 'type: ignore' in the AST (GH-13479)

GH-13238 made extra text after a # type: ignore accepted by the parser.
This finishes the job and actually plumbs the extra text through the
parser and makes it available in the AST.

Author: msullivan

Include/Python-ast.h

@@ -272,7 +272,16 @@ def test_vardecl(self):
 
     def test_ignores(self):
         for tree in self.parse_all(ignores):
-            self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5, 8, 9, 10, 11])
+            self.assertEqual(
+                [(ti.lineno, ti.tag) for ti in tree.type_ignores],
+                [
+                    (2, ''),
+                    (5, ''),
+                    (8, '[excuse]'),
+                    (9, '=excuse'),
+                    (10, ' [excuse]'),
+                    (11, ' whatever'),
+                ])
         tree = self.classic_parse(ignores)
         self.assertEqual(tree.type_ignores, [])
 

Lib/test/test_type_comments.py

@@ -1594,6 +1594,7 @@ Daniel Stutzbach
 Andreas Stührk
 Colin Su
 Pal Subbiah
+Michael J. Sullivan
 Nathan Sullivan
 Mark Summerfield
 Reuben Sumner

Misc/ACKS

@@ -0,0 +1,3 @@
+Store text appearing after a `# type: ignore` comment in the AST. For
+example a type ignore like `# type: ignore[E1000]` will have the string
+`"[E1000]"` stored in its AST node.

Misc/NEWS.d/next/Core and Builtins/2019-05-21-16-21-22.bpo-36878.EFRHZ3.rst

@@ -125,6 +125,5 @@ module Python
 
     withitem = (expr context_expr, expr? optional_vars)
 
-    type_ignore = TypeIgnore(int lineno)
+    type_ignore = TypeIgnore(int lineno, string tag)
 }
-

Parser/Python.asdl

@@ -16,13 +16,16 @@ static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
 static int initerr(perrdetail *err_ret, PyObject * filename);
 
 typedef struct {
-    int *items;
+    struct {
+        int lineno;
+        char *comment;
+    } *items;
     size_t size;
     size_t num_items;
-} growable_int_array;
+} growable_comment_array;
 
 static int
-growable_int_array_init(growable_int_array *arr, size_t initial_size) {
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
     assert(initial_size > 0);
     arr->items = malloc(initial_size * sizeof(*arr->items));
     arr->size = initial_size;
@@ -32,7 +35,7 @@ growable_int_array_init(growable_int_array *arr, size_t initial_size) {
 }
 
 static int
-growable_int_array_add(growable_int_array *arr, int item) {
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
     if (arr->num_items >= arr->size) {
         arr->size *= 2;
         arr->items = realloc(arr->items, arr->size * sizeof(*arr->items));
@@ -41,13 +44,17 @@ growable_int_array_add(growable_int_array *arr, int item) {
         }
     }
 
-    arr->items[arr->num_items] = item;
+    arr->items[arr->num_items].lineno = lineno;
+    arr->items[arr->num_items].comment = comment;
     arr->num_items++;
     return 1;
 }
 
 static void
-growable_int_array_deallocate(growable_int_array *arr) {
+growable_comment_array_deallocate(growable_comment_array *arr) {
+    for (unsigned i = 0; i < arr->num_items; i++) {
+        PyObject_FREE(arr->items[i].comment);
+    }
     free(arr->items);
 }
 
@@ -220,9 +227,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     node *n;
     int started = 0;
     int col_offset, end_col_offset;
-    growable_int_array type_ignores;
+    growable_comment_array type_ignores;
 
-    if (!growable_int_array_init(&type_ignores, 10)) {
+    if (!growable_comment_array_init(&type_ignores, 10)) {
         err_ret->error = E_NOMEM;
         PyTokenizer_Free(tok);
         return NULL;
@@ -320,8 +327,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         }
 
         if (type == TYPE_IGNORE) {
-            PyObject_FREE(str);
-            if (!growable_int_array_add(&type_ignores, tok->lineno)) {
+            if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) {
                 err_ret->error = E_NOMEM;
                 break;
             }
@@ -355,17 +361,24 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             REQ(ch, ENDMARKER);
 
             for (i = 0; i < type_ignores.num_items; i++) {
-                PyNode_AddChild(ch, TYPE_IGNORE, NULL,
-                                type_ignores.items[i], 0,
-                                type_ignores.items[i], 0);
+                int res = PyNode_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment,
+                                          type_ignores.items[i].lineno, 0,
+                                          type_ignores.items[i].lineno, 0);
+                if (res != 0) {
+                    err_ret->error = res;
+                    PyNode_Free(n);
+                    n = NULL;
+                    break;
+                }
+                type_ignores.items[i].comment = NULL;
             }
         }
 
         /* Check that the source for a single input statement really
            is a single statement by looking at what is left in the
            buffer after parsing.  Trailing whitespace and comments
            are OK.  */
-        if (start == single_input) {
+        if (err_ret->error == E_DONE && start == single_input) {
             char *cur = tok->cur;
             char c = *tok->cur;
 
@@ -392,7 +405,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     else
         n = NULL;
 
-    growable_int_array_deallocate(&type_ignores);
+    growable_comment_array_deallocate(&type_ignores);
 
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
     *flags = ps->p_flags;

Parser/parsetok.c

@@ -1269,17 +1269,21 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             /* This is a type comment if we matched all of type_comment_prefix. */
             if (!*prefix) {
                 int is_type_ignore = 1;
+                const char *ignore_end = p + 6;
                 tok_backup(tok, c);  /* don't eat the newline or EOF */
 
                 type_start = p;
 
                 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
                  * or anything non-alphanumeric. */
                 is_type_ignore = (
-                    tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0
-                    && !(tok->cur > p + 6 && isalnum(p[6])));
+                    tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
+                    && !(tok->cur > ignore_end && isalnum(p[6])));
 
                 if (is_type_ignore) {
+                    *p_start = (char *) ignore_end;
+                    *p_end = tok->cur;
+
                     /* If this type ignore is the only thing on the line, consume the newline also. */
                     if (blankline) {
                         tok_nextc(tok);

Parser/tokenizer.c

@@ -830,7 +830,10 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
                 goto out;
 
             for (i = 0; i < num; i++) {
-                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena);
+                string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
+                if (!type_comment)
+                    goto out;
+                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
                 if (!ti)
                    goto out;
                asdl_seq_SET(type_ignores, i, ti);