bpo-39638: Keep ASDL signatures in the AST nodes (GH-18515)

Author: isidentical

Doc/whatsnew/3.9.rst

@@ -164,6 +164,9 @@ be used to unparse an :class:`ast.AST` object and produce a string with code
 that would produce an equivalent :class:`ast.AST` object when parsed.
 (Contributed by Pablo Galindo and Batuhan Taskaya in :issue:`38870`.)
 
+Added docstrings to AST nodes that contains the ASDL signature used to
+construct that node. (Contributed by Batuhan Taskaya in :issue:`39638`.)
+
 asyncio
 -------
 

Lib/test/test_asdl_parser.py

@@ -63,10 +63,10 @@ def test_product(self):
     def test_attributes(self):
         stmt = self.types['stmt']
         self.assertEqual(len(stmt.attributes), 4)
-        self.assertEqual(str(stmt.attributes[0]), 'Field(int, lineno)')
-        self.assertEqual(str(stmt.attributes[1]), 'Field(int, col_offset)')
-        self.assertEqual(str(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)')
-        self.assertEqual(str(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)')
+        self.assertEqual(repr(stmt.attributes[0]), 'Field(int, lineno)')
+        self.assertEqual(repr(stmt.attributes[1]), 'Field(int, col_offset)')
+        self.assertEqual(repr(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)')
+        self.assertEqual(repr(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)')
 
     def test_constructor_fields(self):
         ehandler = self.types['excepthandler']

Lib/test/test_ast.py

@@ -636,6 +636,16 @@ def test_issue39579_dotted_name_end_col_offset(self):
         attr_b = tree.body[0].decorator_list[0].value
         self.assertEqual(attr_b.end_col_offset, 4)
 
+    def test_ast_asdl_signature(self):
+        self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)")
+        self.assertEqual(ast.GtE.__doc__, "GtE")
+        self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)")
+        self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn")
+        expressions = [f"     | {node.__doc__}" for node in ast.expr.__subclasses__()]
+        expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}"
+        self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions)
+
+
 class ASTHelpers_Test(unittest.TestCase):
     maxDiff = None
 

Misc/NEWS.d/next/Library/2020-02-15-20-38-11.bpo-39638.wm_RS3.rst

@@ -0,0 +1,2 @@
+Keep ASDL signatures in the docstrings for ``AST`` nodes. Patch by Batuhan
+Taskaya

Parser/asdl.py

@@ -72,6 +72,16 @@ def __init__(self, type, name=None, seq=False, opt=False):
         self.seq = seq
         self.opt = opt
 
+    def __str__(self):
+        if self.seq:
+            extra = "*"
+        elif self.opt:
+            extra = "?"
+        else:
+            extra = ""
+
+        return "{}{} {}".format(self.type, extra, self.name)
+
     def __repr__(self):
         if self.seq:
             extra = ", seq=True"

Parser/asdl_c.py

@@ -60,6 +60,9 @@ def reflow_lines(s, depth):
         lines.append(padding + cur)
     return lines
 
+def reflow_c_string(s, depth):
+    return '"%s"' % s.replace('\n', '\\n"\n%s"' % (' ' * depth * TABSIZE))
+
 def is_simple(sum):
     """Return True if a sum is a simple.
 
@@ -71,6 +74,21 @@ def is_simple(sum):
             return False
     return True
 
+def asdl_of(name, obj):
+    if isinstance(obj, asdl.Product) or isinstance(obj, asdl.Constructor):
+        fields = ", ".join(map(str, obj.fields))
+        if fields:
+            fields = "({})".format(fields)
+        return "{}{}".format(name, fields)
+    else:
+        if is_simple(obj):
+            types = " | ".join(type.name for type in obj.types)
+        else:
+            sep = "\n{}| ".format(" " * (len(name) + 1))
+            types = sep.join(
+                asdl_of(type.name, type) for type in obj.types
+            )
+        return "{} = {}".format(name, types)
 
 class EmitVisitor(asdl.VisitorBase):
     """Visit that emits lines"""
@@ -764,7 +782,7 @@ def visitModule(self, mod):
 };
 
 static PyObject *
-make_type(const char *type, PyObject* base, const char* const* fields, int num_fields)
+make_type(const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc)
 {
     PyObject *fnames, *result;
     int i;
@@ -778,11 +796,12 @@ def visitModule(self, mod):
         }
         PyTuple_SET_ITEM(fnames, i, field);
     }
-    result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}",
+    result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOOOs}",
                     type, base,
                     astmodulestate_global->_fields, fnames,
                     astmodulestate_global->__module__,
-                    astmodulestate_global->_ast);
+                    astmodulestate_global->_ast,
+                    astmodulestate_global->__doc__, doc);
     Py_DECREF(fnames);
     return result;
 }
@@ -947,8 +966,9 @@ def visitProduct(self, prod, name):
             fields = name+"_fields"
         else:
             fields = "NULL"
-        self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d);' %
+        self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d,' %
                         (name, name, fields, len(prod.fields)), 1)
+        self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False)
         self.emit("if (!state->%s_type) return 0;" % name, 1)
         self.emit_type("AST_type")
         self.emit_type("%s_type" % name)
@@ -961,8 +981,9 @@ def visitProduct(self, prod, name):
         self.emit_defaults(name, prod.attributes, 1)
 
     def visitSum(self, sum, name):
-        self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0);' %
+        self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0,' %
                   (name, name), 1)
+        self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False)
         self.emit_type("%s_type" % name)
         self.emit("if (!state->%s_type) return 0;" % name, 1)
         if sum.attributes:
@@ -980,8 +1001,9 @@ def visitConstructor(self, cons, name, simple):
             fields = cons.name+"_fields"
         else:
             fields = "NULL"
-        self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d);' %
+        self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d,' %
                             (cons.name, cons.name, name, fields, len(cons.fields)), 1)
+        self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False)
         self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
         self.emit_type("%s_type" % cons.name)
         self.emit_defaults(cons.name, cons.fields, 1)
@@ -1279,8 +1301,15 @@ def generate_module_def(f, mod):
         visitor.visit(mod)
         visitor_list.add(visitor)
 
-    state_strings = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
-    module_state = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
+    state_strings = {
+        "_ast",
+        "_fields",
+        "__doc__",
+        "__dict__",
+        "__module__",
+        "_attributes",
+    }
+    module_state = state_strings.copy()
     for visitor in visitor_list:
         for identifier in visitor.identifiers:
             module_state.add(identifier)