Skip to content

bpo-40528: move asdl identifier collection to the new metadata system #26858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 24, 2021
Merged

bpo-40528: move asdl identifier collection to the new metadata system #26858

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 51 additions & 49 deletions Parser/asdl_c.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,21 +103,9 @@ class EmitVisitor(asdl.VisitorBase):

def __init__(self, file, metadata = None):
self.file = file
self.identifiers = set()
self.singletons = set()
self.types = set()
self._metadata = metadata
super(EmitVisitor, self).__init__()

def emit_identifier(self, name):
self.identifiers.add(str(name))

def emit_singleton(self, name):
self.singletons.add(str(name))

def emit_type(self, name):
self.types.add(str(name))

def emit(self, s, depth, reflow=True):
# XXX reflow long lines?
if reflow:
Expand All @@ -143,6 +131,8 @@ def metadata(self, value):
self._metadata = value

class MetadataVisitor(asdl.VisitorBase):
ROOT_TYPE = "AST"

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)

Expand All @@ -151,8 +141,16 @@ def __init__(self, *args, **kwargs):
# names where all the constructors
# belonging to that type lack of any
# fields.
# - identifiers: All identifiers used in the AST decclarations
# - singletons: List of all constructors that originates from
# simple sums.
# - types: List of all top level type names
#
self.metadata = types.SimpleNamespace(
simple_sums=set()
simple_sums=set(),
identifiers=set(),
singletons=set(),
types={self.ROOT_TYPE},
)

def visitModule(self, mod):
Expand All @@ -163,9 +161,35 @@ def visitType(self, type):
self.visit(type.value, type.name)

def visitSum(self, sum, name):
if is_simple(sum):
self.metadata.types.add(name)

simple_sum = is_simple(sum)
if simple_sum:
self.metadata.simple_sums.add(name)

for constructor in sum.types:
if simple_sum:
self.metadata.singletons.add(constructor.name)
self.visitConstructor(constructor)
self.visitFields(sum.attributes)

def visitConstructor(self, constructor):
self.metadata.types.add(constructor.name)
self.visitFields(constructor.fields)

def visitProduct(self, product, name):
self.metadata.types.add(name)
self.visitFields(product.attributes)
self.visitFields(product.fields)

def visitFields(self, fields):
for field in fields:
self.visitField(field)

def visitField(self, field):
self.metadata.identifiers.add(field.name)


class TypeDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
Expand Down Expand Up @@ -683,45 +707,33 @@ def emit_sequence_constructor(self, name, type):
class PyTypesDeclareVisitor(PickleVisitor):

def visitProduct(self, prod, name):
self.emit_type("%s_type" % name)
self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0)
if prod.attributes:
for a in prod.attributes:
self.emit_identifier(a.name)
self.emit("static const char * const %s_attributes[] = {" % name, 0)
for a in prod.attributes:
self.emit('"%s",' % a.name, 1)
self.emit("};", 0)
if prod.fields:
for f in prod.fields:
self.emit_identifier(f.name)
self.emit("static const char * const %s_fields[]={" % name,0)
for f in prod.fields:
self.emit('"%s",' % f.name, 1)
self.emit("};", 0)

def visitSum(self, sum, name):
self.emit_type("%s_type" % name)
if sum.attributes:
for a in sum.attributes:
self.emit_identifier(a.name)
self.emit("static const char * const %s_attributes[] = {" % name, 0)
for a in sum.attributes:
self.emit('"%s",' % a.name, 1)
self.emit("};", 0)
ptype = "void*"
if is_simple(sum):
ptype = get_c_type(name)
for t in sum.types:
self.emit_singleton("%s_singleton" % t.name)
self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0)
for t in sum.types:
self.visitConstructor(t, name)

def visitConstructor(self, cons, name):
if cons.fields:
for t in cons.fields:
self.emit_identifier(t.name)
self.emit("static const char * const %s_fields[]={" % cons.name, 0)
for t in cons.fields:
self.emit('"%s",' % t.name, 1)
Expand Down Expand Up @@ -1099,8 +1111,6 @@ def visitProduct(self, prod, name):
(name, name, fields, len(prod.fields)), 1)
self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False)
self.emit("if (!state->%s_type) return 0;" % name, 1)
self.emit_type("AST_type")
self.emit_type("%s_type" % name)
if prod.attributes:
self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" %
(name, name, len(prod.attributes)), 1)
Expand All @@ -1113,7 +1123,6 @@ def visitSum(self, sum, name):
self.emit('state->%s_type = make_type(state, "%s", state->AST_type, NULL, 0,' %
(name, name), 1)
self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False)
self.emit_type("%s_type" % name)
self.emit("if (!state->%s_type) return 0;" % name, 1)
if sum.attributes:
self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" %
Expand All @@ -1134,7 +1143,6 @@ def visitConstructor(self, cons, name, simple):
(cons.name, cons.name, name, fields, len(cons.fields)), 1)
self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False)
self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
self.emit_type("%s_type" % cons.name)
self.emit_defaults(cons.name, cons.fields, 1)
if simple:
self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)"
Expand Down Expand Up @@ -1439,17 +1447,8 @@ def generate_ast_fini(module_state, f):
"""))


def generate_module_def(mod, f, internal_h):
def generate_module_def(mod, metadata, f, internal_h):
# Gather all the data needed for ModuleSpec
visitor_list = set()
with open(os.devnull, "w") as devnull:
visitor = PyTypesDeclareVisitor(devnull)
visitor.visit(mod)
visitor_list.add(visitor)
visitor = PyTypesVisitor(devnull)
visitor.visit(mod)
visitor_list.add(visitor)

state_strings = {
"ast",
"_fields",
Expand All @@ -1458,16 +1457,19 @@ def generate_module_def(mod, f, internal_h):
"__dict__",
"__module__",
"_attributes",
*metadata.identifiers
}

module_state = state_strings.copy()
for visitor in visitor_list:
for identifier in visitor.identifiers:
module_state.add(identifier)
state_strings.add(identifier)
for singleton in visitor.singletons:
module_state.add(singleton)
for tp in visitor.types:
module_state.add(tp)
module_state.update(
"%s_singleton" % singleton
for singleton in metadata.singletons
)
module_state.update(
"%s_type" % type
for type in metadata.types
)

state_strings = sorted(state_strings)
module_state = sorted(module_state)

Expand Down Expand Up @@ -1583,7 +1585,7 @@ def write_internal_h_footer(mod, f):
"""), file=f)

def write_source(mod, metadata, f, internal_h_file):
generate_module_def(mod, f, internal_h_file)
generate_module_def(mod, metadata, f, internal_h_file)

v = ChainOfVisitors(
SequenceConstructorVisitor(f),
Expand Down