Modify pgen2 to make it produce pgen compatible output

Author: pablogsal

Lib/lib2to3/pgen2/grammar.py

@@ -55,8 +55,7 @@ class Grammar(object):
     dfas          -- a dict mapping symbol numbers to (DFA, first)
                      pairs, where DFA is an item from the states list
                      above, and first is a set of tokens that can
-                     begin this grammar rule (represented by a dict
-                     whose values are always 1).
+                     begin this grammar rule.
 
     labels        -- a list of (x, y) pairs where x is either a token
                      number or a symbol number, and y is either None
@@ -131,53 +130,55 @@ def report(self):
 # Map from operator to number (since tokenize doesn't do this)
 
 opmap_raw = """
-( LPAR
-) RPAR
-[ LSQB
-] RSQB
-: COLON
-, COMMA
-; SEMI
-+ PLUS
-- MINUS
-* STAR
-/ SLASH
-| VBAR
-& AMPER
-< LESS
-> GREATER
-= EQUAL
-. DOT
-% PERCENT
-` BACKQUOTE
-{ LBRACE
-} RBRACE
-@ AT
-@= ATEQUAL
-== EQEQUAL
-!= NOTEQUAL
-<> NOTEQUAL
-<= LESSEQUAL
->= GREATEREQUAL
-~ TILDE
-^ CIRCUMFLEX
-<< LEFTSHIFT
->> RIGHTSHIFT
-** DOUBLESTAR
-+= PLUSEQUAL
--= MINEQUAL
-*= STAREQUAL
-/= SLASHEQUAL
-%= PERCENTEQUAL
-&= AMPEREQUAL
-|= VBAREQUAL
-^= CIRCUMFLEXEQUAL
-<<= LEFTSHIFTEQUAL
->>= RIGHTSHIFTEQUAL
-**= DOUBLESTAREQUAL
-// DOUBLESLASH
-//= DOUBLESLASHEQUAL
--> RARROW
+ ( LPAR
+ ) RPAR
+ [ LSQB
+ ] RSQB
+ : COLON
+ , COMMA
+ ; SEMI
+ + PLUS
+ - MINUS
+ * STAR
+ / SLASH
+ | VBAR
+ & AMPER
+ < LESS
+ > GREATER
+ = EQUAL
+ . DOT
+ % PERCENT
+ { LBRACE
+ } RBRACE
+ == EQEQUAL
+ != NOTEQUAL
+ <> NOTEQUAL
+ <= LESSEQUAL
+ >= GREATEREQUAL
+ ~ TILDE
+ ^ CIRCUMFLEX
+ << LEFTSHIFT
+ >> RIGHTSHIFT
+ ** DOUBLESTAR
+ += PLUSEQUAL
+ -= MINEQUAL
+ *= STAREQUAL
+ /= SLASHEQUAL
+ %= PERCENTEQUAL
+ &= AMPEREQUAL
+ |= VBAREQUAL
+ ^= CIRCUMFLEXEQUAL
+ <<= LEFTSHIFTEQUAL
+ >>= RIGHTSHIFTEQUAL
+ **= DOUBLESTAREQUAL
+ // DOUBLESLASH
+ //= DOUBLESLASHEQUAL
+ @ AT
+ @= ATEQUAL
+ -> RARROW
+ ... ELLIPSIS
+ := COLONEQUAL
+ ` BACKQUOTE
 """
 
 opmap = {}

Lib/lib2to3/pgen2/pgen.py

@@ -3,17 +3,113 @@
 
 # Pgen imports
 from . import grammar, token, tokenize
+import collections
+
 
 class PgenGrammar(grammar.Grammar):
-    pass
+    def produce_graminit_h(self, writer):
+        writer("/* Generated by Parser/pgen2 */\n\n")
+        for number, symbol in self.number2symbol.items():
+            writer("#define {} {}\n".format(symbol, number))
+
+    def produce_graminit_c(self, writer):
+        writer("/* Generated by Parser/pgen2 */\n\n")
+
+        writer('#include "pgenheaders.h"\n')
+        writer('#include "grammar.h"\n')
+        writer("grammar _PyParser_Grammar;\n")
+
+        self.print_dfas(writer)
+        self.print_labels(writer)
+
+        writer("grammar _PyParser_Grammar = {\n")
+        writer("    {n_dfas},\n".format(n_dfas=len(self.dfas)))
+        writer("    dfas,\n")
+        writer("    {{{n_labels}, labels}},\n".format(n_labels=len(self.labels)))
+        writer("    {start_number}\n".format(start_number=self.start))
+        writer("};\n")
+
+    def print_labels(self, writer):
+        writer(
+            "static label labels[{n_labels}] = {{\n".format(n_labels=len(self.labels))
+        )
+        for label, name in self.labels:
+            if name is None:
+                writer("    {{{label}, 0}},\n".format(label=label))
+            else:
+                writer(
+                    '    {{{label}, "{label_name}"}},\n'.format(
+                        label=label, label_name=name
+                    )
+                )
+        writer("};\n")
+
+    def print_dfas(self, writer):
+        self.print_states(writer)
+        writer("static dfa dfas[{}] = {{\n".format(len(self.dfas)))
+        for dfaindex, dfa_elem in enumerate(self.dfas.items()):
+            symbol, (dfa, first_sets) = dfa_elem
+            writer(
+                '    {{{dfa_symbol}, "{symbol_name}", '.format(
+                    dfa_symbol=symbol, symbol_name=self.number2symbol[symbol]
+                )
+                + "0, {n_states}, states_{dfa_index},\n".format(
+                    n_states=len(dfa), dfa_index=dfaindex
+                )
+            )
+            writer('     "')
+
+            k = [name for label, name in self.labels if label in first_sets]
+            bitset = bytearray((len(self.labels) >> 3) + 1)
+            for token in first_sets:
+                bitset[token >> 3] |= 1 << (token & 7)
+            for byte in bitset:
+                writer("\\%03o" % (byte & 0xFF))
+            writer('"},\n')
+        writer("};\n")
+
+    def print_states(self, write):
+        for dfaindex, dfa in enumerate(self.states):
+            self.print_arcs(write, dfaindex, dfa)
+            write(
+                "static state states_{dfa_index}[{n_states}] = {{\n".format(
+                    dfa_index=dfaindex, n_states=len(dfa)
+                )
+            )
+            for stateindex, state in enumerate(dfa):
+                narcs = len(state)
+                write(
+                    "    {{{n_arcs}, arcs_{dfa_index}_{state_index}}},\n".format(
+                        n_arcs=narcs, dfa_index=dfaindex, state_index=stateindex
+                    )
+                )
+            write("};\n")
+
+    def print_arcs(self, write, dfaindex, states):
+        for stateindex, state in enumerate(states):
+            narcs = len(state)
+            write(
+                "static arc arcs_{dfa_index}_{state_index}[{n_arcs}] = {{\n".format(
+                    dfa_index=dfaindex, state_index=stateindex, n_arcs=narcs
+                )
+            )
+            for a, b in state:
+                write(
+                    "    {{{from_label}, {to_state}}},\n".format(
+                        from_label=a, to_state=b
+                    )
+                )
+            write("};\n")
+
 
 class ParserGenerator(object):
 
-    def __init__(self, filename, stream=None):
+    def __init__(self, filename, stream=None, verbose=False):
         close_stream = None
         if stream is None:
             stream = open(filename)
             close_stream = stream.close
+        self.verbose = verbose
         self.filename = filename
         self.stream = stream
         self.generator = tokenize.generate_tokens(stream.readline)
@@ -27,14 +123,14 @@ def __init__(self, filename, stream=None):
     def make_grammar(self):
         c = PgenGrammar()
         names = list(self.dfas.keys())
-        names.sort()
         names.remove(self.startsymbol)
         names.insert(0, self.startsymbol)
         for name in names:
             i = 256 + len(c.symbol2number)
             c.symbol2number[name] = i
             c.number2symbol[i] = name
         for name in names:
+            self.make_label(c, name)
             dfa = self.dfas[name]
             states = []
             for state in dfa:
@@ -47,15 +143,30 @@ def make_grammar(self):
             c.states.append(states)
             c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
         c.start = c.symbol2number[self.startsymbol]
+
+        if self.verbose:
+            print("")
+            print("Grammar summary")
+            print("===============")
+
+            print("- {n_labels} labels".format(n_labels=len(c.labels)))
+            print("- {n_dfas} dfas".format(n_dfas=len(c.dfas)))
+            print("- {n_tokens} tokens".format(n_tokens=len(c.tokens)))
+            print("- {n_keywords} keywords".format(n_keywords=len(c.keywords)))
+            print(
+                "- Start symbol: {start_symbol}".format(
+                    start_symbol=c.number2symbol[c.start]
+                )
+            )
         return c
 
     def make_first(self, c, name):
         rawfirst = self.first[name]
-        first = {}
+        first = set()
         for label in sorted(rawfirst):
             ilabel = self.make_label(c, label)
             ##assert ilabel not in first # XXX failed on <> ... !=
-            first[ilabel] = 1
+            first.add(ilabel)
         return first
 
     def make_label(self, c, label):
@@ -106,17 +217,20 @@ def make_label(self, c, label):
 
     def addfirstsets(self):
         names = list(self.dfas.keys())
-        names.sort()
         for name in names:
             if name not in self.first:
                 self.calcfirst(name)
-            #print name, self.first[name].keys()
+
+            if self.verbose:
+                print("First set for {dfa_name}".format(dfa_name=name))
+                for item in self.first[name]:
+                    print("    - {terminal}".format(terminal=item))
 
     def calcfirst(self, name):
         dfa = self.dfas[name]
         self.first[name] = None # dummy to detect left recursion
         state = dfa[0]
-        totalset = {}
+        totalset = set()
         overlapcheck = {}
         for label, next in state.arcs.items():
             if label in self.dfas:
@@ -130,8 +244,8 @@ def calcfirst(self, name):
                 totalset.update(fset)
                 overlapcheck[label] = fset
             else:
-                totalset[label] = 1
-                overlapcheck[label] = {label: 1}
+                totalset.add(label)
+                overlapcheck[label] = {label}
         inverse = {}
         for label, itsfirst in overlapcheck.items():
             for symbol in itsfirst:
@@ -143,20 +257,24 @@ def calcfirst(self, name):
         self.first[name] = totalset
 
     def parse(self):
-        dfas = {}
+        dfas = collections.OrderedDict()
         startsymbol = None
         # MSTART: (NEWLINE | RULE)* ENDMARKER
         while self.type != token.ENDMARKER:
             while self.type == token.NEWLINE:
                 self.gettoken()
             # RULE: NAME ':' RHS NEWLINE
             name = self.expect(token.NAME)
+            if self.verbose:
+                print("Processing rule {dfa_name}".format(dfa_name=name))
             self.expect(token.OP, ":")
             a, z = self.parse_rhs()
             self.expect(token.NEWLINE)
-            #self.dump_nfa(name, a, z)
+            if self.verbose:
+                self.dump_nfa(name, a, z)
             dfa = self.make_dfa(a, z)
-            #self.dump_dfa(name, dfa)
+            if self.verbose:
+                self.dump_dfa(name, dfa)
             oldlen = len(dfa)
             self.simplify_dfa(dfa)
             newlen = len(dfa)
@@ -174,14 +292,14 @@ def make_dfa(self, start, finish):
         assert isinstance(start, NFAState)
         assert isinstance(finish, NFAState)
         def closure(state):
-            base = {}
+            base = set()
             addclosure(state, base)
             return base
         def addclosure(state, base):
             assert isinstance(state, NFAState)
             if state in base:
                 return
-            base[state] = 1
+            base.add(state)
             for label, next in state.arcs:
                 if label is None:
                     addclosure(next, base)
@@ -191,7 +309,7 @@ def addclosure(state, base):
             for nfastate in state.nfaset:
                 for label, next in nfastate.arcs:
                     if label is not None:
-                        addclosure(next, arcs.setdefault(label, {}))
+                        addclosure(next, arcs.setdefault(label, set()))
             for label, nfaset in sorted(arcs.items()):
                 for st in states:
                     if st.nfaset == nfaset:
@@ -347,7 +465,7 @@ def addarc(self, next, label=None):
 class DFAState(object):
 
     def __init__(self, nfaset, final):
-        assert isinstance(nfaset, dict)
+        assert isinstance(nfaset, set)
         assert isinstance(next(iter(nfaset)), NFAState)
         assert isinstance(final, NFAState)
         self.nfaset = nfaset
@@ -381,6 +499,6 @@ def __eq__(self, other):
 
     __hash__ = None # For Py3 compatibility.
 
-def generate_grammar(filename="Grammar.txt"):
-    p = ParserGenerator(filename)
+def generate_grammar(filename="Grammar.txt", verbose=False):
+    p = ParserGenerator(filename, verbose=verbose)
     return p.make_grammar()