Use a monkey-patched version of pgen2 in Parser/pgen

Author: pablogsal

Parser/pgen/__main__.py

@@ -1,18 +1,87 @@
 import os
 import sys
 import argparse
+import collections
+
+from lib2to3.pgen2 import pgen, grammar, tokenize
+
+from . import token
+from . import grammar as pgen_grammar
+
+for name in dir(token):
+    setattr(tokenize, name, getattr(token, name))
+
+pgen.token = token
+pgen.grammar = pgen_grammar
+
+class ParserGenerator(pgen.ParserGenerator):
+    def parse(self):
+        dfas = collections.OrderedDict()
+        startsymbol = None
+        # MSTART: (NEWLINE | RULE)* ENDMARKER
+        while self.type != token.ENDMARKER:
+            while self.type == token.NEWLINE:
+                self.gettoken()
+            # RULE: NAME ':' RHS NEWLINE
+            name = self.expect(token.NAME)
+            self.expect(token.OP, ":")
+            a, z = self.parse_rhs()
+            self.expect(token.NEWLINE)
+            dfa = self.make_dfa(a, z)
+            oldlen = len(dfa)
+            self.simplify_dfa(dfa)
+            newlen = len(dfa)
+            dfas[name] = dfa
+            #print name, oldlen, newlen
+            if startsymbol is None:
+                startsymbol = name
+        return dfas, startsymbol
+
+    def make_grammar(self, verbose=False):
+        c = pgen_grammar.Grammar()
+        names = list(self.dfas.keys())
+        names.remove(self.startsymbol)
+        names.insert(0, self.startsymbol)
+        for name in names:
+            i = 256 + len(c.symbol2number)
+            c.symbol2number[name] = i
+            c.number2symbol[i] = name
+        for name in names:
+            self.make_label(c, name)
+            dfa = self.dfas[name]
+            states = []
+            for state in dfa:
+                arcs = []
+                for label, next in sorted(state.arcs.items()):
+                    arcs.append((self.make_label(c, label), dfa.index(next)))
+                if state.isfinal:
+                    arcs.append((0, dfa.index(state)))
+                states.append(arcs)
+            c.states.append(states)
+            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
+        c.start = c.symbol2number[self.startsymbol]
+
+        if verbose:
+            print("")
+            print("Grammar summary")
+            print("===============")
+
+            print("- {n_labels} labels".format(n_labels=len(c.labels)))
+            print("- {n_dfas} dfas".format(n_dfas=len(c.dfas)))
+            print("- {n_tokens} tokens".format(n_tokens=len(c.tokens)))
+            print("- {n_keywords} keywords".format(n_keywords=len(c.keywords)))
+            print(
+                "- Start symbol: {start_symbol}".format(
+                    start_symbol=c.number2symbol[c.start]
+                )
+            )
+        return c
 
-# Make sure we pick the current version of pgen2 (and not whatever is installed)
-CURRENT_FOLDER_LOCATION = os.path.dirname(os.path.realpath(__file__))
-LIB2TO3LOCATION = os.path.realpath(os.path.join(CURRENT_FOLDER_LOCATION,
-                                                '..', '..', 'Lib', 'lib2to3'))
 
-sys.path.insert(0, LIB2TO3LOCATION)
-from pgen2 import pgen
-sys.path.pop(0)
 
 def main(grammar_file, gramminit_h_file, gramminit_c_file, verbose):
-    grammar = pgen.generate_grammar(grammar_file, verbose=verbose)
+    p = ParserGenerator(grammar_file)
+    grammar = p.make_grammar(verbose=verbose)
     grammar.produce_graminit_h(gramminit_h_file.write)
     grammar.produce_graminit_c(gramminit_c_file.write)
 

Parser/pgen/grammar.py

@@ -0,0 +1,161 @@
+from lib2to3.pgen2 import grammar
+
+from . import token
+
+
+class Grammar(grammar.Grammar):
+
+    def produce_graminit_h(self, writer):
+        writer("/* Generated by Parser/pgen2 */\n\n")
+        for number, symbol in self.number2symbol.items():
+            writer("#define {} {}\n".format(symbol, number))
+
+    def produce_graminit_c(self, writer):
+       writer("/* Generated by Parser/pgen2 */\n\n")
+
+       writer('#include "pgenheaders.h"\n')
+       writer('#include "grammar.h"\n')
+       writer("grammar _PyParser_Grammar;\n")
+
+       self.print_dfas(writer)
+       self.print_labels(writer)
+
+       writer("grammar _PyParser_Grammar = {\n")
+       writer("    {n_dfas},\n".format(n_dfas=len(self.dfas)))
+       writer("    dfas,\n")
+       writer("    {{{n_labels}, labels}},\n".format(n_labels=len(self.labels)))
+       writer("    {start_number}\n".format(start_number=self.start))
+       writer("};\n")
+
+    def print_labels(self, writer):
+       writer(
+           "static label labels[{n_labels}] = {{\n".format(n_labels=len(self.labels))
+       )
+       for label, name in self.labels:
+           if name is None:
+               writer("    {{{label}, 0}},\n".format(label=label))
+           else:
+               writer(
+                   '    {{{label}, "{label_name}"}},\n'.format(
+                       label=label, label_name=name
+                   )
+               )
+       writer("};\n")
+
+    def print_dfas(self, writer):
+       self.print_states(writer)
+       writer("static dfa dfas[{}] = {{\n".format(len(self.dfas)))
+       for dfaindex, dfa_elem in enumerate(self.dfas.items()):
+           symbol, (dfa, first_sets) = dfa_elem
+           writer(
+               '    {{{dfa_symbol}, "{symbol_name}", '.format(
+                   dfa_symbol=symbol, symbol_name=self.number2symbol[symbol]
+               )
+               + "0, {n_states}, states_{dfa_index},\n".format(
+                   n_states=len(dfa), dfa_index=dfaindex
+               )
+           )
+           writer('     "')
+
+           k = [name for label, name in self.labels if label in first_sets]
+           bitset = bytearray((len(self.labels) >> 3) + 1)
+           for token in first_sets:
+               bitset[token >> 3] |= 1 << (token & 7)
+           for byte in bitset:
+               writer("\\%03o" % (byte & 0xFF))
+           writer('"},\n')
+       writer("};\n")
+
+    def print_states(self, write):
+       for dfaindex, dfa in enumerate(self.states):
+           self.print_arcs(write, dfaindex, dfa)
+           write(
+               "static state states_{dfa_index}[{n_states}] = {{\n".format(
+                   dfa_index=dfaindex, n_states=len(dfa)
+               )
+           )
+           for stateindex, state in enumerate(dfa):
+               narcs = len(state)
+               write(
+                   "    {{{n_arcs}, arcs_{dfa_index}_{state_index}}},\n".format(
+                       n_arcs=narcs, dfa_index=dfaindex, state_index=stateindex
+                   )
+               )
+           write("};\n")
+
+    def print_arcs(self, write, dfaindex, states):
+       for stateindex, state in enumerate(states):
+           narcs = len(state)
+           write(
+               "static arc arcs_{dfa_index}_{state_index}[{n_arcs}] = {{\n".format(
+                   dfa_index=dfaindex, state_index=stateindex, n_arcs=narcs
+               )
+           )
+           for a, b in state:
+               write(
+                   "    {{{from_label}, {to_state}}},\n".format(
+                       from_label=a, to_state=b
+                   )
+               )
+           write("};\n")
+
+
+
+
+opmap_raw = """
+( LPAR
+) RPAR
+[ LSQB
+] RSQB
+: COLON
+, COMMA
+; SEMI
++ PLUS
+- MINUS
+* STAR
+/ SLASH
+| VBAR
+& AMPER
+< LESS
+> GREATER
+= EQUAL
+. DOT
+% PERCENT
+{ LBRACE
+} RBRACE
+== EQEQUAL
+!= NOTEQUAL
+<> NOTEQUAL
+<= LESSEQUAL
+>= GREATEREQUAL
+~ TILDE
+^ CIRCUMFLEX
+<< LEFTSHIFT
+>> RIGHTSHIFT
+** DOUBLESTAR
++= PLUSEQUAL
+-= MINEQUAL
+*= STAREQUAL
+/= SLASHEQUAL
+%= PERCENTEQUAL
+&= AMPEREQUAL
+|= VBAREQUAL
+^= CIRCUMFLEXEQUAL
+<<= LEFTSHIFTEQUAL
+>>= RIGHTSHIFTEQUAL
+**= DOUBLESTAREQUAL
+// DOUBLESLASH
+//= DOUBLESLASHEQUAL
+@ AT
+@= ATEQUAL
+-> RARROW
+... ELLIPSIS
+:= COLONEQUAL
+` BACKQUOTE
+"""
+
+opmap = {}
+for line in opmap_raw.splitlines():
+    if line:
+        op, name = line.split()
+        opmap[op] = getattr(token, name)

Parser/pgen/token.py

@@ -0,0 +1,92 @@
+#! /usr/bin/env python3
+
+"""Token constants (from "token.h")."""
+
+#  Taken from Python (r53757) and modified to include some tokens
+#   originally monkeypatched in by pgen2.tokenize
+
+# --start constants--
+ENDMARKER = 0
+NAME = 1
+NUMBER = 2
+STRING = 3
+NEWLINE = 4
+INDENT = 5
+DEDENT = 6
+LPAR = 7
+RPAR = 8
+LSQB = 9
+RSQB = 10
+COLON = 11
+COMMA = 12
+SEMI = 13
+PLUS = 14
+MINUS = 15
+STAR = 16
+SLASH = 17
+VBAR = 18
+AMPER = 19
+LESS = 20
+GREATER = 21
+EQUAL = 22
+DOT = 23
+PERCENT = 24
+LBRACE = 25
+RBRACE = 26
+EQEQUAL = 27
+NOTEQUAL = 28
+LESSEQUAL = 29
+GREATEREQUAL = 30
+TILDE = 31
+CIRCUMFLEX = 32
+LEFTSHIFT = 33
+RIGHTSHIFT = 34
+DOUBLESTAR = 35
+PLUSEQUAL = 36
+MINEQUAL = 37
+STAREQUAL = 38
+SLASHEQUAL = 39
+PERCENTEQUAL = 40
+AMPEREQUAL = 41
+VBAREQUAL = 42
+CIRCUMFLEXEQUAL = 43
+LEFTSHIFTEQUAL = 44
+RIGHTSHIFTEQUAL = 45
+DOUBLESTAREQUAL = 46
+DOUBLESLASH = 47
+DOUBLESLASHEQUAL = 48
+AT = 49
+ATEQUAL = 50
+RARROW = 51
+ELLIPSIS = 52
+COLONEQUAL = 53
+OP = 54
+TYPE_IGNORE = 55
+TYPE_COMMENT = 56
+ERRORTOKEN = 57
+COMMENT = 58
+NL = 59
+ENCODING = 60
+ASYNC = 61
+AWAIT = 62
+BACKQUOTE = 63
+N_TOKENS = 64
+NT_OFFSET = 256
+# --end constants--
+
+tok_name = {}
+for _name, _value in list(globals().items()):
+    if type(_value) is type(0):
+        tok_name[_value] = _name
+
+
+def ISTERMINAL(x):
+    return x < NT_OFFSET
+
+
+def ISNONTERMINAL(x):
+    return x >= NT_OFFSET
+
+
+def ISEOF(x):
+    return x == ENDMARKER