[WIP] bpo-17611: Move unwinding of stack from interpreter to compiler

Include/opcode.h

@@ -12,6 +12,7 @@ extern "C" {
 #define ROT_THREE                 3
 #define DUP_TOP                   4
 #define DUP_TOP_TWO               5
+#define ROT_FOUR                  6
 #define NOP                       9
 #define UNARY_POSITIVE           10
 #define UNARY_NEGATIVE           11
@@ -29,9 +30,12 @@ extern "C" {
 #define BINARY_TRUE_DIVIDE       27
 #define INPLACE_FLOOR_DIVIDE     28
 #define INPLACE_TRUE_DIVIDE      29
+#define WITH_CLEANUP_START       40
+#define WITH_CLEANUP_FINISH      41
 #define GET_AITER                50
 #define GET_ANEXT                51
 #define BEFORE_ASYNC_WITH        52
+#define ENTER_WITH               53
 #define INPLACE_ADD              55
 #define INPLACE_SUBTRACT         56
 #define INPLACE_MULTIPLY         57
@@ -55,15 +59,13 @@ extern "C" {
 #define INPLACE_AND              77
 #define INPLACE_XOR              78
 #define INPLACE_OR               79
-#define BREAK_LOOP               80
-#define WITH_CLEANUP_START       81
-#define WITH_CLEANUP_FINISH      82
+#define RERAISE                  80
 #define RETURN_VALUE             83
 #define IMPORT_STAR              84
 #define SETUP_ANNOTATIONS        85
 #define YIELD_VALUE              86
 #define POP_BLOCK                87
-#define END_FINALLY              88
+#define PUSH_NO_EXCEPT           88
 #define POP_EXCEPT               89
 #define HAVE_ARGUMENT            90
 #define STORE_NAME               90
@@ -92,8 +94,6 @@ extern "C" {
 #define POP_JUMP_IF_FALSE       114
 #define POP_JUMP_IF_TRUE        115
 #define LOAD_GLOBAL             116
-#define CONTINUE_LOOP           119
-#define SETUP_LOOP              120
 #define SETUP_EXCEPT            121
 #define SETUP_FINALLY           122
 #define LOAD_FAST               124
@@ -110,7 +110,6 @@ extern "C" {
 #define DELETE_DEREF            138
 #define CALL_FUNCTION_KW        141
 #define CALL_FUNCTION_EX        142
-#define SETUP_WITH              143
 #define EXTENDED_ARG            144
 #define LIST_APPEND             145
 #define SET_ADD                 146
@@ -121,7 +120,6 @@ extern "C" {
 #define BUILD_MAP_UNPACK_WITH_CALL 151
 #define BUILD_TUPLE_UNPACK      152
 #define BUILD_SET_UNPACK        153
-#define SETUP_ASYNC_WITH        154
 #define FORMAT_VALUE            155
 #define BUILD_CONST_KEY_MAP     156
 #define BUILD_STRING            157

Lib/importlib/_bootstrap_external.py

@@ -241,6 +241,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.6b2  3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257)
 #     Python 3.6rc1 3379 (more thorough __class__ validation #23722)
 #     Python 3.7a0  3390 (add LOAD_METHOD and CALL_METHOD opcodes)
+#     Python 3.7a0  3391 (bpo-17611: move frame block handling to compiler)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -249,7 +250,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3390).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3391).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/opcode.py

@@ -60,6 +60,7 @@ def jabs_op(name, op):
 def_op('ROT_THREE', 3)
 def_op('DUP_TOP', 4)
 def_op('DUP_TOP_TWO', 5)
+def_op('ROT_FOUR', 6)
 
 def_op('NOP', 9)
 def_op('UNARY_POSITIVE', 10)
@@ -83,9 +84,13 @@ def jabs_op(name, op):
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
+def_op('WITH_CLEANUP_START', 40)
+def_op('WITH_CLEANUP_FINISH', 41)
+
 def_op('GET_AITER', 50)
 def_op('GET_ANEXT', 51)
 def_op('BEFORE_ASYNC_WITH', 52)
+def_op('ENTER_WITH', 53)
 
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
@@ -113,16 +118,14 @@ def jabs_op(name, op):
 def_op('INPLACE_AND', 77)
 def_op('INPLACE_XOR', 78)
 def_op('INPLACE_OR', 79)
-def_op('BREAK_LOOP', 80)
-def_op('WITH_CLEANUP_START', 81)
-def_op('WITH_CLEANUP_FINISH', 82)
+def_op('RERAISE', 80)
 
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
 def_op('SETUP_ANNOTATIONS', 85)
 def_op('YIELD_VALUE', 86)
 def_op('POP_BLOCK', 87)
-def_op('END_FINALLY', 88)
+def_op('PUSH_NO_EXCEPT', 88)
 def_op('POP_EXCEPT', 89)
 
 HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
@@ -158,9 +161,7 @@ def jabs_op(name, op):
 
 name_op('LOAD_GLOBAL', 116)     # Index in name list
 
-jabs_op('CONTINUE_LOOP', 119)   # Target address
-jrel_op('SETUP_LOOP', 120)      # Distance to target address
-jrel_op('SETUP_EXCEPT', 121)    # ""
+jrel_op('SETUP_EXCEPT', 121)    # Distance to target address
 jrel_op('SETUP_FINALLY', 122)   # ""
 
 def_op('LOAD_FAST', 124)        # Local variable number
@@ -187,7 +188,8 @@ def jabs_op(name, op):
 def_op('CALL_FUNCTION_KW', 141)  # #args + #kwargs
 def_op('CALL_FUNCTION_EX', 142)  # Flags
 
-jrel_op('SETUP_WITH', 143)
+def_op('EXTENDED_ARG', 144)
+EXTENDED_ARG = 144
 
 def_op('LIST_APPEND', 145)
 def_op('SET_ADD', 146)
@@ -196,17 +198,12 @@ def jabs_op(name, op):
 def_op('LOAD_CLASSDEREF', 148)
 hasfree.append(148)
 
-def_op('EXTENDED_ARG', 144)
-EXTENDED_ARG = 144
-
 def_op('BUILD_LIST_UNPACK', 149)
 def_op('BUILD_MAP_UNPACK', 150)
 def_op('BUILD_MAP_UNPACK_WITH_CALL', 151)
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
-jrel_op('SETUP_ASYNC_WITH', 154)
-
 def_op('FORMAT_VALUE', 155)
 def_op('BUILD_CONST_KEY_MAP', 156)
 def_op('BUILD_STRING', 157)

Lib/test/test_compile.py

@@ -671,7 +671,7 @@ def __fspath__(self):
         compile("42", PathLike("test_compile_pathlike"), "single")
 
 
-class TestStackSize(unittest.TestCase):
+class TestExpressionStackSize(unittest.TestCase):
     # These tests check that the computed stack size for a code object
     # stays within reasonable bounds (see issue #21523 for an example
     # dysfunction).
@@ -709,5 +709,102 @@ def test_func_and(self):
         self.check_stack_size(code)
 
 
+class TestStackSizeStability(unittest.TestCase):
+    # Check that repeating certain snippets doesn't increase the stack size
+    # beyond what a single snippet requires.
+
+    def check_stack_size(self, snippet):
+        sizes = []
+        for i in range(2, 5):
+            ns = {}
+            code = """def f():\n""" + i * snippet
+            code = compile(code, "<foo>", "exec")
+            exec(code, ns, ns)
+            sizes.append(ns['f'].__code__.co_stacksize)
+
+        self.assertEqual(len(set(sizes)), 1,
+                         "stack sizes diverge with # of consecutive snippets: %s" % (sizes,))
+
+    def test_if_else(self):
+        snippet = """
+            if x:
+                a
+            elif y:
+                b
+            else:
+                c
+            """
+        self.check_stack_size(snippet)
+
+    def test_try_except_bare(self):
+        snippet = """
+            try:
+                a
+            except:
+                b
+            """
+        self.check_stack_size(snippet)
+
+    def test_try_except_qualified(self):
+        snippet = """
+            try:
+                a
+            except ImportError:
+                b
+            except:
+                c
+            else:
+                d
+            """
+        self.check_stack_size(snippet)
+
+    # This one unfortunately "leaks" a few stack slots for each snippet
+    @unittest.expectedFailure
+    def test_try_except_as(self):
+        snippet = """
+            try:
+                a
+            except ImportError as e:
+                b
+            except:
+                c
+            else:
+                d
+            """
+        self.check_stack_size(snippet)
+
+    def test_try_finally(self):
+        snippet = """
+            try:
+                a
+            finally:
+                b
+            """
+        self.check_stack_size(snippet)
+
+    def test_for_else(self):
+        snippet = """
+            for x in y:
+                a
+            else:
+                b
+            """
+        self.check_stack_size(snippet)
+
+    def test_for_break_continue(self):
+        snippet = """
+            for x in y:
+                if z:
+                    break
+                elif u:
+                    continue
+                else:
+                    a
+            else:
+                b
+            """
+        self.check_stack_size(snippet)
+
+
 if __name__ == "__main__":
     unittest.main()