Merge branch 'main' into func-watchers

Author: mpage

Doc/library/index.rst

@@ -27,8 +27,8 @@ as a collection of packages, so it may be necessary to use the packaging
 tools provided with the operating system to obtain some or all of the
 optional components.
 
-In addition to the standard library, there is a growing collection of
-several thousand components (from individual programs and modules to
+In addition to the standard library, there is an active collection of
+hundreds of thousands of components (from individual programs and modules to
 packages and entire application development frameworks), available from
 the `Python Package Index <https://pypi.org>`_.
 

Include/internal/pycore_code.h

@@ -91,8 +91,6 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
 
-extern uint8_t _PyOpcode_Adaptive[256];
-
 // Borrowed references to common callables:
 struct callable_cache {
     PyObject *isinstance;
@@ -219,11 +217,14 @@ extern int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr,
                                    PyObject *name);
 extern int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr,
                                     PyObject *name);
-extern int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name);
-extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
-extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr);
-extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr,
-                               int nargs, PyObject *kwnames);
+extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins,
+                                      _Py_CODEUNIT *instr, PyObject *name);
+extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container,
+                                        _Py_CODEUNIT *instr);
+extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub,
+                                       _Py_CODEUNIT *instr);
+extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr,
+                                int nargs, PyObject *kwnames);
 extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                                     int oparg, PyObject **locals);
 extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
@@ -377,8 +378,22 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 
 /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
 #define ADAPTIVE_BACKOFF_BITS 4
-/* The initial counter value is 1 == 2**ADAPTIVE_BACKOFF_START - 1 */
-#define ADAPTIVE_BACKOFF_START 1
+
+// A value of 1 means that we attempt to specialize the *second* time each
+// instruction is executed. Executing twice is a much better indicator of
+// "hotness" than executing once, but additional warmup delays only prevent
+// specialization. Most types stabilize by the second execution, too:
+#define ADAPTIVE_WARMUP_VALUE 1
+#define ADAPTIVE_WARMUP_BACKOFF 1
+
+// A value of 52 means that we attempt to re-specialize after 53 misses (a prime
+// number, useful for avoiding artifacts if every nth value is a different type
+// or something). Setting the backoff to 0 means that the counter is reset to
+// the same state as a warming-up instruction (value == 1, backoff == 1) after
+// deoptimization. This isn't strictly necessary, but it is bit easier to reason
+// about when thinking about the opcode transitions as a state machine:
+#define ADAPTIVE_COOLDOWN_VALUE 52
+#define ADAPTIVE_COOLDOWN_BACKOFF 0
 
 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)
 
@@ -390,9 +405,15 @@ adaptive_counter_bits(int value, int backoff) {
 }
 
 static inline uint16_t
-adaptive_counter_start(void) {
-    unsigned int value = (1 << ADAPTIVE_BACKOFF_START) - 1;
-    return adaptive_counter_bits(value, ADAPTIVE_BACKOFF_START);
+adaptive_counter_warmup(void) {
+    return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE,
+                                 ADAPTIVE_WARMUP_BACKOFF);
+}
+
+static inline uint16_t
+adaptive_counter_cooldown(void) {
+    return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE,
+                                 ADAPTIVE_COOLDOWN_BACKOFF);
 }
 
 static inline uint16_t
@@ -435,6 +456,16 @@ _PyCode_LineNumberFromArray(PyCodeObject *co, int index)
     }
 }
 
+typedef struct _PyShimCodeDef {
+    const uint8_t *code;
+    int codelen;
+    int stacksize;
+    const char *cname;
+} _PyShimCodeDef;
+
+extern PyCodeObject *
+_Py_MakeShimCode(const _PyShimCodeDef *code);
+
 
 #ifdef __cplusplus
 }

Include/internal/pycore_frame.h

@@ -42,27 +42,27 @@ typedef enum _framestate {
 enum _frameowner {
     FRAME_OWNED_BY_THREAD = 0,
     FRAME_OWNED_BY_GENERATOR = 1,
-    FRAME_OWNED_BY_FRAME_OBJECT = 2
+    FRAME_OWNED_BY_FRAME_OBJECT = 2,
+    FRAME_OWNED_BY_CSTACK = 3,
 };
 
 typedef struct _PyInterpreterFrame {
     /* "Specials" section */
-    PyObject *f_funcobj; /* Strong reference */
-    PyObject *f_globals; /* Borrowed reference */
-    PyObject *f_builtins; /* Borrowed reference */
-    PyObject *f_locals; /* Strong reference, may be NULL */
+    PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */
+    PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
+    PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */
+    PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
     PyCodeObject *f_code; /* Strong reference */
-    PyFrameObject *frame_obj; /* Strong reference, may be NULL */
+    PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
     /* Linkage section */
     struct _PyInterpreterFrame *previous;
     // NOTE: This is not necessarily the last instruction started in the given
     // frame. Rather, it is the code unit *prior to* the *next* instruction. For
     // example, it may be an inline CACHE entry, an instruction we just jumped
     // over, or (in the case of a newly-created frame) a totally invalid value:
     _Py_CODEUNIT *prev_instr;
-    int stacktop;     /* Offset of TOS from localsplus  */
+    int stacktop;  /* Offset of TOS from localsplus  */
     uint16_t yield_offset;
-    bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
     char owner;
     /* Locals and stack */
     PyObject *localsplus[1];
@@ -110,7 +110,6 @@ _PyFrame_InitializeSpecials(
     frame->stacktop = code->co_nlocalsplus;
     frame->frame_obj = NULL;
     frame->prev_instr = _PyCode_CODE(code) - 1;
-    frame->is_entry = false;
     frame->yield_offset = 0;
     frame->owner = FRAME_OWNED_BY_THREAD;
 }

Include/internal/pycore_global_objects_fini_generated.h

@@ -48,6 +48,7 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(newline, "\n")
         STRUCT_FOR_STR(open_br, "{")
         STRUCT_FOR_STR(percent, "%")
+        STRUCT_FOR_STR(shim_name, "<shim>")
         STRUCT_FOR_STR(utf_8, "utf-8")
     } literals;
 

Include/internal/pycore_global_strings.h

@@ -188,6 +188,7 @@ struct _is {
     struct ast_state ast;
     struct types_state types;
     struct callable_cache callable_cache;
+    PyCodeObject *interpreter_trampoline;
 
     /* The following fields are here to avoid allocation during init.
        The data is exposed through PyInterpreterState pointer fields.