bpo-45508: Specialize INPLACE_ADD

Include/internal/pycore_code.h

@@ -268,6 +268,7 @@ int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNI
 int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
 int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
 int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
+int _Py_Specialize_InplaceAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
 int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
 int _Py_Specialize_CallFunction(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins);
 

Lib/opcode.py

@@ -225,6 +225,11 @@ def jabs_op(name, op):
     "BINARY_ADD_FLOAT",
     "BINARY_ADD_UNICODE",
     "BINARY_ADD_UNICODE_INPLACE_FAST",
+    "INPLACE_ADD_ADAPTIVE",
+    "INPLACE_ADD_INT",
+    "INPLACE_ADD_FLOAT",
+    "INPLACE_ADD_UNICODE",
+    "INPLACE_ADD_UNICODE_FAST",
     "BINARY_MULTIPLY_ADAPTIVE",
     "BINARY_MULTIPLY_INT",
     "BINARY_MULTIPLY_FLOAT",

Misc/NEWS.d/next/Core and Builtins/2021-10-18-09-54-55.bpo-45508.6sPZsS.rst

@@ -0,0 +1 @@
+Specialized the ``INPLACE_ADD`` opcode to match the specializations of ``BINARY_ADD`` (see PEP 659 for details).

Python/ceval.c

@@ -92,8 +92,6 @@ static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);
 static int import_all_from(PyThreadState *, PyObject *, PyObject *);
 static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);
 static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
-static PyObject * unicode_concatenate(PyThreadState *, PyObject *, PyObject *,
-                                      InterpreterFrame *, const _Py_CODEUNIT *);
 static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);
 static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);
 static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int);
@@ -2437,21 +2435,109 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
         }
 
         TARGET(INPLACE_ADD) {
+            PREDICTED(INPLACE_ADD);
+            STAT_INC(INPLACE_ADD, unquickened);
             PyObject *right = POP();
             PyObject *left = TOP();
-            PyObject *sum;
-            if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) {
-                sum = unicode_concatenate(tstate, left, right, frame, next_instr);
-                /* unicode_concatenate consumed the ref to left */
+            PyObject *sum = PyNumber_InPlaceAdd(left, right);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            SET_TOP(sum);
+            if (sum == NULL)
+                goto error;
+            DISPATCH();
+        }
+
+        TARGET(INPLACE_ADD_ADAPTIVE) {
+            if (oparg == 0) {
+                PyObject *left = SECOND();
+                PyObject *right = TOP();
+                next_instr--;
+                if (_Py_Specialize_InplaceAdd(left, right, next_instr) < 0) {
+                    goto error;
+                }
+                DISPATCH();
             }
             else {
-                sum = PyNumber_InPlaceAdd(left, right);
-                Py_DECREF(left);
+                STAT_INC(INPLACE_ADD, deferred);
+                UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1);
+                STAT_DEC(INPLACE_ADD, unquickened);
+                JUMP_TO_INSTRUCTION(INPLACE_ADD);
             }
+        }
+
+        TARGET(INPLACE_ADD_INT) {
+            PyObject *left = SECOND();
+            PyObject *right = TOP();
+            DEOPT_IF(!PyLong_CheckExact(left), INPLACE_ADD);
+            DEOPT_IF(!PyLong_CheckExact(right), INPLACE_ADD);
+            STAT_INC(INPLACE_ADD, hit);
+            PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
+            SET_SECOND(sum);
+            Py_DECREF(left);
             Py_DECREF(right);
-            SET_TOP(sum);
-            if (sum == NULL)
+            STACK_SHRINK(1);
+            if (sum == NULL) {
                 goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(INPLACE_ADD_FLOAT) {
+            PyObject *left = SECOND();
+            PyObject *right = TOP();
+            DEOPT_IF(!PyFloat_CheckExact(left), INPLACE_ADD);
+            DEOPT_IF(!PyFloat_CheckExact(right), INPLACE_ADD);
+            STAT_INC(INPLACE_ADD, hit);
+            double dsum = ((PyFloatObject *)left)->ob_fval +
+                ((PyFloatObject *)right)->ob_fval;
+            PyObject *sum = PyFloat_FromDouble(dsum);
+            SET_SECOND(sum);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            STACK_SHRINK(1);
+            if (sum == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(INPLACE_ADD_UNICODE) {
+            PyObject *left = SECOND();
+            PyObject *right = TOP();
+            DEOPT_IF(!PyUnicode_CheckExact(left), INPLACE_ADD);
+            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), INPLACE_ADD);
+            STAT_INC(BINARY_ADD, hit);
+            PyObject *res = PyUnicode_Concat(left, right);
+            STACK_SHRINK(1);
+            SET_TOP(res);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            if (TOP() == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(INPLACE_ADD_UNICODE_FAST) {
+            PyObject *left = SECOND();
+            PyObject *right = TOP();
+            DEOPT_IF(!PyUnicode_CheckExact(left), INPLACE_ADD);
+            DEOPT_IF(!PyUnicode_CheckExact(right), INPLACE_ADD);
+            DEOPT_IF(Py_REFCNT(left) != 2, INPLACE_ADD);
+            int next_oparg = _Py_OPARG(*next_instr);
+            assert(_Py_OPCODE(*next_instr) == STORE_FAST);
+            PyObject *var = GETLOCAL(next_oparg);
+            DEOPT_IF(var != left, INPLACE_ADD);
+            STAT_INC(INPLACE_ADD, hit);
+            GETLOCAL(next_oparg) = NULL;
+            Py_DECREF(left);
+            STACK_SHRINK(1);
+            PyUnicode_Append(&TOP(), right);
+            Py_DECREF(right);
+            if (TOP() == NULL) {
+                goto error;
+            }
             DISPATCH();
         }
 
@@ -5134,6 +5220,7 @@ MISS_WITH_CACHE(LOAD_METHOD)
 MISS_WITH_CACHE(CALL_FUNCTION)
 MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR)
 MISS_WITH_OPARG_COUNTER(BINARY_ADD)
+MISS_WITH_OPARG_COUNTER(INPLACE_ADD)
 MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY)
 
 binary_subscr_dict_error:
@@ -7149,60 +7236,6 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop
     }
 }
 
-static PyObject *
-unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w,
-                    InterpreterFrame *frame, const _Py_CODEUNIT *next_instr)
-{
-    PyObject *res;
-    if (Py_REFCNT(v) == 2) {
-        /* In the common case, there are 2 references to the value
-         * stored in 'variable' when the += is performed: one on the
-         * value stack (in 'v') and one still stored in the
-         * 'variable'.  We try to delete the variable now to reduce
-         * the refcnt to 1.
-         */
-        int opcode, oparg;
-        NEXTOPARG();
-        next_instr++;
-        switch (opcode) {
-        case STORE_FAST:
-        {
-            if (GETLOCAL(oparg) == v)
-                SETLOCAL(oparg, NULL);
-            break;
-        }
-        case STORE_DEREF:
-        {
-            PyObject *c = _PyFrame_GetLocalsArray(frame)[oparg];
-            if (PyCell_GET(c) ==  v) {
-                PyCell_SET(c, NULL);
-                Py_DECREF(v);
-            }
-            break;
-        }
-        case STORE_NAME:
-        {
-            PyObject *names = frame->f_code->co_names;
-            PyObject *name = GETITEM(names, oparg);
-            PyObject *locals = frame->f_locals;
-            if (locals && PyDict_CheckExact(locals)) {
-                PyObject *w = PyDict_GetItemWithError(locals, name);
-                if ((w == v && PyDict_DelItem(locals, name) != 0) ||
-                    (w == NULL && _PyErr_Occurred(tstate)))
-                {
-                    Py_DECREF(v);
-                    return NULL;
-                }
-            }
-            break;
-        }
-        }
-    }
-    res = v;
-    PyUnicode_Append(&res, w);
-    return res;
-}
-
 #ifdef DYNAMIC_EXECUTION_PROFILE
 
 static PyObject *