Merge branch 'main' into gh-118527-code-consts

Author: colesbury

.github/workflows/posix-deps-apt.sh

@@ -15,6 +15,7 @@ apt-get -yq install \
     libgdbm-dev \
     libgdbm-compat-dev \
     liblzma-dev \
+    libmpdec-dev \
     libncurses5-dev \
     libreadline6-dev \
     libsqlite3-dev \

Doc/using/configure.rst

@@ -804,11 +804,18 @@ Libraries options
 
 .. option:: --with-system-libmpdec
 
-   Build the ``_decimal`` extension module using an installed ``mpdec``
-   library, see the :mod:`decimal` module (default is no).
+   Build the ``_decimal`` extension module using an installed ``mpdecimal``
+   library, see the :mod:`decimal` module (default is yes).
 
    .. versionadded:: 3.3
 
+   .. versionchanged:: 3.13
+      Default to using the installed ``mpdecimal`` library.
+
+   .. deprecated-removed:: 3.13 3.15
+      A copy of the ``mpdecimal`` library sources will no longer be distributed
+      with Python 3.15.
+
    .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`.
 
 .. option:: --with-readline=readline|editline

Doc/whatsnew/3.13.rst

@@ -1807,6 +1807,10 @@ Changes in the Python API
 Build Changes
 =============
 
+* The :file:`configure` option :option:`--with-system-libmpdec` now defaults
+  to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python
+  3.15.
+
 * Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate
   the :file:`configure` script.
   (Contributed by Christian Heimes in :gh:`89886`.)
@@ -2336,6 +2340,7 @@ Pending Removal in Python 3.14
 Pending Removal in Python 3.15
 ------------------------------
 
+* The bundled copy of ``libmpdecimal``.
 * :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`.
 * :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead.
 * :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead.

Include/cpython/optimizer.h

@@ -141,16 +141,22 @@ void _Py_ExecutorDetach(_PyExecutorObject *);
 void _Py_BloomFilter_Init(_PyBloomFilter *);
 void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj);
 PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj);
-PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation);
-PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation);
-
 /* For testing */
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);
 
 #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6
 
+#ifdef _Py_TIER2
+PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation);
+PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation);
+#else
+#  define _Py_Executors_InvalidateDependency(A, B, C) ((void)0)
+#  define _Py_Executors_InvalidateAll(A, B) ((void)0)
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif

Lib/_pyrepl/__main__.py

@@ -1,7 +1,8 @@
 import os
 import sys
 
-CAN_USE_PYREPL = True
+CAN_USE_PYREPL = sys.platform != "win32"
+
 
 def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
     global CAN_USE_PYREPL
@@ -21,7 +22,7 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
         sys.ps1 = ">>> "
     if not hasattr(sys, "ps2"):
         sys.ps2 = "... "
-    #
+
     run_interactive = None
     try:
         import errno
@@ -33,7 +34,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
         from .simple_interact import run_multiline_interactive_console
         run_interactive = run_multiline_interactive_console
     except Exception as e:
-        print(f"warning: can't use pyrepl: {e}", file=sys.stderr)
+        from .trace import trace
+        msg = f"warning: can't use pyrepl: {e}"
+        trace(msg)
+        print(msg, file=sys.stderr)
         CAN_USE_PYREPL = False
     if run_interactive is None:
         return sys._baserepl()

Lib/pdb.py

@@ -77,10 +77,12 @@
 import code
 import glob
 import token
+import types
 import codeop
 import pprint
 import signal
 import inspect
+import textwrap
 import tokenize
 import traceback
 import linecache
@@ -624,11 +626,96 @@ def _disable_command_completion(self):
             self.completenames = completenames
         return
 
+    def _exec_in_closure(self, source, globals, locals):
+        """ Run source code in closure so code object created within source
+            can find variables in locals correctly
+
+            returns True if the source is executed, False otherwise
+        """
+
+        # Determine if the source should be executed in closure. Only when the
+        # source compiled to multiple code objects, we should use this feature.
+        # Otherwise, we can just raise an exception and normal exec will be used.
+
+        code = compile(source, "<string>", "exec")
+        if not any(isinstance(const, CodeType) for const in code.co_consts):
+            return False
+
+        # locals could be a proxy which does not support pop
+        # copy it first to avoid modifying the original locals
+        locals_copy = dict(locals)
+
+        locals_copy["__pdb_eval__"] = {
+            "result": None,
+            "write_back": {}
+        }
+
+        # If the source is an expression, we need to print its value
+        try:
+            compile(source, "<string>", "eval")
+        except SyntaxError:
+            pass
+        else:
+            source = "__pdb_eval__['result'] = " + source
+
+        # Add write-back to update the locals
+        source = ("try:\n" +
+                  textwrap.indent(source, "  ") + "\n" +
+                  "finally:\n" +
+                  "  __pdb_eval__['write_back'] = locals()")
+
+        # Build a closure source code with freevars from locals like:
+        # def __pdb_outer():
+        #   var = None
+        #   def __pdb_scope():  # This is the code object we want to execute
+        #     nonlocal var
+        #     <source>
+        #   return __pdb_scope.__code__
+        source_with_closure = ("def __pdb_outer():\n" +
+                               "\n".join(f"  {var} = None" for var in locals_copy) + "\n" +
+                               "  def __pdb_scope():\n" +
+                               "\n".join(f"    nonlocal {var}" for var in locals_copy) + "\n" +
+                               textwrap.indent(source, "    ") + "\n" +
+                               "  return __pdb_scope.__code__"
+                               )
+
+        # Get the code object of __pdb_scope()
+        # The exec fills locals_copy with the __pdb_outer() function and we can call
+        # that to get the code object of __pdb_scope()
+        ns = {}
+        try:
+            exec(source_with_closure, {}, ns)
+        except Exception:
+            return False
+        code = ns["__pdb_outer"]()
+
+        cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars)
+
+        try:
+            exec(code, globals, locals_copy, closure=cells)
+        except Exception:
+            return False
+
+        # get the data we need from the statement
+        pdb_eval = locals_copy["__pdb_eval__"]
+
+        # __pdb_eval__ should not be updated back to locals
+        pdb_eval["write_back"].pop("__pdb_eval__")
+
+        # Write all local variables back to locals
+        locals.update(pdb_eval["write_back"])
+        eval_result = pdb_eval["result"]
+        if eval_result is not None:
+            print(repr(eval_result))
+
+        return True
+
     def default(self, line):
         if line[:1] == '!': line = line[1:].strip()
         locals = self.curframe_locals
         globals = self.curframe.f_globals
         try:
+            buffer = line
             if (code := codeop.compile_command(line + '\n', '<stdin>', 'single')) is None:
                 # Multi-line mode
                 with self._disable_command_completion():
@@ -661,7 +748,8 @@ def default(self, line):
                 sys.stdin = self.stdin
                 sys.stdout = self.stdout
                 sys.displayhook = self.displayhook
-                exec(code, globals, locals)
+                if not self._exec_in_closure(buffer, globals, locals):
+                    exec(code, globals, locals)
             finally:
                 sys.stdout = save_stdout
                 sys.stdin = save_stdin

Lib/test/test_capi/test_opt.py

@@ -1321,5 +1321,18 @@ def testfunc(n):
         self.assertIsNotNone(ex)
         self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex))
 
+    def test_modified_local_is_seen_by_optimized_code(self):
+        l = sys._getframe().f_locals
+        a = 1
+        s = 0
+        for j in range(1 << 10):
+            a + a
+            l["xa"[j >> 9]] = 1.0
+            s += a
+        self.assertIs(type(a), float)
+        self.assertIs(type(s), float)
+        self.assertEqual(s, 1024.0)
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_free_threading/test_monitoring.py

@@ -8,20 +8,14 @@
 
 from sys import monitoring
 from test.support import is_wasi
-from threading import Thread
+from threading import Thread, _PyRLock
 from unittest import TestCase
 
 
 class InstrumentationMultiThreadedMixin:
-    if not hasattr(sys, "gettotalrefcount"):
-        thread_count = 50
-        func_count = 1000
-        fib = 15
-    else:
-        # Run a little faster in debug builds...
-        thread_count = 25
-        func_count = 500
-        fib = 15
+    thread_count = 10
+    func_count = 200
+    fib = 12
 
     def after_threads(self):
         """Runs once after all the threads have started"""
@@ -175,9 +169,6 @@ def during_threads(self):
 @unittest.skipIf(is_wasi, "WASI has no threads.")
 class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase):
     """Uses sys.setprofile and repeatedly toggles instrumentation on and off"""
-    thread_count = 25
-    func_count = 200
-    fib = 15
 
     def setUp(self):
         self.set = False
@@ -227,6 +218,28 @@ def test_register_callback(self):
         for ref in self.refs:
             self.assertEqual(ref(), None)
 
+    def test_set_local_trace_opcodes(self):
+        def trace(frame, event, arg):
+            frame.f_trace_opcodes = True
+            return trace
+
+        sys.settrace(trace)
+        try:
+            l = _PyRLock()
+
+            def f():
+                for i in range(3000):
+                    with l:
+                        pass
+
+            t = Thread(target=f)
+            t.start()
+            for i in range(3000):
+                with l:
+                    pass
+            t.join()
+        finally:
+            sys.settrace(None)
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_pdb.py

@@ -2224,8 +2224,71 @@ def test_pdb_multiline_statement():
     (Pdb) c
     """
 
+def test_pdb_closure():
+    """Test for all expressions/statements that involve closure
+
+    >>> k = 0
+    >>> g = 1
+    >>> def test_function():
+    ...     x = 2
+    ...     g = 3
+    ...     import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace()
+
+    >>> with PdbTestInput([  # doctest: +NORMALIZE_WHITESPACE
+    ...     'k',
+    ...     'g',
+    ...     'y = y',
+    ...     'global g; g',
+    ...     'global g; (lambda: g)()',
+    ...     '(lambda: x)()',
+    ...     '(lambda: g)()',
+    ...     'lst = [n for n in range(10) if (n % x) == 0]',
+    ...     'lst',
+    ...     'sum(n for n in lst if n > x)',
+    ...     'x = 1; raise Exception()',
+    ...     'x',
+    ...     'def f():',
+    ...     '  return x',
+    ...     '',
+    ...     'f()',
+    ...     'c'
+    ... ]):
+    ...     test_function()
+    > <doctest test.test_pdb.test_pdb_closure[2]>(4)test_function()
+    -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace()
+    (Pdb) k
+    0
+    (Pdb) g
+    3
+    (Pdb) y = y
+    *** NameError: name 'y' is not defined
+    (Pdb) global g; g
+    1
+    (Pdb) global g; (lambda: g)()
+    1
+    (Pdb) (lambda: x)()
+    2
+    (Pdb) (lambda: g)()
+    3
+    (Pdb) lst = [n for n in range(10) if (n % x) == 0]
+    (Pdb) lst
+    [0, 2, 4, 6, 8]
+    (Pdb) sum(n for n in lst if n > x)
+    18
+    (Pdb) x = 1; raise Exception()
+    *** Exception
+    (Pdb) x
+    1
+    (Pdb) def f():
+    ...     return x
+    ...
+    (Pdb) f()
+    1
+    (Pdb) c
+    """
+
 def test_pdb_show_attribute_and_item():
-    """Test for multiline statement
+    """Test for expressions with command prefix
 
     >>> def test_function():
     ...     n = lambda x: x

Misc/NEWS.d/next/Build/2024-05-06-00-39-06.gh-issue-115119.LT27pF.rst

@@ -0,0 +1,2 @@
+The :file:`configure` option :option:`--with-system-libmpdec` now defaults to ``yes``.
+The bundled copy of ``libmpdecimal`` will be removed in Python 3.15.

Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst

@@ -0,0 +1,3 @@
+Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access the
+local variables of the current frame, which made it possible for multi-scope
+code like generators or nested function to work.

Modules/_tkinter.c

@@ -2972,6 +2972,7 @@ Tkapp_Dealloc(PyObject *self)
     ENTER_TCL
     Tcl_DeleteInterp(Tkapp_Interp(self));
     LEAVE_TCL
+    Py_XDECREF(((TkappObject *)self)->trace);
     PyObject_Free(self);
     Py_DECREF(tp);
     DisableEventHook();