Merge branch 'main' into use-extra-assertions

Author: JelleZijlstra

Doc/c-api/init_config.rst

@@ -1054,6 +1054,15 @@ These are the UTF-8 codec APIs:
 
    As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size.
 
+   .. warning::
+
+      This function does not have any special behavior for
+      `null characters <https://en.wikipedia.org/wiki/Null_character>`_ embedded within
+      *unicode*. As a result, strings containing null characters will remain in the returned
+      string, which some C functions might interpret as the end of the string, leading to
+      truncation. If truncation is an issue, it is recommended to use :c:func:`PyUnicode_AsUTF8AndSize`
+      instead.
+
    .. versionadded:: 3.3
 
    .. versionchanged:: 3.7

Doc/c-api/unicode.rst

@@ -811,9 +811,11 @@ Glossary
       processed.
 
    loader
-      An object that loads a module. It must define a method named
-      :meth:`load_module`. A loader is typically returned by a
-      :term:`finder`. See also:
+      An object that loads a module.
+      It must define the :meth:`!exec_module` and :meth:`!create_module` methods
+      to implement the :class:`~importlib.abc.Loader` interface.
+      A loader is typically returned by a :term:`finder`.
+      See also:
 
       * :ref:`finders-and-loaders`
       * :class:`importlib.abc.Loader`

Doc/glossary.rst

@@ -12,7 +12,6 @@ Doc/c-api/stable.rst
 Doc/c-api/type.rst
 Doc/c-api/typeobj.rst
 Doc/extending/extending.rst
-Doc/glossary.rst
 Doc/library/ast.rst
 Doc/library/asyncio-extending.rst
 Doc/library/asyncio-subprocess.rst

Doc/tools/.nitignore

@@ -1375,3 +1375,7 @@ Removed
 
 * Creating :c:data:`immutable types <Py_TPFLAGS_IMMUTABLETYPE>` with mutable
   bases was deprecated since 3.12 and now raises a :exc:`TypeError`.
+
+* Remove the private ``_Py_InitializeMain()`` function. It was a
+  :term:`provisional API` added to Python 3.8 by :pep:`587`.
+  (Contributed by Victor Stinner in :gh:`129033`.)

Doc/whatsnew/3.14.rst

@@ -25,9 +25,6 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs(
 PyAPI_FUNC(PyStatus) Py_InitializeFromConfig(
     const PyConfig *config);
 
-// Python 3.8 provisional API (PEP 587)
-PyAPI_FUNC(PyStatus) _Py_InitializeMain(void);
-
 PyAPI_FUNC(int) Py_RunMain(void);
 
 

Include/cpython/pylifecycle.h

@@ -341,43 +341,6 @@ extern void _PyInterpreterState_SetWhence(
 
 extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp);
 
-// Get a copy of the current interpreter configuration.
-//
-// Return 0 on success. Raise an exception and return -1 on error.
-//
-// The caller must initialize 'config', using PyConfig_InitPythonConfig()
-// for example.
-//
-// Python must be preinitialized to call this method.
-// The caller must hold the GIL.
-//
-// Once done with the configuration, PyConfig_Clear() must be called to clear
-// it.
-//
-// Export for '_testinternalcapi' shared extension.
-PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy(
-    struct PyConfig *config);
-
-// Set the configuration of the current interpreter.
-//
-// This function should be called during or just after the Python
-// initialization.
-//
-// Update the sys module with the new configuration. If the sys module was
-// modified directly after the Python initialization, these changes are lost.
-//
-// Some configuration like faulthandler or warnoptions can be updated in the
-// configuration, but don't reconfigure Python (don't enable/disable
-// faulthandler and don't reconfigure warnings filters).
-//
-// Return 0 on success. Raise an exception and return -1 on error.
-//
-// The configuration should come from _PyInterpreterState_GetConfigCopy().
-//
-// Export for '_testinternalcapi' shared extension.
-PyAPI_FUNC(int) _PyInterpreterState_SetConfig(
-    const struct PyConfig *config);
-
 
 /*
 Runtime Feature Flags

Include/internal/pycore_interp.h

@@ -148,15 +148,6 @@ extern PyTypeObject _PyDefaultOptimizer_Type;
 extern PyTypeObject _PyUOpExecutor_Type;
 extern PyTypeObject _PyUOpOptimizer_Type;
 
-/* Symbols */
-/* See explanation in optimizer_symbols.c */
-
-struct _Py_UopsSymbol {
-    int flags;  // 0 bits: Top; 2 or more bits: Bottom
-    PyTypeObject *typ;  // Borrowed reference
-    PyObject *const_val;  // Owned reference (!)
-    unsigned int type_version; // currently stores type version
-};
 
 #define UOP_FORMAT_TARGET 0
 #define UOP_FORMAT_JUMP 1
@@ -193,27 +184,74 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 // handle before rejoining the rest of the program.
 #define MAX_CHAIN_DEPTH 4
 
-typedef struct _Py_UopsSymbol _Py_UopsSymbol;
+/* Symbols */
+/* See explanation in optimizer_symbols.c */
+
+
+typedef enum _JitSymType {
+    JIT_SYM_UNKNOWN_TAG = 1,
+    JIT_SYM_NULL_TAG = 2,
+    JIT_SYM_NON_NULL_TAG = 3,
+    JIT_SYM_BOTTOM_TAG = 4,
+    JIT_SYM_TYPE_VERSION_TAG = 5,
+    JIT_SYM_KNOWN_CLASS_TAG = 6,
+    JIT_SYM_KNOWN_VALUE_TAG = 7,
+    JIT_SYM_TUPLE_TAG = 8,
+} JitSymType;
+
+typedef struct _jit_opt_known_class {
+    uint8_t tag;
+    uint32_t version;
+    PyTypeObject *type;
+} JitOptKnownClass;
+
+typedef struct _jit_opt_known_version {
+    uint8_t tag;
+    uint32_t version;
+} JitOptKnownVersion;
+
+typedef struct _jit_opt_known_value {
+    uint8_t tag;
+    PyObject *value;
+} JitOptKnownValue;
+
+#define MAX_SYMBOLIC_TUPLE_SIZE 7
+
+typedef struct _jit_opt_tuple {
+    uint8_t tag;
+    uint8_t length;
+    uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE];
+} JitOptTuple;
+
+typedef union _jit_opt_symbol {
+    uint8_t tag;
+    JitOptKnownClass cls;
+    JitOptKnownValue value;
+    JitOptKnownVersion version;
+    JitOptTuple tuple;
+} JitOptSymbol;
+
+
 
 struct _Py_UOpsAbstractFrame {
     // Max stacklen
     int stack_len;
     int locals_len;
 
-    _Py_UopsSymbol **stack_pointer;
-    _Py_UopsSymbol **stack;
-    _Py_UopsSymbol **locals;
+    JitOptSymbol **stack_pointer;
+    JitOptSymbol **stack;
+    JitOptSymbol **locals;
 };
 
 typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame;
 
 typedef struct ty_arena {
     int ty_curr_number;
     int ty_max_number;
-    _Py_UopsSymbol arena[TY_ARENA_SIZE];
+    JitOptSymbol arena[TY_ARENA_SIZE];
 } ty_arena;
 
-struct _Py_UOpsContext {
+typedef struct _JitOptContext {
     char done;
     char out_of_space;
     bool contradiction;
@@ -225,46 +263,47 @@ struct _Py_UOpsContext {
     // Arena for the symbolic types.
     ty_arena t_arena;
 
-    _Py_UopsSymbol **n_consumed;
-    _Py_UopsSymbol **limit;
-    _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
-};
-
-typedef struct _Py_UOpsContext _Py_UOpsContext;
-
-extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym);
-extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym);
-extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym);
-extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym);
-extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx);
-extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx);
-extern _Py_UopsSymbol *_Py_uop_sym_new_type(
-    _Py_UOpsContext *ctx, PyTypeObject *typ);
-extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val);
-extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx);
-extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym);
-extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ);
-extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version);
-extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym);
-extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym);
-extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ);
-extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version);
-extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val);
-extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym);
-extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym);
-extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym);
-
-
-extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx);
-extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx);
+    JitOptSymbol **n_consumed;
+    JitOptSymbol **limit;
+    JitOptSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
+} JitOptContext;
+
+extern bool _Py_uop_sym_is_null(JitOptSymbol *sym);
+extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym);
+extern bool _Py_uop_sym_is_const(JitOptSymbol *sym);
+extern PyObject *_Py_uop_sym_get_const(JitOptSymbol *sym);
+extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx);
+extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx);
+extern JitOptSymbol *_Py_uop_sym_new_type(
+    JitOptContext *ctx, PyTypeObject *typ);
+extern JitOptSymbol *_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val);
+extern JitOptSymbol *_Py_uop_sym_new_null(JitOptContext *ctx);
+extern bool _Py_uop_sym_has_type(JitOptSymbol *sym);
+extern bool _Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ);
+extern bool _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version);
+extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym);
+extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym);
+extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ);
+extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version);
+extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val);
+extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym);
+extern int _Py_uop_sym_truthiness(JitOptSymbol *sym);
+extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym);
+extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym);
+extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args);
+extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item);
+extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym);
+
+extern void _Py_uop_abstractcontext_init(JitOptContext *ctx);
+extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx);
 
 extern _Py_UOpsAbstractFrame *_Py_uop_frame_new(
-    _Py_UOpsContext *ctx,
+    JitOptContext *ctx,
     PyCodeObject *co,
     int curr_stackentries,
-    _Py_UopsSymbol **args,
+    JitOptSymbol **args,
     int arg_len);
-extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx);
+extern int _Py_uop_frame_pop(JitOptContext *ctx);
 
 PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
 

Include/internal/pycore_optimizer.h

@@ -300,6 +300,19 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
 // See also PyInterpreterState_Get() and _PyInterpreterState_GET().
 extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
 
+#ifndef NDEBUG
+/* Modern equivalent of assert(PyGILState_Check()) */
+static inline void
+_Py_AssertHoldsTstateFunc(const char *func)
+{
+    PyThreadState *tstate = _PyThreadState_GET();
+    _Py_EnsureFuncTstateNotNULL(func, tstate);
+}
+#define _Py_AssertHoldsTstate() _Py_AssertHoldsTstateFunc(__func__)
+#else
+#define _Py_AssertHoldsTstate()
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pystate.h

@@ -26,14 +26,17 @@ class ANSIColors:
         setattr(NoColors, attr, "")
 
 
-def get_colors(colorize: bool = False) -> ANSIColors:
-    if colorize or can_colorize():
+def get_colors(colorize: bool = False, *, file=None) -> ANSIColors:
+    if colorize or can_colorize(file=file):
         return ANSIColors()
     else:
         return NoColors
 
 
-def can_colorize() -> bool:
+def can_colorize(*, file=None) -> bool:
+    if file is None:
+        file = sys.stdout
+
     if not sys.flags.ignore_environment:
         if os.environ.get("PYTHON_COLORS") == "0":
             return False
@@ -49,7 +52,7 @@ def can_colorize() -> bool:
         if os.environ.get("TERM") == "dumb":
             return False
 
-    if not hasattr(sys.stderr, "fileno"):
+    if not hasattr(file, "fileno"):
         return False
 
     if sys.platform == "win32":
@@ -62,6 +65,6 @@ def can_colorize() -> bool:
             return False
 
     try:
-        return os.isatty(sys.stderr.fileno())
+        return os.isatty(file.fileno())
     except io.UnsupportedOperation:
-        return sys.stderr.isatty()
+        return file.isatty()

Lib/_colorize.py

@@ -1558,7 +1558,7 @@ def out(s):
         save_displayhook = sys.displayhook
         sys.displayhook = sys.__displayhook__
         saved_can_colorize = _colorize.can_colorize
-        _colorize.can_colorize = lambda: False
+        _colorize.can_colorize = lambda *args, **kwargs: False
         color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None}
         for key in color_variables:
             color_variables[key] = os.environ.pop(key, None)

Lib/doctest.py

@@ -718,6 +718,9 @@ def expand_makefile_vars(s, vars):
     """
     import re
 
+    _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)"
+    _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}"
+
     # This algorithm does multiple expansion, so if vars['foo'] contains
     # "${bar}", it will expand ${foo} to ${bar}, and then expand
     # ${bar}... and so forth.  This is fine as long as 'vars' comes from