Merge remote-tracking branch 'upstream/main' into sqlite-threadsafety

Author: Erlend E. Aasland

.github/workflows/posix-deps-apt.sh

@@ -3,6 +3,7 @@ apt-get update
 
 apt-get -yq install \
     build-essential \
+    pkg-config \
     ccache \
     gdb \
     lcov \

.gitignore

@@ -120,6 +120,7 @@ Tools/unicode/data/
 Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
+Tools/freeze/test/outdir
 
 # The frozen modules are always generated by the build so we don't
 # keep them in the repo.  Also see Tools/scripts/freeze_modules.py.

Doc/c-api/gcsupport.rst

@@ -33,6 +33,14 @@ Constructors for container types must conform to two rules:
 #. Once all the fields which may contain references to other containers are
    initialized, it must call :c:func:`PyObject_GC_Track`.
 
+Similarly, the deallocator for the object must conform to a similar pair of
+rules:
+
+#. Before fields which refer to other containers are invalidated,
+   :c:func:`PyObject_GC_UnTrack` must be called.
+
+#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`.
+
    .. warning::
       If a type adds the Py_TPFLAGS_HAVE_GC, then it *must* implement at least
       a :c:member:`~PyTypeObject.tp_traverse` handler or explicitly use one
@@ -100,14 +108,6 @@ Constructors for container types must conform to two rules:
 
    .. versionadded:: 3.9
 
-Similarly, the deallocator for the object must conform to a similar pair of
-rules:
-
-#. Before fields which refer to other containers are invalidated,
-   :c:func:`PyObject_GC_UnTrack` must be called.
-
-#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`.
-
 
 .. c:function:: void PyObject_GC_Del(void *op)
 

Doc/c-api/typeobj.rst

@@ -668,6 +668,18 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    :c:func:`PyObject_GC_Del` if the instance was allocated using
    :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`.
 
+   If the type supports garbage collection (has the :const:`Py_TPFLAGS_HAVE_GC`
+   flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack`
+   before clearing any member fields.
+
+   .. code-block:: c
+
+     static void foo_dealloc(foo_object *self) {
+         PyObject_GC_UnTrack(self);
+         Py_CLEAR(self->ref);
+         Py_TYPE(self)->tp_free((PyObject *)self);
+     }
+
    Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the
    deallocator should decrement the reference count for its type object after
    calling the type deallocator. In order to avoid dangling pointers, the

Doc/extending/newtypes.rst

@@ -73,7 +73,19 @@ function::
    newdatatype_dealloc(newdatatypeobject *obj)
    {
        free(obj->obj_UnderlyingDatatypePtr);
-       Py_TYPE(obj)->tp_free(obj);
+       Py_TYPE(obj)->tp_free((PyObject *)obj);
+   }
+
+If your type supports garbage collection, the destructor should call
+:c:func:`PyObject_GC_UnTrack` before clearing any member fields::
+
+   static void
+   newdatatype_dealloc(newdatatypeobject *obj)
+   {
+       PyObject_GC_UnTrack(obj);
+       Py_CLEAR(obj->other_obj);
+       ...
+       Py_TYPE(obj)->tp_free((PyObject *)obj);
    }
 
 .. index::

Doc/library/collections.rst

@@ -343,7 +343,7 @@ All of those tests treat missing elements as having zero counts so that
 ``Counter(a=1) == Counter(a=1, b=0)`` returns true.
 
 .. versionadded:: 3.10
-   Rich comparison operations we were added
+   Rich comparison operations were added.
 
 .. versionchanged:: 3.10
    In equality tests, missing elements are treated as having zero counts.

Doc/library/multiprocessing.rst

@@ -2636,12 +2636,13 @@ handler type) for messages from different processes to get mixed up.
    inherited.
 
 .. currentmodule:: multiprocessing
-.. function:: log_to_stderr()
+.. function:: log_to_stderr(level=None)
 
    This function performs a call to :func:`get_logger` but in addition to
    returning the logger created by get_logger, it adds a handler which sends
    output to :data:`sys.stderr` using format
    ``'[%(levelname)s/%(processName)s] %(message)s'``.
+   You can modify ``levelname`` of the logger by passing a ``level`` argument.
 
 Below is an example session with logging turned on::
 

Doc/library/sqlite3.rst

@@ -117,6 +117,24 @@ Module functions and constants
 ------------------------------
 
 
+.. data:: apilevel
+
+   String constant stating the supported DB-API level. Required by the DB-API.
+   Hard-coded to ``"2.0"``.
+
+.. data:: paramstyle
+
+   String constant stating the type of parameter marker formatting expected by
+   the :mod:`sqlite3` module. Required by the DB-API. Hard-coded to
+   ``"qmark"``.
+
+   .. note::
+
+      The :mod:`sqlite3` module supports both ``qmark`` and ``numeric`` DB-API
+      parameter styles, because that is what the underlying SQLite library
+      supports. However, the DB-API does not allow multiple values for
+      the ``paramstyle`` attribute.
+
 .. data:: version
 
    The version number of this module, as a string. This is not the version of
@@ -139,6 +157,26 @@ Module functions and constants
    The version number of the run-time SQLite library, as a tuple of integers.
 
 
+.. data:: threadsafety
+
+   Integer constant required by the DB-API, stating the level of thread safety
+   the :mod:`sqlite3` module supports. Currently hard-coded to ``1``, meaning
+   *"Threads may share the module, but not connections."* However, this may not
+   always be true. You can check the underlying SQLite library's compile-time
+   threaded mode using the following query::
+
+     import sqlite3
+     con = sqlite3.connect(":memory:")
+     con.execute("""
+         select * from pragma_compile_options
+         where compile_options like 'THREADSAFE=%'
+     """).fetchall()
+
+   Note that the `SQLITE_THREADSAFE levels
+   <https://sqlite.org/compile.html#threadsafe>`_ do not match the DB-API 2.0
+   ``threadsafety`` levels.
+
+
 .. data:: PARSE_DECLTYPES
 
    This constant is meant to be used with the *detect_types* parameter of the
@@ -710,6 +748,14 @@ Cursor Objects
       The cursor will be unusable from this point forward; a :exc:`ProgrammingError`
       exception will be raised if any operation is attempted with the cursor.
 
+   .. method:: setinputsizes(sizes)
+
+      Required by the DB-API. Is a no-op in :mod:`sqlite3`.
+
+   .. method:: setoutputsize(size [, column])
+
+      Required by the DB-API. Is a no-op in :mod:`sqlite3`.
+
    .. attribute:: rowcount
 
       Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this

Doc/library/typing.rst

@@ -17,13 +17,11 @@
 
 --------------
 
-This module provides runtime support for type hints as specified by
-:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, :pep:`591`,
-:pep:`593`, :pep:`612`, :pep:`613` and :pep:`647`.
-The most fundamental support consists of the types :data:`Any`, :data:`Union`,
-:data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and
-:class:`Generic`.  For full specification please see :pep:`484`.  For
-a simplified introduction to type hints see :pep:`483`.
+This module provides runtime support for type hints. The most fundamental
+support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`,
+:data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For a full
+specification, please see :pep:`484`. For a simplified introduction to type
+hints, see :pep:`483`.
 
 
 The function below takes and returns a string and is annotated as follows::
@@ -35,6 +33,42 @@ In the function ``greeting``, the argument ``name`` is expected to be of type
 :class:`str` and the return type :class:`str`. Subtypes are accepted as
 arguments.
 
+.. _relevant-peps:
+
+Relevant PEPs
+=============
+
+Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a
+number of PEPs have modified and enhanced Python's framework for type
+annotations. These include:
+
+* :pep:`526`: Syntax for Variable Annotations
+     *Introducing* syntax for annotating variables outside of function
+     definitions, and :data:`ClassVar`
+* :pep:`544`: Protocols: Structural subtyping (static duck typing)
+     *Introducing* :class:`Protocol` and the
+     :func:`@runtime_checkable<runtime_checkable>` decorator
+* :pep:`585`: Type Hinting Generics In Standard Collections
+     *Introducing* the ability to use builtin collections and ABCs as
+     :term:`generic types<generic type>`
+* :pep:`586`: Literal Types
+     *Introducing* :data:`Literal`
+* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys
+     *Introducing* :class:`TypedDict`
+* :pep:`591`: Adding a final qualifier to typing
+     *Introducing* :data:`Final` and the :func:`@final<final>` decorator
+* :pep:`593`: Flexible function and variable annotations
+     *Introducing* :data:`Annotated`
+* :pep:`604`: Allow writing union types as ``X | Y``
+     *Introducing* :data:`types.UnionType` and the ability to use
+     the binary-or operator ``|`` as syntactic sugar for a union of types
+* :pep:`612`: Parameter Specification Variables
+     *Introducing* :class:`ParamSpec` and :data:`Concatenate`
+* :pep:`613`: Explicit Type Aliases
+     *Introducing* :data:`TypeAlias`
+* :pep:`647`: User-Defined Type Guards
+     *Introducing* :data:`TypeGuard`
+
 .. _type-aliases:
 
 Type aliases

Doc/reference/datamodel.rst

@@ -2577,8 +2577,8 @@ left undefined.
    return the value of the object truncated to an :class:`~numbers.Integral`
    (typically an :class:`int`).
 
-   If :meth:`__int__` is not defined then the built-in function :func:`int`
-   falls back to :meth:`__trunc__`.
+   The built-in function :func:`int` falls back to :meth:`__trunc__` if neither
+   :meth:`__int__` nor :meth:`__index__` is defined.
 
 
 .. _context-managers:

Include/cpython/pystate.h

@@ -46,6 +46,8 @@ typedef struct _cframe {
      * accessed outside of their lifetime.
      */
     int use_tracing;
+    /* Pointer to the currently executing frame (it can be NULL) */
+    struct _interpreter_frame *current_frame;
     struct _cframe *previous;
 } CFrame;
 
@@ -77,8 +79,6 @@ struct _ts {
     struct _ts *next;
     PyInterpreterState *interp;
 
-    /* Borrowed reference to the current frame (it can be NULL) */
-    struct _interpreter_frame *frame;
     int recursion_depth;
     int recursion_headroom; /* Allow 50 more calls to handle any errors. */
     int stackcheck_counter;

Include/internal/pycore_frame.h

@@ -152,6 +152,21 @@ _PyFrame_LocalsToFast(InterpreterFrame *frame, int clear);
 InterpreterFrame *_PyThreadState_PushFrame(
     PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals);
 
+extern InterpreterFrame *
+_PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size);
+
+static inline InterpreterFrame *
+_PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
+{
+    PyObject **base = tstate->datastack_top;
+    PyObject **top = base + size;
+    if (top < tstate->datastack_limit) {
+        tstate->datastack_top = top;
+        return (InterpreterFrame *)base;
+    }
+    return _PyThreadState_BumpFramePointerSlow(tstate, size);
+}
+
 void _PyThreadState_PopFrame(PyThreadState *tstate, InterpreterFrame *frame);
 
 #ifdef __cplusplus

Include/internal/pycore_interp.h

@@ -335,7 +335,7 @@ struct _is {
        The integers that are preallocated are those in the range
        -_PY_NSMALLNEGINTS (inclusive) to _PY_NSMALLPOSINTS (not inclusive).
     */
-    PyLongObject* small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS];
+    PyLongObject small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS];
     struct _Py_bytes_state bytes;
     struct _Py_unicode_state unicode;
     struct _Py_float_state float_state;

Include/internal/pycore_long.h

@@ -17,7 +17,7 @@ static inline PyObject* __PyLong_GetSmallInt_internal(int value)
     PyInterpreterState *interp = _PyInterpreterState_GET();
     assert(-_PY_NSMALLNEGINTS <= value && value < _PY_NSMALLPOSINTS);
     size_t index = _PY_NSMALLNEGINTS + value;
-    PyObject *obj = (PyObject*)interp->small_ints[index];
+    PyObject *obj = (PyObject*)&interp->small_ints[index];
     // _PyLong_GetZero(), _PyLong_GetOne() and get_small_int() must not be
     // called before _PyLong_Init() nor after _PyLong_Fini().
     assert(obj != NULL);

Include/internal/pycore_pylifecycle.h

@@ -53,7 +53,7 @@ extern PyStatus _PyUnicode_Init(PyInterpreterState *interp);
 extern PyStatus _PyUnicode_InitTypes(void);
 extern PyStatus _PyBytes_Init(PyInterpreterState *interp);
 extern int _PyStructSequence_Init(void);
-extern int _PyLong_Init(PyInterpreterState *interp);
+extern void _PyLong_Init(PyInterpreterState *interp);
 extern int _PyLong_InitTypes(void);
 extern PyStatus _PyTuple_Init(PyInterpreterState *interp);
 extern PyStatus _PyFaulthandler_Init(int enable);

Lib/doctest.py

@@ -1034,10 +1034,8 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen):
         if inspect.isclass(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 # Special handling for staticmethod/classmethod.
-                if isinstance(val, staticmethod):
-                    val = getattr(obj, valname)
-                if isinstance(val, classmethod):
-                    val = getattr(obj, valname).__func__
+                if isinstance(val, (staticmethod, classmethod)):
+                    val = val.__func__
 
                 # Recurse to methods, properties, and nested classes.
                 if ((inspect.isroutine(val) or inspect.isclass(val) or

Lib/inspect.py

@@ -1325,6 +1325,8 @@ def getargvalues(frame):
 def formatannotation(annotation, base_module=None):
     if getattr(annotation, '__module__', None) == 'typing':
         return repr(annotation).replace('typing.', '')
+    if isinstance(annotation, types.GenericAlias):
+        return str(annotation)
     if isinstance(annotation, type):
         if annotation.__module__ in ('builtins', base_module):
             return annotation.__qualname__

Lib/sqlite3/__init__.py

@@ -20,6 +20,40 @@
 #    misrepresented as being the original software.
 # 3. This notice may not be removed or altered from any source distribution.
 
+"""
+The sqlite3 extension module provides a DB-API 2.0 (PEP 249) compilant
+interface to the SQLite library, and requires SQLite 3.7.15 or newer.
+
+To use the module, you must first create a database Connection object:
+
+    import sqlite3
+    cx = sqlite3.connect("test.db")  # test.db will be created or opened
+
+You can also use the special database name ":memory:" to connect to a transient
+in-memory database:
+
+    cx = sqlite3.connect(":memory:")  # connect to a database in RAM
+
+Once you have a Connection object, you can create a Cursor object and call its
+execute() method to perform SQL queries:
+
+    cu = cx.cursor()
+
+    # create a table
+    cu.execute("create table lang(name, first_appeared)")
+
+    # insert values into a table
+    cu.execute("insert into lang values (?, ?)", ("C", 1972))
+
+    # execute a query and iterate over the result
+    for row in cu.execute("select * from lang"):
+        print(row)
+
+    cx.close()
+
+The sqlite3 module is written by Gerhard Häring <[email protected]>.
+"""
+
 from sqlite3.dbapi2 import *
 
 

Lib/test/pickletester.py

@@ -2383,7 +2383,8 @@ def test_bad_getattr(self):
         # Issue #3514: crash when there is an infinite loop in __getattr__
         x = BadGetattr()
         for proto in protocols:
-            self.assertRaises(RuntimeError, self.dumps, x, proto)
+            with support.infinite_recursion():
+                self.assertRaises(RuntimeError, self.dumps, x, proto)
 
     def test_reduce_bad_iterator(self):
         # Issue4176: crash when 4th and 5th items of __reduce__()