Merge branch 'main' into bpo-40956/finale

Author: Erlend E. Aasland

.github/workflows/build.yml

@@ -218,3 +218,46 @@ jobs:
       run: make pythoninfo
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
+
+
+  build_asan:
+    name: 'Address sanitizer'
+    runs-on: ubuntu-20.04
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    env:
+      OPENSSL_VER: 1.1.1k
+      ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
+    steps:
+    - uses: actions/checkout@v2
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/[email protected]
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/ccache-action@v1
+    - name: Configure CPython
+      run: ./configure --with-address-sanitizer --without-pymalloc
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu -x test_ctypes test_crypt test_decimal test_faulthandler test_interpreters test___all__ test_idle test_tix test_tk test_ttk_guionly test_ttk_textonly"

.github/workflows/doc.yml

@@ -38,7 +38,7 @@ jobs:
     - name: 'Install build dependencies'
       run: make -C Doc/ PYTHON=../python venv
     - name: 'Build documentation'
-      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html suspicious
+      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html
     - name: 'Upload'
       uses: actions/[email protected]
       with:

.travis.yml

@@ -53,7 +53,7 @@ matrix:
         - cd Doc
         - make venv PYTHON=python
       script:
-        - make check html suspicious SPHINXOPTS="-q -W -j4"
+        - make check html SPHINXOPTS="-q -W -j4"
     - name: "Documentation tests"
       os: linux
       language: c

Doc/c-api/structures.rst

@@ -99,10 +99,7 @@ the definition of all other Python objects.
 
    Return a :term:`borrowed reference`.
 
-   Use the :c:func:`Py_SET_TYPE` function to set an object type.
-
-   .. versionchanged:: 3.11
-      :c:func:`Py_TYPE()` is changed to an inline static function.
+   The :c:func:`Py_SET_TYPE` function must be used to set an object type.
 
 
 .. c:function:: int Py_IS_TYPE(PyObject *o, PyTypeObject *type)
@@ -124,10 +121,9 @@ the definition of all other Python objects.
 
    Get the reference count of the Python object *o*.
 
-   Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count.
-
    .. versionchanged:: 3.10
       :c:func:`Py_REFCNT()` is changed to the inline static function.
+      Use :c:func:`Py_SET_REFCNT()` to set an object reference count.
 
 
 .. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt)
@@ -141,10 +137,7 @@ the definition of all other Python objects.
 
    Get the size of the Python object *o*.
 
-   Use the :c:func:`Py_SET_SIZE` function to set an object size.
-
-   .. versionchanged:: 3.11
-      :c:func:`Py_SIZE()` is changed to an inline static function.
+   The :c:func:`Py_SET_SIZE` function must be used to set an object size.
 
 
 .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size)

Doc/library/dis.rst

@@ -1056,19 +1056,34 @@ All of the following opcodes use their arguments.
    Deletes local ``co_varnames[var_num]``.
 
 
+.. opcode:: MAKE_CELL (i)
+
+   Creates a new cell in slot ``i``.  If that slot is empty then
+   that value is stored into the new cell.
+
+   .. versionadded:: 3.11
+
+
 .. opcode:: LOAD_CLOSURE (i)
 
-   Pushes a reference to the cell contained in slot *i* of the cell and free
-   variable storage.  The name of the variable is ``co_cellvars[i]`` if *i* is
-   less than the length of *co_cellvars*.  Otherwise it is ``co_freevars[i -
-   len(co_cellvars)]``.
+   Pushes a reference to the cell contained in slot ``i`` of the "fast locals"
+   storage.  The name of the variable is ``co_fastlocalnames[i]``.
+
+   Note that ``LOAD_CLOSURE`` is effectively an alias for ``LOAD_FAST``.
+   It exists to keep bytecode a little more readable.
+
+   .. versionchanged:: 3.11
+      ``i`` is no longer offset by the length of ``co_varnames``.
 
 
 .. opcode:: LOAD_DEREF (i)
 
-   Loads the cell contained in slot *i* of the cell and free variable storage.
+   Loads the cell contained in slot ``i`` of the "fast locals" storage.
    Pushes a reference to the object the cell contains on the stack.
 
+   .. versionchanged:: 3.11
+      ``i`` is no longer offset by the length of ``co_varnames``.
+
 
 .. opcode:: LOAD_CLASSDEREF (i)
 
@@ -1078,20 +1093,29 @@ All of the following opcodes use their arguments.
 
    .. versionadded:: 3.4
 
+   .. versionchanged:: 3.11
+      ``i`` is no longer offset by the length of ``co_varnames``.
+
 
 .. opcode:: STORE_DEREF (i)
 
-   Stores TOS into the cell contained in slot *i* of the cell and free variable
+   Stores TOS into the cell contained in slot ``i`` of the "fast locals"
    storage.
 
+   .. versionchanged:: 3.11
+      ``i`` is no longer offset by the length of ``co_varnames``.
+
 
 .. opcode:: DELETE_DEREF (i)
 
-   Empties the cell contained in slot *i* of the cell and free variable storage.
+   Empties the cell contained in slot ``i`` of the "fast locals" storage.
    Used by the :keyword:`del` statement.
 
    .. versionadded:: 3.2
 
+   .. versionchanged:: 3.11
+      ``i`` is no longer offset by the length of ``co_varnames``.
+
 
 .. opcode:: RAISE_VARARGS (argc)
 

Doc/library/enum.rst

@@ -28,8 +28,8 @@ An enumeration:
 
 * is a set of symbolic names (members) bound to unique values
 * can be iterated over to return its members in definition order
-* uses :meth:`call` syntax to return members by value
-* uses :meth:`index` syntax to return members by name
+* uses *call* syntax to return members by value
+* uses *index* syntax to return members by name
 
 Enumerations are created either by using the :keyword:`class` syntax, or by
 using function-call syntax::
@@ -91,6 +91,12 @@ Module Contents
       the bitwise operators without losing their :class:`IntFlag` membership.
       :class:`IntFlag` members are also subclasses of :class:`int`.
 
+   :class:`EnumCheck`
+
+      An enumeration with the values ``CONTINUOUS``, ``NAMED_FLAGS``, and
+      ``UNIQUE``, for use with :func:`verify` to ensure various constraints
+      are met by a given enumeration.
+
    :class:`FlagBoundary`
 
       An enumeration with the values ``STRICT``, ``CONFORM``, ``EJECT``, and
@@ -117,9 +123,14 @@ Module Contents
 
       Enum class decorator that ensures only one name is bound to any one value.
 
+   :func:`verify`
+
+      Enum class decorator that checks user-selectable constraints on an
+      enumeration.
+
 
 .. versionadded:: 3.6  ``Flag``, ``IntFlag``, ``auto``
-.. versionadded:: 3.10  ``StrEnum``
+.. versionadded:: 3.10  ``StrEnum``, ``EnumCheck``, ``FlagBoundary``
 
 
 Data Types
@@ -514,6 +525,65 @@ Data Types
    Using :class:`auto` with :class:`IntFlag` results in integers that are powers
    of two, starting with ``1``.
 
+.. class:: EnumCheck
+
+   *EnumCheck* contains the options used by the :func:`verify` decorator to ensure
+   various constraints; failed constraints result in a :exc:`TypeError`.
+
+   .. attribute:: UNIQUE
+
+      Ensure that each value has only one name::
+
+         >>> from enum import Enum, verify, UNIQUE
+         >>> @verify(UNIQUE)
+         ... class Color(Enum):
+         ...     RED = 1
+         ...     GREEN = 2
+         ...     BLUE = 3
+         ...     CRIMSON = 1
+         Traceback (most recent call last):
+         ...
+         ValueError: aliases found in <enum 'Color'>: CRIMSON -> RED
+
+
+   .. attribute:: CONTINUOUS
+
+      Ensure that there are no missing values between the lowest-valued member
+      and the highest-valued member::
+
+         >>> from enum import Enum, verify, CONTINUOUS
+         >>> @verify(CONTINUOUS)
+         ... class Color(Enum):
+         ...     RED = 1
+         ...     GREEN = 2
+         ...     BLUE = 5
+         Traceback (most recent call last):
+         ...
+         ValueError: invalid enum 'Color': missing values 3, 4
+
+   .. attribute:: NAMED_FLAGS
+
+      Ensure that any flag groups/masks contain only named flags -- useful when
+      values are specified instead of being generated by :func:`auto`
+
+         >>> from enum import Flag, verify, NAMED_FLAGS
+         >>> @verify(NAMED_FLAGS)
+         ... class Color(Flag):
+         ...     RED = 1
+         ...     GREEN = 2
+         ...     BLUE = 4
+         ...     WHITE = 15
+         ...     NEON = 31
+         Traceback (most recent call last):
+         ...
+         ValueError: invalid Flag 'Color': 'WHITE' is missing a named flag for value 8; 'NEON' is missing named flags for values 8, 16
+
+.. note::
+
+   CONTINUOUS and NAMED_FLAGS are designed to work with integer-valued members.
+
+.. versionadded:: 3.10
+
 .. class:: FlagBoundary
 
    *FlagBoundary* controls how out-of-range values are handled in *Flag* and its
@@ -575,7 +645,7 @@ Data Types
          >>> KeepFlag(2**2 + 2**4)
          KeepFlag.BLUE|0x10
 
-.. versionadded:: 3.10  ``FlagBoundary``
+.. versionadded:: 3.10
 
 
 Utilites and Decorators
@@ -632,3 +702,11 @@ Utilites and Decorators
       Traceback (most recent call last):
       ...
       ValueError: duplicate values found in <enum 'Mistake'>: FOUR -> THREE
+
+.. decorator:: verify
+
+   A :keyword:`class` decorator specifically for enumerations.  Members from
+   :class:`EnumCheck` are used to specify which constraints should be checked
+   on the decorated enumeration.
+
+.. versionadded:: 3.10

Doc/library/exceptions.rst

@@ -409,14 +409,16 @@ The following exceptions are the exceptions that are usually raised.
 
    .. versionadded:: 3.5
 
-.. exception:: SyntaxError
+.. exception:: SyntaxError(message, details)
 
    Raised when the parser encounters a syntax error.  This may occur in an
-   :keyword:`import` statement, in a call to the built-in functions :func:`exec`
+   :keyword:`import` statement, in a call to the built-in functions
+   :func:`compile`, :func:`exec`,
    or :func:`eval`, or when reading the initial script or standard input
    (also interactively).
 
    The :func:`str` of the exception instance returns only the error message.
+   Details is a tuple whose members are also available as separate attributes.
 
    .. attribute:: filename
 
@@ -446,6 +448,11 @@ The following exceptions are the exceptions that are usually raised.
       The column in the end line where the error occurred finishes. This is
       1-indexed: the first character in the line has an ``offset`` of 1.
 
+   For errors in f-string fields, the message is prefixed by "f-string: "
+   and the offsets are offsets in a text constructed from the replacement
+   expression.  For example, compiling f'Bad {a b} field' results in this
+   args attribute: ('f-string: ...', ('', 1, 2, '(a b)\n', 1, 5)).
+
    .. versionchanged:: 3.10
       Added the :attr:`end_lineno` and :attr:`end_offset` attributes.
 
@@ -814,3 +821,4 @@ Exception hierarchy
 The class hierarchy for built-in exceptions is:
 
 .. literalinclude:: ../../Lib/test/exception_hierarchy.txt
+  :language: text

Doc/library/fractions.rst

@@ -42,7 +42,8 @@ another rational number, or from a string.
 
    where the optional ``sign`` may be either '+' or '-' and
    ``numerator`` and ``denominator`` (if present) are strings of
-   decimal digits.  In addition, any string that represents a finite
+   decimal digits (underscores may be used to delimit digits as with
+   integral literals in code).  In addition, any string that represents a finite
    value and is accepted by the :class:`float` constructor is also
    accepted by the :class:`Fraction` constructor.  In either form the
    input string may also have leading and/or trailing whitespace.
@@ -89,6 +90,10 @@ another rational number, or from a string.
       and *denominator*. :func:`math.gcd` always return a :class:`int` type.
       Previously, the GCD type depended on *numerator* and *denominator*.
 
+   .. versionchanged:: 3.11
+      Underscores are now permitted when creating a :class:`Fraction` instance
+      from a string, following :PEP:`515` rules.
+
    .. attribute:: numerator
 
       Numerator of the Fraction in lowest term.

Doc/library/functools.rst

@@ -154,15 +154,16 @@ The :mod:`functools` module defines the following functions:
 
        @lru_cache
        def count_vowels(sentence):
-           sentence = sentence.casefold()
-           return sum(sentence.count(vowel) for vowel in 'aeiou')
+           return sum(sentence.count(vowel) for vowel in 'AEIOUaeiou')
 
    If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can
    grow without bound.
 
    If *typed* is set to true, function arguments of different types will be
-   cached separately.  For example, ``f(3)`` and ``f(3.0)`` will be treated
-   as distinct calls with distinct results.
+   cached separately.  For example, ``f(3)`` and ``f(3.0)`` will always be
+   treated as distinct calls with distinct results.  If *typed* is false,
+   the implementation will usually but not always regard them as equivalent
+   calls and only cache a single result.
 
    The wrapped function is instrumented with a :func:`cache_parameters`
    function that returns a new :class:`dict` showing the values for *maxsize*
@@ -172,8 +173,7 @@ The :mod:`functools` module defines the following functions:
    To help measure the effectiveness of the cache and tune the *maxsize*
    parameter, the wrapped function is instrumented with a :func:`cache_info`
    function that returns a :term:`named tuple` showing *hits*, *misses*,
-   *maxsize* and *currsize*.  In a multi-threaded environment, the hits
-   and misses are approximate.
+   *maxsize* and *currsize*.
 
    The decorator also provides a :func:`cache_clear` function for clearing or
    invalidating the cache.
@@ -182,6 +182,9 @@ The :mod:`functools` module defines the following functions:
    :attr:`__wrapped__` attribute.  This is useful for introspection, for
    bypassing the cache, or for rewrapping the function with a different cache.
 
+   The cache keeps references to the arguments and return values until they age
+   out of the cache or until the cache is cleared.
+
    An `LRU (least recently used) cache
    <https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU)>`_
    works best when the most recent calls are the best predictors of upcoming