Merge branch 'master' into randrange-index

Author: serhiy-storchaka

.github/CODEOWNERS

@@ -19,6 +19,7 @@ Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
 Python/ceval.c                @markshannon
 Python/compile.c              @markshannon
+Python/ast_opt.c              @isidentical
 
 # Hashing
 **/*hashlib*                  @python/crypto-team @tiran
@@ -84,6 +85,12 @@ Include/pytime.h              @pganssle @abalkin
 /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 /Grammar/python.gram          @pablogsal @lysnikolaou
 
+# AST
+Python/ast.c                  @isidentical
+Parser/asdl.py                @isidentical
+Parser/asdl_c.py              @isidentical
+Lib/ast.py                    @isidentical
+
 # SQLite 3
 **/*sqlite*                   @berkerpeksag
 

.github/problem-matchers/msvc.json

@@ -0,0 +1,19 @@
+{
+    "__comment": "Taken from vscode's vs/workbench/contrib/tasks/common/problemMatcher.ts msCompile rule",
+    "problemMatcher": [
+      {
+        "owner": "msvc-problem-matcher",
+        "pattern": [
+          {
+            "regexp": "^(?:\\s+\\d+\\>)?([^\\s].*)\\((\\d+),?(\\d+)?(?:,\\d+,\\d+)?\\)\\s*:\\s+(error|warning|info)\\s+(\\w{1,2}\\d+)\\s*:\\s*(.*)$",
+            "file": 1,
+            "line": 2,
+            "column": 3,
+            "severity": 4,
+            "code": 5,
+            "message": 6
+          }
+        ]
+      }
+    ]
+  }

.github/workflows/build.yml

@@ -99,6 +99,8 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
     - uses: actions/checkout@v2
+    - name: Register MSVC problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
     - name: Build CPython
       run: .\PCbuild\build.bat -e -p x64
     - name: Display build info

Doc/library/itertools.rst

@@ -786,6 +786,18 @@ which incur interpreter overhead.
    def dotproduct(vec1, vec2):
        return sum(map(operator.mul, vec1, vec2))
 
+   def convolve(signal, kernel):
+       # See:  https://betterexplained.com/articles/intuitive-convolution/
+       # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur)
+       # convolve(data, [1, -1]) --> 1st finite difference (1st derivative)
+       # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative)
+       kernel = list(reversed(kernel))
+       n = len(kernel)
+       window = collections.deque([0] * n, maxlen=n)
+       for x in chain(signal, repeat(0, n-1)):
+           window.append(x)
+           yield sum(map(operator.mul, kernel, window))
+
    def flatten(list_of_lists):
        "Flatten one level of nesting"
        return chain.from_iterable(list_of_lists)

Doc/library/random.rst

@@ -145,7 +145,6 @@ Functions for integers
       or ``randrange('10')`` will be changed from :exc:`ValueError` to
       :exc:`TypeError`.
 
-
 .. function:: randint(a, b)
 
    Return a random integer *N* such that ``a <= N <= b``.  Alias for

Doc/library/sqlite3.rst

@@ -546,7 +546,7 @@ Connection Objects
          con.close()
 
 
-   .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250)
+   .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250)
 
       This method makes a backup of a SQLite database even while it's being accessed
       by other clients, or concurrently by the same connection.  The copy will be

Doc/library/subprocess.rst

@@ -1187,8 +1187,9 @@ calls these functions.
    The arguments shown above are merely some common ones.
    The full function signature is largely the same as that of :func:`run` -
    most arguments are passed directly through to that interface.
-   However, explicitly passing ``input=None`` to inherit the parent's
-   standard input file handle is not supported.
+   One API deviation from :func:`run` behavior exists: passing ``input=None``
+   will behave the same as ``input=b''`` (or ``input=''``, depending on other
+   arguments) rather than using the parent's standard input file handle.
 
    By default, this function will return the data as encoded bytes. The actual
    encoding of the output data may depend on the command being invoked, so the

Doc/library/tkinter.font.rst

@@ -91,6 +91,9 @@ The different font weights and slants are:
 
    Return the names of defined fonts.
 
-.. function:: nametofont(name)
+.. function:: nametofont(name, root=None)
 
-   Return a :class:`Font` representation of a tk named font.
+   Return a :class:`Font` representation of a tk named font.
+
+   .. versionchanged:: 3.10
+      The *root* parameter was added.

Doc/whatsnew/3.9.rst

@@ -1482,4 +1482,37 @@ and to match the behavior of static type checkers specified in the PEP.
         File "<stdin>", line 1, in <module>
       TypeError: unhashable type: 'set'
 
-(Contributed by Yurii Karabas in :issue:`42345`.)
+(Contributed by Yurii Karabas in :issue:`42345`.)
+
+macOS 11.0 (Big Sur) and Apple Silicon Mac support
+--------------------------------------------------
+
+As of 3.9.1, Python now fully supports building and running on macOS 11.0
+(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture).
+A new universal build variant, ``universal2``, is now available to natively
+support both ``ARM64`` and ``Intel 64`` in one set of executables. Binaries
+can also now be built on current versions of macOS to be deployed on a range
+of older macOS versions (tested to 10.9) while making some newer OS
+functions and options conditionally available based on the operating system
+version in use at runtime ("weaklinking").
+
+(Contributed by Ronald Oussoren and Lawrence D'Anna in :issue:`41100`.)
+
+Notable changes in Python 3.9.2
+===============================
+
+collections.abc
+---------------
+
+:class:`collections.abc.Callable` generic now flattens type parameters, similar
+to what :data:`typing.Callable` currently does.  This means that
+``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of
+``(int, str, str)``; previously this was ``([int, str], str)``.  To allow this
+change, :class:`types.GenericAlias` can now be subclassed, and a subclass will
+be returned when subscripting the :class:`collections.abc.Callable` type.
+Code which accesses the arguments via :func:`typing.get_args` or ``__args__``
+need to account for this change.  A :exc:`DeprecationWarning` may be emitted for
+invalid forms of parameterizing :class:`collections.abc.Callable` which may have
+passed silently in Python 3.9.1.  This :exc:`DeprecationWarning` will
+become a :exc:`TypeError` in Python 3.10.
+(Contributed by Ken Jin in :issue:`42195`.)

Include/cpython/object.h

@@ -35,12 +35,13 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
    _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*.
 */
 typedef struct _Py_Identifier {
-    struct _Py_Identifier *next;
     const char* string;
-    PyObject *object;
+    // Index in PyInterpreterState.unicode.ids.array. It is process-wide
+    // unique and must be initialized to -1.
+    Py_ssize_t index;
 } _Py_Identifier;
 
-#define _Py_static_string_init(value) { .next = NULL, .string = value, .object = NULL }
+#define _Py_static_string_init(value) { .string = value, .index = -1 }
 #define _Py_static_string(varname, value)  static _Py_Identifier varname = _Py_static_string_init(value)
 #define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname)
 

Include/internal/pycore_interp.h

@@ -64,13 +64,31 @@ struct _Py_bytes_state {
     PyBytesObject *characters[256];
 };
 
+struct _Py_unicode_ids {
+    Py_ssize_t size;
+    PyObject **array;
+};
+
 struct _Py_unicode_state {
     // The empty Unicode object is a singleton to improve performance.
     PyObject *empty_string;
     /* Single character Unicode strings in the Latin-1 range are being
        shared as well. */
     PyObject *latin1[256];
     struct _Py_unicode_fs_codec fs_codec;
+
+    /* This dictionary holds all interned unicode strings.  Note that references
+       to strings in this dictionary are *not* counted in the string's ob_refcnt.
+       When the interned string reaches a refcnt of 0 the string deallocation
+       function will delete the reference from this dictionary.
+
+       Another way to look at this is that to say that the actual reference
+       count of a string is:  s->ob_refcnt + (s->state ? 2 : 0)
+    */
+    PyObject *interned;
+
+    // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId()
+    struct _Py_unicode_ids ids;
 };
 
 struct _Py_float_state {
@@ -173,6 +191,27 @@ struct atexit_state {
 };
 
 
+// Type attribute lookup cache: speed up attribute and method lookups,
+// see _PyType_Lookup().
+struct type_cache_entry {
+    unsigned int version;  // initialized from type->tp_version_tag
+    PyObject *name;        // reference to exactly a str or None
+    PyObject *value;       // borrowed reference or NULL
+};
+
+#define MCACHE_SIZE_EXP 12
+#define MCACHE_STATS 0
+
+struct type_cache {
+    struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP];
+#if MCACHE_STATS
+    size_t hits;
+    size_t misses;
+    size_t collisions;
+#endif
+};
+
+
 /* interpreter state */
 
 #define _PY_NSMALLPOSINTS           257
@@ -277,6 +316,7 @@ struct _is {
     struct _Py_exc_state exc_state;
 
     struct ast_state ast;
+    struct type_cache type_cache;
 };
 
 extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);

Include/internal/pycore_object.h

@@ -27,6 +27,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
     return ((type->tp_flags & feature) != 0);
 }
 
+extern void _PyType_InitCache(PyInterpreterState *interp);
+
+
 /* Inline functions trading binary compatibility for speed:
    _PyObject_Init() is the fast version of PyObject_Init(), and
    _PyObject_InitVar() is the fast version of PyObject_InitVar().

Include/internal/pycore_pylifecycle.h

@@ -76,7 +76,7 @@ extern void _PyExc_Fini(PyThreadState *tstate);
 extern void _PyImport_Fini(void);
 extern void _PyImport_Fini2(void);
 extern void _PyGC_Fini(PyThreadState *tstate);
-extern void _PyType_Fini(void);
+extern void _PyType_Fini(PyThreadState *tstate);
 extern void _Py_HashRandomization_Fini(void);
 extern void _PyUnicode_Fini(PyThreadState *tstate);
 extern void _PyUnicode_ClearInterned(PyThreadState *tstate);

Include/internal/pycore_runtime.h

@@ -49,6 +49,11 @@ typedef struct _Py_AuditHookEntry {
     void *userData;
 } _Py_AuditHookEntry;
 
+struct _Py_unicode_runtime_ids {
+    PyThread_type_lock lock;
+    Py_ssize_t next_index;
+};
+
 /* Full Python runtime state */
 
 typedef struct pyruntimestate {
@@ -106,6 +111,8 @@ typedef struct pyruntimestate {
     void *open_code_userdata;
     _Py_AuditHookEntry *audit_hook_head;
 
+    struct _Py_unicode_runtime_ids unicode_ids;
+
     // XXX Consolidate globals found via the check-c-globals script.
 } _PyRuntimeState;
 

Lib/ast.py

@@ -63,7 +63,10 @@ def literal_eval(node_or_string):
     if isinstance(node_or_string, Expression):
         node_or_string = node_or_string.body
     def _raise_malformed_node(node):
-        raise ValueError(f'malformed node or string: {node!r}')
+        msg = "malformed node or string"
+        if lno := getattr(node, 'lineno', None):
+            msg += f' on line {lno}'
+        raise ValueError(msg + f': {node!r}')
     def _convert_num(node):
         if not isinstance(node, Constant) or type(node.value) not in (int, float, complex):
             _raise_malformed_node(node)

Lib/enum.py

@@ -178,7 +178,7 @@ class EnumMeta(type):
     Metaclass for Enum
     """
     @classmethod
-    def __prepare__(metacls, cls, bases):
+    def __prepare__(metacls, cls, bases, **kwds):
         # check that previous enum members do not exist
         metacls._check_for_existing_members(cls, bases)
         # create the namespace dict
@@ -235,10 +235,10 @@ def __new__(metacls, cls, bases, classdict, **kwds):
         # create our new Enum type
         if bases:
             bases = (_NoInitSubclass, ) + bases
-            enum_class = type.__new__(metacls, cls, bases, classdict)
+            enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
             enum_class.__bases__ = enum_class.__bases__[1:] #or (object, )
         else:
-            enum_class = type.__new__(metacls, cls, bases, classdict)
+            enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
         old_init_subclass = getattr(enum_class, '__init_subclass__', None)
         # and restore the new one (if there was one)
         if new_init_subclass is not None:

Lib/random.py

@@ -51,9 +51,9 @@
 from math import tau as TWOPI, floor as _floor, isfinite as _isfinite
 from os import urandom as _urandom
 from _collections_abc import Set as _Set, Sequence as _Sequence
+from operator import index as _index
 from itertools import accumulate as _accumulate, repeat as _repeat
 from bisect import bisect as _bisect
-from operator import index as _index
 import os as _os
 import _random
 

Lib/subprocess.py

@@ -420,7 +420,11 @@ def check_output(*popenargs, timeout=None, **kwargs):
     if 'input' in kwargs and kwargs['input'] is None:
         # Explicitly passing input=None was previously equivalent to passing an
         # empty string. That is maintained here for backwards compatibility.
-        kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
+        if kwargs.get('universal_newlines') or kwargs.get('text'):
+            empty = ''
+        else:
+            empty = b''
+        kwargs['input'] = empty
 
     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
                **kwargs).stdout

Lib/test/crashers/bogus_code_obj.py

@@ -14,6 +14,6 @@
 
 import types
 
-co = types.CodeType(0, 0, 0, 0, 0, b'\x04\x71\x00\x00',
+co = types.CodeType(0, 0, 0, 0, 0, 0, b'\x04\x00\x71\x00',
                     (), (), (), '', '', 1, b'')
 exec(co)

Lib/test/test_asdl_parser.py

@@ -1,6 +1,7 @@
 """Tests for the asdl parser in Parser/asdl.py"""
 
 import importlib.machinery
+import importlib.util
 import os
 from os.path import dirname
 import sys
@@ -26,7 +27,10 @@ def setUpClass(cls):
         sys.path.insert(0, parser_dir)
         loader = importlib.machinery.SourceFileLoader(
                 'asdl', os.path.join(parser_dir, 'asdl.py'))
-        cls.asdl = loader.load_module()
+        spec = importlib.util.spec_from_loader('asdl', loader)
+        module = importlib.util.module_from_spec(spec)
+        loader.exec_module(module)
+        cls.asdl = module
         cls.mod = cls.asdl.parse(os.path.join(parser_dir, 'Python.asdl'))
         cls.assertTrue(cls.asdl.check(cls.mod), 'Module validation failed')
 

Lib/test/test_ast.py

@@ -1011,6 +1011,18 @@ def test_literal_eval_trailing_ws(self):
         self.assertEqual(ast.literal_eval(" \t -1"), -1)
         self.assertRaises(IndentationError, ast.literal_eval, "\n -1")
 
+    def test_literal_eval_malformed_lineno(self):
+        msg = r'malformed node or string on line 3:'
+        with self.assertRaisesRegex(ValueError, msg):
+            ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}")
+
+        node = ast.UnaryOp(
+            ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6)))
+        self.assertIsNone(getattr(node, 'lineno', None))
+        msg = r'malformed node or string:'
+        with self.assertRaisesRegex(ValueError, msg):
+            ast.literal_eval(node)
+
     def test_bad_integer(self):
         # issue13436: Bad error message with invalid numeric values
         body = [ast.ImportFrom(module='time',