Merge remote-tracking branch 'upstream/main' into special_decref

Author: sweeneyde

.azure-pipelines/windows-release.yml

@@ -34,6 +34,10 @@ parameters:
   displayName: "Signature description"
   type: string
   default: 'Built: $(Build.BuildNumber)'
+- name: DoARM64
+  displayName: "Publish ARM64 build"
+  type: boolean
+  default: true
 - name: DoPGO
   displayName: "Run PGO"
   type: boolean
@@ -64,25 +68,26 @@ parameters:
   default: '0'
 
 variables:
-    __RealSigningCertificate: 'Python Software Foundation'
-    ${{ if ne(parameters.GitRemote, '(Other)') }}:
-      GitRemote: ${{ parameters.GitRemote }}
-    ${{ else }}:
-      GitRemote: ${{ parameters.GitRemote_Other }}
-    SourceTag: ${{ parameters.SourceTag }}
-    DoPGO: ${{ parameters.DoPGO }}
-    ${{ if ne(parameters.SigningCertificate, 'Unsigned') }}:
-      SigningCertificate: ${{ parameters.SigningCertificate }}
-    SigningDescription: ${{ parameters.SigningDescription }}
-    DoLayout: ${{ parameters.DoLayout }}
-    DoMSIX: ${{ parameters.DoMSIX }}
-    DoNuget: ${{ parameters.DoNuget }}
-    DoEmbed: ${{ parameters.DoEmbed }}
-    DoMSI: ${{ parameters.DoMSI }}
-    DoPublish: ${{ parameters.DoPublish }}
+  __RealSigningCertificate: 'Python Software Foundation'
+  ${{ if ne(parameters.GitRemote, '(Other)') }}:
+    GitRemote: ${{ parameters.GitRemote }}
+  ${{ else }}:
+    GitRemote: ${{ parameters.GitRemote_Other }}
+  SourceTag: ${{ parameters.SourceTag }}
+  DoPGO: ${{ parameters.DoPGO }}
+  ${{ if ne(parameters.SigningCertificate, 'Unsigned') }}:
+    SigningCertificate: ${{ parameters.SigningCertificate }}
+  SigningDescription: ${{ parameters.SigningDescription }}
+  DoLayout: ${{ parameters.DoLayout }}
+  DoMSIX: ${{ parameters.DoMSIX }}
+  DoNuget: ${{ parameters.DoNuget }}
+  DoEmbed: ${{ parameters.DoEmbed }}
+  DoMSI: ${{ parameters.DoMSI }}
+  DoPublish: ${{ parameters.DoPublish }}
+  PublishARM64: ${{ parameters.DoARM64 }}
 # QUEUE TIME VARIABLES
-#    PyDotOrgUsername: ''
-#    PyDotOrgServer: ''
+#  PyDotOrgUsername: ''
+#  PyDotOrgServer: ''
 
 trigger: none
 pr: none

.azure-pipelines/windows-release/msi-steps.yml

@@ -45,6 +45,20 @@ steps:
       artifactName: bin_amd64_d
       targetPath: $(Build.BinariesDirectory)\amd64
 
+  - task: DownloadPipelineArtifact@1
+    displayName: 'Download artifact: bin_arm64'
+    condition: and(succeeded(), eq(variables['PublishARM64'], 'true'))
+    inputs:
+      artifactName: bin_arm64
+      targetPath: $(Build.BinariesDirectory)\arm64
+
+  - task: DownloadPipelineArtifact@1
+    displayName: 'Download artifact: bin_arm64_d'
+    condition: and(succeeded(), eq(variables['PublishARM64'], 'true'))
+    inputs:
+      artifactName: bin_arm64_d
+      targetPath: $(Build.BinariesDirectory)\arm64
+
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: tcltk_lib_win32'
     inputs:
@@ -84,32 +98,45 @@ steps:
 
   - script: |
       %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true
-      %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false
     displayName: 'Build win32 installer'
     env:
       Platform: x86
       Py_OutDir: $(Build.BinariesDirectory)
       PYTHON: $(Build.BinariesDirectory)\win32\python.exe
+      PythonForBuild: $(Build.BinariesDirectory)\win32\python.exe
       PYTHONHOME: $(Build.SourcesDirectory)
       TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32
       BuildForRelease: true
       SuppressMinGWLib: true
 
   - script: |
       %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true
-      %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false
     displayName: 'Build amd64 installer'
     env:
       Platform: x64
       Py_OutDir: $(Build.BinariesDirectory)
       PYTHON: $(Build.BinariesDirectory)\amd64\python.exe
+      PythonForBuild: $(Build.BinariesDirectory)\amd64\python.exe
       PYTHONHOME: $(Build.SourcesDirectory)
       TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64
       BuildForRelease: true
       SuppressMinGWLib: true
 
+  - script: |
+      %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true
+    displayName: 'Build arm64 installer'
+    condition: and(succeeded(), eq(variables['PublishARM64'], 'true'))
+    env:
+      Platform: ARM64
+      Py_OutDir: $(Build.BinariesDirectory)
+      PYTHON: $(Build.BinariesDirectory)\win32\python.exe
+      PythonForBuild: $(Build.BinariesDirectory)\win32\python.exe
+      PYTHONHOME: $(Build.SourcesDirectory)
+      BuildForRelease: true
+      SuppressMinGWLib: true
+
   - task: CopyFiles@2
-    displayName: 'Assemble artifact: msi (1/2)'
+    displayName: 'Assemble artifact: msi (win32)'
     inputs:
       sourceFolder: $(Build.BinariesDirectory)\win32\en-us
       targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32
@@ -119,7 +146,7 @@ steps:
         *.exe
 
   - task: CopyFiles@2
-    displayName: 'Assemble artifact: msi (2/2)'
+    displayName: 'Assemble artifact: msi (amd64)'
     inputs:
       sourceFolder: $(Build.BinariesDirectory)\amd64\en-us
       targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64
@@ -128,6 +155,17 @@ steps:
         *.cab
         *.exe
 
+  - task: CopyFiles@2
+    displayName: 'Assemble artifact: msi (arm64)'
+    condition: and(succeeded(), eq(variables['PublishARM64'], 'true'))
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\arm64\en-us
+      targetFolder: $(Build.ArtifactStagingDirectory)\msi\arm64
+      contents: |
+        *.msi
+        *.cab
+        *.exe
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish MSI'
     inputs:

.azure-pipelines/windows-release/stage-publish-nugetorg.yml

@@ -36,9 +36,9 @@ jobs:
 
 
   - powershell: 'gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del'
-    displayName: 'Prevent publishing ARM/ARM64 packages'
+    displayName: 'Prevent publishing ARM64 packages'
     workingDirectory: '$(Build.BinariesDirectory)\nuget'
-    condition: and(succeeded(), not(variables['PublishArmPackages']))
+    condition: and(succeeded(), eq(variables['PublishARM64'], 'true'))
 
   - task: NuGetCommand@2
     displayName: Push packages

.azure-pipelines/windows-release/stage-publish-pythonorg.yml

@@ -77,10 +77,11 @@ jobs:
         downloadPath: $(Build.BinariesDirectory)
 
 
+  # Note that ARM64 MSIs are skipped at build when this option is specified
   - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del'
-    displayName: 'Prevent publishing ARM/ARM64 packages'
+    displayName: 'Prevent publishing ARM64 packages'
     workingDirectory: '$(Build.BinariesDirectory)\embed'
-    condition: and(succeeded(), not(variables['PublishArmPackages']))
+    condition: and(succeeded(), eq(variables['PublishARM64'], 'true'))
 
 
   - template: ./gpg-sign.yml

.azure-pipelines/windows-release/stage-test-msi.yml

@@ -11,19 +11,19 @@ jobs:
   strategy:
     matrix:
       win32_User:
-        ExeMatch: 'python-[\dabrc.]+-webinstall\.exe'
+        ExeMatch: 'python-[\dabrc.]+\.exe'
         Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User
         InstallAllUsers: 0
       win32_Machine:
-        ExeMatch: 'python-[\dabrc.]+-webinstall\.exe'
+        ExeMatch: 'python-[\dabrc.]+\.exe'
         Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine
         InstallAllUsers: 1
       amd64_User:
-        ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe'
+        ExeMatch: 'python-[\dabrc.]+-amd64\.exe'
         Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User
         InstallAllUsers: 0
       amd64_Machine:
-        ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe'
+        ExeMatch: 'python-[\dabrc.]+-amd64\.exe'
         Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine
         InstallAllUsers: 1
 

.github/workflows/build.yml

@@ -100,6 +100,8 @@ jobs:
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
+      - name: Check global objects
+        run: make check-global-objects
 
   build_win32:
     name: 'Windows (x86)'
@@ -311,4 +313,7 @@ jobs:
       #
       # Skip multiprocessing and concurrent.futures tests which are affected by
       # bpo-45200 bug: libasan dead lock in pthread_create().
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu -x test_ctypes test_crypt test_decimal test_faulthandler test_interpreters test___all__ test_idle test_tix test_tk test_ttk_guionly test_ttk_textonly test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_spawn test_tools test_peg_generator test_concurrent_futures"
+      #
+      # test___all__ is skipped because importing some modules directly can trigger
+      # known problems with ASAN (like tk or crypt).
+      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu -x test___all__ test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_spawn test_tools test_peg_generator test_concurrent_futures"

.github/workflows/build_msi.yml

@@ -37,3 +37,11 @@ jobs:
     - uses: actions/checkout@v2
     - name: Build CPython installer
       run: .\Tools\msi\build.bat -x64
+
+  build_win_arm64:
+    name: 'Windows (ARM64) Installer'
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build CPython installer
+      run: .\Tools\msi\build.bat -arm64

.github/workflows/stale.yml

@@ -9,6 +9,7 @@ permissions:
 
 jobs:
   stale:
+    if: github.repository_owner == 'python'
 
     runs-on: ubuntu-latest
 

Doc/Makefile

@@ -7,11 +7,13 @@
 PYTHON       = python3
 VENVDIR      = ./venv
 SPHINXBUILD  = PATH=$(VENVDIR)/bin:$$PATH sphinx-build
+SPHINXLINT   = PATH=$(VENVDIR)/bin:$$PATH sphinx-lint
 BLURB        = PATH=$(VENVDIR)/bin:$$PATH blurb
 PAPER        =
 SOURCES      =
 DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
 SPHINXERRORHANDLING = -W
+SERVE_PORT   =
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_elements.papersize=a4paper
@@ -213,11 +215,11 @@ dist:
 	rm dist/python-$(DISTVERSION)-docs-texinfo.tar
 
 check:
-	$(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst
-	$(PYTHON) tools/rstlint.py ../Misc/NEWS.d/next/
+	$(SPHINXLINT) -i tools -i $(VENVDIR) -i README.rst
+	$(SPHINXLINT) ../Misc/NEWS.d/next/
 
 serve:
-	$(PYTHON) ../Tools/scripts/serve.py build/html
+	$(PYTHON) ../Tools/scripts/serve.py build/html $(SERVE_PORT)
 
 # Targets for daily automated doc build
 # By default, Sphinx only rebuilds pages where the page content has changed.

Doc/README.rst

@@ -91,7 +91,7 @@ Available make targets are:
 
 * "pydoc-topics", which builds a Python module containing a dictionary with
   plain text documentation for the labels defined in
-  `tools/pyspecific.py` -- pydoc needs these to show topic and keyword help.
+  ``tools/pyspecific.py`` -- pydoc needs these to show topic and keyword help.
 
 * "suspicious", which checks the parsed markup for text that looks like
   malformed and thus unconverted reST.

Doc/c-api/buffer.rst

@@ -499,6 +499,13 @@ Buffer-related functions
    This function fails if *len* != *src->len*.
 
 
+.. c:function:: int PyObject_CopyData(Py_buffer *dest, Py_buffer *src)
+
+   Copy data from *src* to *dest* buffer. Can convert between C-style and
+   or Fortran-style buffers.
+
+   ``0`` is returned on success, ``-1`` on error.
+
 .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order)
 
    Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if

Doc/c-api/import.rst

@@ -256,8 +256,12 @@ Importing Modules
           const char *name;
           const unsigned char *code;
           int size;
+          bool is_package;
       };
 
+   .. versionchanged:: 3.11
+      The new ``is_package`` field indicates whether the module is a package or not.
+      This replaces setting the ``size`` field to a negative value.
 
 .. c:var:: const struct _frozen* PyImport_FrozenModules
 

Doc/c-api/type.rst

@@ -149,6 +149,8 @@ Type Objects
    ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses
    are not necessarily defined in the same module as their superclass.
    See :c:type:`PyCMethod` to get the class that defines the method.
+   See :c:func:`PyType_GetModuleByDef` for cases when ``PyCMethod`` cannot
+   be used.
 
    .. versionadded:: 3.9
 
@@ -166,6 +168,21 @@ Type Objects
 
    .. versionadded:: 3.9
 
+.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def)
+
+   Find the first superclass whose module was created from
+   the given :c:type:`PyModuleDef` *def*, and return that module.
+
+   If no module is found, raises a :py:class:`TypeError` and returns ``NULL``.
+
+   This function is intended to be used together with
+   :c:func:`PyModule_GetState()` to get module state from slot methods (such as
+   :c:member:`~PyTypeObject.tp_init` or :c:member:`~PyNumberMethods.nb_add`)
+   and other places where a method's defining class cannot be passed using the
+   :c:type:`PyCMethod` calling convention.
+
+   .. versionadded:: 3.11
+
 
 Creating Heap-Allocated Types
 .............................
@@ -272,12 +289,6 @@ The following functions and structs are used to create
       * :c:member:`~PyTypeObject.tp_vectorcall_offset`
         (see :ref:`PyMemberDef <pymemberdef-offsets>`)
 
-      The following fields cannot be set using :c:type:`PyType_Spec` and
-      :c:type:`PyType_Slot` under the limited API:
-
-      * :c:member:`~PyBufferProcs.bf_getbuffer`
-      * :c:member:`~PyBufferProcs.bf_releasebuffer`
-
       Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be
       problematic on some platforms.
       To avoid issues, use the *bases* argument of
@@ -287,6 +298,11 @@ The following functions and structs are used to create
 
         Slots in :c:type:`PyBufferProcs` in may be set in the unlimited API.
 
+     .. versionchanged:: 3.11
+        :c:member:`~PyBufferProcs.bf_getbuffer` and
+        :c:member:`~PyBufferProcs.bf_releasebuffer` are now available
+        under limited API.
+
    .. c:member:: void *PyType_Slot.pfunc
 
       The desired value of the slot. In most cases, this is a pointer