Merge branch 'main' into sqlite-cleanup-init

Author: Erlend E. Aasland

.azure-pipelines/find-tools.yml

@@ -0,0 +1,26 @@
+# Locate a set of the tools used for builds
+
+steps:
+  - template: windows-release/find-sdk.yml
+    parameters:
+      toolname: 'signtool.exe'
+
+  - powershell: |
+      $vcvarsall = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" `
+          -prerelease `
+          -latest `
+          -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
+          -find VC\Auxiliary\Build\vcvarsall.bat)
+      Write-Host "Found vcvarsall at $vcvarsall"
+      Write-Host "##vso[task.setVariable variable=vcvarsall]$vcvarsall"
+    displayName: 'Find vcvarsall.bat'
+
+  - powershell: |
+      $msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" `
+          -prerelease `
+          -latest `
+          -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
+          -find MSBuild\Current\Bin\msbuild.exe)
+      Write-Host "Found MSBuild at $msbuild"
+      Write-Host "##vso[task.setVariable variable=msbuild]$msbuild"
+    displayName: 'Find MSBuild'

.azure-pipelines/libffi-build.yml

@@ -0,0 +1,86 @@
+name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+  IntDir: '$(Build.BinariesDirectory)'
+  OutDir: '$(Build.ArtifactStagingDirectory)'
+
+  # MUST BE SET AT QUEUE TIME
+  # SigningCertificate: 'Python Software Foundation'
+  # SourcesRepo: 'https://github.com/python/cpython-source-deps'
+  # SourceTag: 'libffi-3.4.2'
+
+jobs:
+- job: Build_LibFFI
+  displayName: LibFFI
+  pool:
+    vmImage: windows-latest
+
+  workspace:
+    clean: all
+
+  steps:
+    - checkout: none
+
+    - template: ./find-tools.yml
+
+    - powershell: |
+       mkdir -Force "$(IntDir)\script"
+       iwr "https://github.com/python/cpython/raw/main/PCbuild/prepare_libffi.bat" `
+           -outfile "$(IntDir)\script\prepare_libffi.bat"
+      displayName: 'Download build script'
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(SourceTag) --depth 1 -c core.autocrlf=false -c core.eol=lf .
+      displayName: 'Check out LibFFI sources'
+
+    - script: 'prepare_libffi.bat --install-cygwin'
+      workingDirectory: '$(IntDir)\script'
+      displayName: 'Install Cygwin and build'
+      env:
+        VCVARSALL: '$(vcvarsall)'
+        LIBFFI_SOURCE: '$(Build.SourcesDirectory)'
+        LIBFFI_OUT: '$(OutDir)'
+
+    - powershell: |
+       if ((gci *\*.dll).Count -lt 4) {
+           Write-Error "Did not generate enough DLL files"
+       }
+       if ((gci *\Include\ffi.h).Count -lt 4) {
+           Write-Error "Did not generate enough include files"
+       }
+      failOnStderr: true
+      workingDirectory: '$(OutDir)'
+      displayName: 'Verify files were created'
+
+    - publish: '$(OutDir)'
+      artifact: 'unsigned'
+      displayName: 'Publish unsigned build'
+
+- job: Sign_LibFFI
+  displayName: Sign LibFFI
+  dependsOn: Build_LibFFI
+  pool:
+    name: 'Windows Release'
+
+  workspace:
+    clean: all
+
+  steps:
+    - checkout: none
+    - download: current
+      artifact: unsigned
+
+    - template: ./find-tools.yml
+
+    - powershell: |
+        signtool sign /q /a `
+            /n "Python Software Foundation" `
+            /fd sha256 `
+            /tr http://timestamp.digicert.com/ /td sha256 `
+            /d "LibFFI for Python" `
+            (gci "$(Pipeline.Workspace)\unsigned\*.dll" -r)
+      displayName: 'Sign files'
+
+    - publish: '$(Pipeline.Workspace)\unsigned'
+      artifact: 'libffi'
+      displayName: 'Publish libffi'

.azure-pipelines/openssl-build.yml

@@ -0,0 +1,110 @@
+name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+  IntDir: '$(Build.BinariesDirectory)'
+  OutDir: '$(Build.ArtifactStagingDirectory)'
+
+  # MUST BE SET AT QUEUE TIME
+  # SigningCertificate: 'Python Software Foundation'
+  # SourcesRepo: 'https://github.com/python/cpython-source-deps'
+  # SourceTag: 'openssl-1.1.1k'
+
+jobs:
+- job: Build_SSL
+  displayName: OpenSSL
+  pool:
+    name: 'Windows Release'
+    #vmImage: windows-latest
+
+  strategy:
+    matrix:
+      win32:
+        Platform: 'win32'
+        VCPlatform: 'amd64_x86'
+        OpenSSLPlatform: 'VC-WIN32 no-asm'
+      amd64:
+        Platform: 'amd64'
+        VCPlatform: 'amd64'
+        OpenSSLPlatform: 'VC-WIN64A-masm'
+      arm32:
+        Platform: 'arm32'
+        VCPlatform: 'amd64_arm'
+        OpenSSLPlatform: 'VC-WIN32-ARM'
+      arm64:
+        Platform: 'arm64'
+        VCPlatform: 'amd64_arm64'
+        OpenSSLPlatform: 'VC-WIN64-ARM'
+
+  workspace:
+    clean: all
+
+  steps:
+    - checkout: none
+
+    - template: ./find-tools.yml
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(SourceTag) --depth 1 .
+      displayName: 'Check out OpenSSL sources'
+
+    - powershell: |
+        $f = gi ms\uplink.c
+        $c1 = gc $f
+        $c2 = $c1 -replace '\(\(h = GetModuleHandle\(NULL\)\) == NULL\)', '((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL /*patched*/)'
+        if ($c2 -ne $c1) {
+            $c2 | Out-File $f -Encoding ASCII
+        } else {
+            Write-Host '##warning Failed to patch uplink.c'
+        }
+      displayName: 'Apply uplink.c patch'
+
+    - script: |
+        call "$(vcvarsall)" $(VCPlatform)
+        perl "$(Build.SourcesDirectory)\Configure" $(OpenSSLPlatform)
+        nmake
+      workingDirectory: '$(IntDir)'
+      displayName: 'Build OpenSSL'
+
+    - script: |
+        call "$(vcvarsall)" $(VCPlatform)
+        signtool sign /q /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "OpenSSL for Python" *.dll
+      workingDirectory: '$(IntDir)'
+      displayName: 'Sign OpenSSL Build'
+      condition: and(succeeded(), variables['SigningCertificate'])
+
+    - task: CopyFiles@2
+      displayName: 'Copy built libraries for upload'
+      inputs:
+        SourceFolder: '$(IntDir)'
+        Contents: |
+          lib*.dll
+          lib*.pdb
+          lib*.lib
+          include\openssl\*.h
+        TargetFolder: '$(OutDir)'
+
+    - task: CopyFiles@2
+      displayName: 'Copy header files for upload'
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)'
+        Contents: |
+          include\openssl\*
+        TargetFolder: '$(OutDir)'
+
+    - task: CopyFiles@2
+      displayName: 'Copy applink files for upload'
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)\ms'
+        Contents: applink.c
+        TargetFolder: '$(OutDir)\include'
+
+    - task: CopyFiles@2
+      displayName: 'Copy LICENSE for upload'
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)'
+        Contents: LICENSE
+        TargetFolder: '$(OutDir)'
+
+    - publish: '$(OutDir)'
+      artifact: '$(Platform)'
+      displayName: 'Publishing $(Platform)'

.azure-pipelines/tcltk-build.yml

@@ -0,0 +1,65 @@
+name: tcl$(TkSourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+  IntDir: '$(Build.BinariesDirectory)\obj'
+  ExternalsDir: '$(Build.BinariesDirectory)\externals'
+  OutDir: '$(Build.ArtifactStagingDirectory)'
+  Configuration: 'Release'
+  
+  # MUST BE SET AT QUEUE TIME
+  # SigningCertificate: 'Python Software Foundation'
+  # SourcesRepo: 'https://github.com/python/cpython-source-deps'
+  # TclSourceTag: 'tcl-core-8.6.12.0'
+  # TkSourceTag: 'tk-8.6.12.0'
+  # TixSourceTag: 'tix-8.4.3.6'
+
+jobs:
+- job: Build_TclTk
+  displayName: 'Tcl/Tk'
+  pool:
+    name: 'Windows Release'
+    #vmImage: windows-latest
+
+  workspace:
+    clean: all
+
+  steps:
+    - template: ./find-tools.yml
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(TclSourceTag) --depth 1 "$(ExternalsDir)\$(TclSourceTag)"
+      displayName: 'Check out Tcl sources'
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(TkSourceTag) --depth 1 "$(ExternalsDir)\$(TkSourceTag)"
+      displayName: 'Check out Tk sources'
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(TixSourceTag) --depth 1 "$(ExternalsDir)\$(TixSourceTag)"
+      displayName: 'Check out Tix sources'
+
+    # This msbuild.rsp file will be used by the build to forcibly override these variables
+    - powershell: |
+        del -Force -EA 0 msbuild.rsp
+        "/p:IntDir=$(IntDir)\" >> msbuild.rsp
+        "/p:ExternalsDir=$(ExternalsDir)\" >> msbuild.rsp
+        "/p:tclDir=$(ExternalsDir)\$(TclSourceTag)\" >> msbuild.rsp
+        "/p:tkDir=$(ExternalsDir)\$(TkSourceTag)\" >> msbuild.rsp
+        "/p:tixDir=$(ExternalsDir)\$(TixSourceTag)\" >> msbuild.rsp
+      displayName: 'Generate msbuild.rsp'
+
+    - powershell: |
+        & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32"
+        & "$(msbuild)" PCbuild\tk.vcxproj  "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32"
+        & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32"
+      displayName: 'Build for win32'
+
+    - powershell: |
+        & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64"
+        & "$(msbuild)" PCbuild\tk.vcxproj  "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64"
+        & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64"
+      displayName: 'Build for amd64'
+
+    - publish: '$(OutDir)'
+      artifact: 'tcltk'
+      displayName: 'Publishing tcltk'

.gitattributes

@@ -46,6 +46,7 @@ Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
+Python/deepfreeze/*.c       linguist-generated=true
 Python/frozen_modules/*.h   linguist-generated=true
 Python/frozen_modules/MANIFEST  linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true

.github/workflows/posix-deps-apt.sh

@@ -3,6 +3,7 @@ apt-get update
 
 apt-get -yq install \
     build-essential \
+    pkg-config \
     ccache \
     gdb \
     lcov \

.gitignore

@@ -59,8 +59,18 @@ Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
 !Lib/test/data/README
+/_bootstrap_python
 /Makefile
 /Makefile.pre
+Mac/Makefile
+Mac/PythonLauncher/Info.plist
+Mac/PythonLauncher/Makefile
+Mac/PythonLauncher/Python Launcher
+Mac/PythonLauncher/Python Launcher.app/*
+Mac/Resources/app/Info.plist
+Mac/Resources/framework/Info.plist
+Mac/pythonw
+/*.framework/
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
@@ -106,6 +116,7 @@ Tools/unicode/data/
 /platform
 /profile-clean-stamp
 /profile-run-stamp
+/Python/deepfreeze/*.c
 /pybuilddir.txt
 /pyconfig.h
 /python-config
@@ -120,6 +131,7 @@ Tools/unicode/data/
 Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
+Tools/freeze/test/outdir
 
 # The frozen modules are always generated by the build so we don't
 # keep them in the repo.  Also see Tools/scripts/freeze_modules.py.

Doc/c-api/gcsupport.rst

@@ -33,6 +33,14 @@ Constructors for container types must conform to two rules:
 #. Once all the fields which may contain references to other containers are
    initialized, it must call :c:func:`PyObject_GC_Track`.
 
+Similarly, the deallocator for the object must conform to a similar pair of
+rules:
+
+#. Before fields which refer to other containers are invalidated,
+   :c:func:`PyObject_GC_UnTrack` must be called.
+
+#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`.
+
    .. warning::
       If a type adds the Py_TPFLAGS_HAVE_GC, then it *must* implement at least
       a :c:member:`~PyTypeObject.tp_traverse` handler or explicitly use one
@@ -100,14 +108,6 @@ Constructors for container types must conform to two rules:
 
    .. versionadded:: 3.9
 
-Similarly, the deallocator for the object must conform to a similar pair of
-rules:
-
-#. Before fields which refer to other containers are invalidated,
-   :c:func:`PyObject_GC_UnTrack` must be called.
-
-#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`.
-
 
 .. c:function:: void PyObject_GC_Del(void *op)
 

Doc/c-api/typeobj.rst

@@ -668,6 +668,18 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    :c:func:`PyObject_GC_Del` if the instance was allocated using
    :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`.
 
+   If the type supports garbage collection (has the :const:`Py_TPFLAGS_HAVE_GC`
+   flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack`
+   before clearing any member fields.
+
+   .. code-block:: c
+
+     static void foo_dealloc(foo_object *self) {
+         PyObject_GC_UnTrack(self);
+         Py_CLEAR(self->ref);
+         Py_TYPE(self)->tp_free((PyObject *)self);
+     }
+
    Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the
    deallocator should decrement the reference count for its type object after
    calling the type deallocator. In order to avoid dangling pointers, the

Doc/data/stable_abi.dat

@@ -189,6 +189,7 @@ var,PyExc_ArithmeticError,3.2,
 var,PyExc_AssertionError,3.2,
 var,PyExc_AttributeError,3.2,
 var,PyExc_BaseException,3.2,
+var,PyExc_BaseExceptionGroup,3.11,
 var,PyExc_BlockingIOError,3.7,
 var,PyExc_BrokenPipeError,3.7,
 var,PyExc_BufferError,3.2,