Merge main branch

Author: encukou

.azure-pipelines/ci.yml

@@ -1,14 +1,14 @@
 variables:
   coverage: false
 
-trigger: ['main', '3.9', '3.8', '3.7']
+trigger: ['main', '3.10', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -20,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   steps:
   - template: ./docs-steps.yml
@@ -40,7 +40,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: macos-10.14
+    vmImage: macos-10.15
 
   steps:
   - template: ./macos-steps.yml
@@ -52,12 +52,12 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1k
+    openssl_version: 1.1.1l
 
   steps:
   - template: ./posix-steps.yml
@@ -78,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1k
+    openssl_version: 1.1.1l
 
   steps:
   - template: ./posix-steps.yml

.azure-pipelines/pr.yml

@@ -1,14 +1,14 @@
 variables:
   coverage: false
 
-pr: ['main', '3.9', '3.8', '3.7']
+pr: ['main', '3.10', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -20,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   steps:
   - template: ./docs-steps.yml
@@ -38,7 +38,7 @@ jobs:
     testRunPlatform: macos
 
   pool:
-    vmImage: macos-10.14
+    vmImage: macos-10.15
 
   steps:
   - template: ./macos-steps.yml
@@ -52,12 +52,12 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1k
+    openssl_version: 1.1.1l
 
   steps:
   - template: ./posix-steps.yml
@@ -78,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-18.04
+    vmImage: ubuntu-20.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1k
+    openssl_version: 1.1.1l
 
   steps:
   - template: ./posix-steps.yml

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*.{py,c,cpp,h,rst,md,yml}]
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+
+[*.{py,c,cpp,h}]
+indent_size = 4
+
+[*.yml]
+indent_size = 2

.gitattributes

@@ -28,6 +28,7 @@ Lib/test/test_email/data/*.txt -text
 Lib/test/xmltestdata/* -text
 Lib/test/coding20731.py -text
 Lib/test/test_importlib/data01/* -text
+Lib/test/test_importlib/namespacedata01/* -text
 
 # CRLF files
 *.bat text eol=crlf
@@ -45,8 +46,8 @@ Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
-Python/importlib.h          linguist-generated=true
-Python/importlib_external.h linguist-generated=true
+Python/frozen_modules/*.h   linguist-generated=true
+Python/frozen_modules/MANIFEST  linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true

.github/CODEOWNERS

@@ -53,11 +53,11 @@ Python/ast_opt.c              @isidentical
 
 
 # SSL
-**/*ssl*                      @python/crypto-team @tiran
-**/*.pem                      @python/crypto-team @tiran
+**/*ssl*                      @python/crypto-team
+**/*.pem                      @python/crypto-team
 
 # CSPRNG
-Python/bootstrap_hash.c       @python/crypto-team @tiran
+Python/bootstrap_hash.c       @python/crypto-team
 
 # Dates and times
 **/*datetime*                 @pganssle @abalkin
@@ -128,7 +128,7 @@ Lib/ast.py                    @isidentical
 
 **/*idlelib*                  @terryjreedy
 
-**/*typing*                   @gvanrossum @ilevkivskyi
+**/*typing*                   @gvanrossum @Fidget-Spinner
 
 **/*asyncore                  @giampaolo
 **/*asynchat                  @giampaolo

.github/SECURITY.md

@@ -14,4 +14,4 @@ official website](https://www.python.org/dev/security/) for
 instructions on how to report a security-related problem to
 the Python team responsibly.
 
-To reach the response team, email `security at python dot org`. 
+To reach the response team, email `security at python dot org`.

.github/problem-matchers/sphinx.json

@@ -34,7 +34,7 @@
                     "line": 2,
                     "message": 3
                 }
-            ]           
+            ]
         }
     ]
 }

.github/workflows/build.yml

@@ -71,18 +71,23 @@ jobs:
           make regen-stdlib-module-names
       - name: Check for changes
         run: |
+          git add -u
           changes=$(git status --porcelain)
           # Check for changes in regenerated files
           if ! test -z "$changes"
           then
-            echo "Generated files not up to date. Perhaps you forgot to run make regen-all ;)"
+            echo "Generated files not up to date. Perhaps you forgot to run make regen-all or build.bat --regen ;)"
             echo "$changes"
             exit 1
           fi
       - name: Check exported libpython symbols
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
+      - name: Check Autoconf version 2.69
+        run: |
+          grep "Generated by GNU Autoconf 2.69" configure
+          grep "PKG_PROG_PKG_CONFIG" aclocal.m4
 
   build_win32:
     name: 'Windows (x86)'
@@ -119,10 +124,16 @@ jobs:
     runs-on: macos-latest
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
+    env:
+      PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v2
+    - name: Prepare homebrew environment variables
+      run: |
+        echo "LDFLAGS=-L$(brew --prefix tcl-tk)/lib" >> $GITHUB_ENV
+        echo "PKG_CONFIG_PATH=$(brew --prefix [email protected])/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV
     - name: Configure CPython
-      run: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev
+      run: ./configure --with-pydebug --prefix=/opt/python-dev
     - name: Build CPython
       run: make -j4
     - name: Display build info
@@ -136,7 +147,8 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1k
+      OPENSSL_VER: 1.1.1l
+      PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v2
     - name: Register gcc problem matcher
@@ -150,7 +162,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/[email protected].5
+      uses: actions/[email protected].6
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -179,7 +191,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1k, 3.0.0-alpha17]
+        openssl_ver: [1.1.1l, 3.0.0]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -198,7 +210,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/[email protected].5
+      uses: actions/[email protected].6
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -218,3 +230,47 @@ jobs:
       run: make pythoninfo
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
+
+
+  build_asan:
+    name: 'Address sanitizer'
+    runs-on: ubuntu-20.04
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    env:
+      OPENSSL_VER: 1.1.1l
+      PYTHONSTRICTEXTENSIONBUILD: 1
+      ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
+    steps:
+    - uses: actions/checkout@v2
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/[email protected]
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/ccache-action@v1
+    - name: Configure CPython
+      run: ./configure --with-address-sanitizer --without-pymalloc
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu -x test_ctypes test_crypt test_decimal test_faulthandler test_interpreters test___all__ test_idle test_tix test_tk test_ttk_guionly test_ttk_textonly test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_spawn"

.github/workflows/doc.yml

@@ -38,9 +38,9 @@ jobs:
     - name: 'Install build dependencies'
       run: make -C Doc/ PYTHON=../python venv
     - name: 'Build documentation'
-      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html suspicious
+      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html
     - name: 'Upload'
-      uses: actions/[email protected].3
+      uses: actions/[email protected].4
       with:
         name: doc-html
         path: Doc/build/html

.github/workflows/posix-deps-apt.sh

@@ -9,6 +9,7 @@ apt-get -yq install \
     libbz2-dev \
     libffi-dev \
     libgdbm-dev \
+    libgdbm-compat-dev \
     liblzma-dev \
     libncurses5-dev \
     libreadline6-dev \

.github/workflows/stale.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v4
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.'

.gitignore

@@ -35,6 +35,7 @@ gmon.out
 .coverage
 .mypy_cache/
 .pytest_cache/
+.DS_Store
 
 *.exe
 !Lib/distutils/command/*.exe
@@ -67,7 +68,7 @@ Modules/Setup.config
 Modules/Setup.local
 Modules/config.c
 Modules/ld_so_aix
-Programs/_freeze_importlib
+Programs/_freeze_module
 Programs/_testembed
 PC/python_nt*.h
 PC/pythonnt_rc*.h
@@ -120,6 +121,12 @@ Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
 
+# The frozen modules are always generated by the build so we don't
+# keep them in the repo.  Also see Tools/scripts/freeze_modules.py.
+Python/frozen_modules/*.h
+# The manifest can be generated at any time with "make regen-frozen".
+Python/frozen_modules/MANIFEST
+
 # Two-trick pony for OSX and other case insensitive file systems:
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python