Merge branch 'main' into incremental-gc-3

Author: markshannon

.cirrus-DISABLED.yml

@@ -1,6 +1,6 @@
 root = true
 
-[*.{py,c,cpp,h,rst,md,yml}]
+[*.{py,c,cpp,h,js,rst,md,yml}]
 trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = space
@@ -11,5 +11,5 @@ indent_size = 4
 [*.rst]
 indent_size = 3
 
-[*.yml]
+[*.{js,yml}]
 indent_size = 2

.editorconfig

@@ -77,6 +77,7 @@ Include/internal/pycore_opcode.h                    generated
 Include/internal/pycore_opcode_metadata.h           generated
 Include/internal/pycore_*_generated.h               generated
 Include/internal/pycore_uop_ids.h                   generated
+Include/internal/pycore_uop_metadata.h              generated
 Include/opcode.h                                    generated
 Include/opcode_ids.h                                generated
 Include/token.h                                     generated
@@ -94,6 +95,7 @@ Programs/test_frozenmain.h                          generated
 Python/Python-ast.c                                 generated
 Python/executor_cases.c.h                           generated
 Python/generated_cases.c.h                          generated
+Python/optimizer_cases.c.h                          generated
 Python/opcode_targets.h                             generated
 Python/stdlib_module_names.h                        generated
 Tools/peg_generator/pegen/grammar_parser.py         generated

.gitattributes

@@ -21,6 +21,7 @@ configure*                    @erlend-aasland @corona10
 **/*context*                  @1st1
 **/*genobject*                @markshannon
 **/*hamt*                     @1st1
+**/*jit*                      @brandtbucher
 Objects/set*                  @rhettinger
 Objects/dict*                 @methane @markshannon
 Objects/typevarobject.c       @JelleZijlstra
@@ -36,12 +37,40 @@ Python/flowgraph.c            @markshannon @iritkatriel
 Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon @gvanrossum
 Python/optimizer*.c           @markshannon @gvanrossum
+Python/optimizer_analysis.c   @Fidget-Spinner
+Python/optimizer_bytecodes.c  @Fidget-Spinner
 Lib/test/test_patma.py        @brandtbucher
-Lib/test/test_peepholer.py    @brandtbucher
 Lib/test/test_type_*.py       @JelleZijlstra
 Lib/test/test_capi/test_misc.py  @markshannon @gvanrossum
 Tools/c-analyzer/             @ericsnowcurrently
 
+# dbm
+**/*dbm*                      @corona10 @erlend-aasland @serhiy-storchaka
+
+# runtime state/lifecycle
+**/*pylifecycle*              @ericsnowcurrently
+**/*pystate*                  @ericsnowcurrently
+**/*preconfig*                @ericsnowcurrently
+**/*initconfig*               @ericsnowcurrently
+**/*pathconfig*               @ericsnowcurrently
+**/*sysmodule*                @ericsnowcurrently
+**/*bltinmodule*              @ericsnowcurrently
+**/*gil*                      @ericsnowcurrently
+Include/internal/pycore_runtime.h   @ericsnowcurrently
+Include/internal/pycore_interp.h    @ericsnowcurrently
+Include/internal/pycore_tstate.h    @ericsnowcurrently
+Include/internal/pycore_*_state.h   @ericsnowcurrently
+Include/internal/pycore_*_init.h    @ericsnowcurrently
+Include/internal/pycore_atexit.h    @ericsnowcurrently
+Include/internal/pycore_freelist.h  @ericsnowcurrently
+Include/internal/pycore_global_objects.h  @ericsnowcurrently
+Include/internal/pycore_obmalloc.h  @ericsnowcurrently
+Include/internal/pycore_pymem.h     @ericsnowcurrently
+Modules/main.c                @ericsnowcurrently
+Programs/_bootstrap_python.c  @ericsnowcurrently
+Programs/python.c             @ericsnowcurrently
+Tools/build/generate_global_objects.py  @ericsnowcurrently
+
 # Exceptions
 Lib/traceback.py              @iritkatriel
 Lib/test/test_except*.py      @iritkatriel
@@ -50,13 +79,13 @@ Objects/exceptions.c          @iritkatriel
 Python/traceback.c            @iritkatriel
 
 # Hashing
-**/*hashlib*                  @tiran
-**/*pyhash*                   @tiran
-**/*sha*                      @tiran
-**/*md5*                      @tiran
-**/*blake*                    @tiran
-/Modules/_blake2/**           @tiran
-/Modules/_sha3/**             @tiran
+**/*hashlib*                  @gpshead @tiran
+**/*pyhash*                   @gpshead @tiran
+**/sha*                       @gpshead @tiran
+Modules/md5*                  @gpshead @tiran
+**/*blake*                    @gpshead @tiran
+Modules/_blake2/**            @gpshead @tiran
+Modules/_hacl/**              @gpshead
 
 # logging
 **/*logging*                  @vsajip
@@ -76,7 +105,20 @@ Python/traceback.c            @iritkatriel
 # Import (including importlib).
 **/*import*                   @brettcannon @ericsnowcurrently @ncoghlan @warsaw
 /Python/import.c              @kumaraditya303
-**/*importlib/resources/*      @jaraco @warsaw @FFY00
+Python/dynload_*.c            @ericsnowcurrently
+**/*freeze*                   @ericsnowcurrently
+**/*frozen*                   @ericsnowcurrently
+**/*modsupport*               @ericsnowcurrently
+**/*modulefinder*             @ericsnowcurrently
+**/*moduleobject*             @ericsnowcurrently
+**/*multiphase*               @ericsnowcurrently
+**/*pkgutil*                  @ericsnowcurrently
+**/*pythonrun*                @ericsnowcurrently
+**/*runpy*                    @ericsnowcurrently
+**/*singlephase*              @ericsnowcurrently
+Lib/test/test_module/         @ericsnowcurrently
+Doc/c-api/module.rst          @ericsnowcurrently
+**/*importlib/resources/*     @jaraco @warsaw @FFY00
 **/importlib/metadata/*       @jaraco @warsaw
 
 # Dates and times
@@ -120,6 +162,9 @@ Lib/ast.py                    @isidentical
 /Lib/unittest/mock.py         @cjw296
 /Lib/test/test_unittest/testmock/* @cjw296
 
+# multiprocessing
+**/*multiprocessing*          @gpshead
+
 # SQLite 3
 **/*sqlite*                   @berkerpeksag @erlend-aasland
 
@@ -187,11 +232,13 @@ Doc/c-api/stable.rst          @encukou
 **/*zipfile/_path/*           @jaraco
 
 # Argument Clinic
-/Tools/clinic/**              @erlend-aasland @AlexWaygood
-/Lib/test/test_clinic.py      @erlend-aasland @AlexWaygood
+/Tools/clinic/**              @erlend-aasland
+/Lib/test/test_clinic.py      @erlend-aasland
 Doc/howto/clinic.rst          @erlend-aasland
 
 # Subinterpreters
+**/*interpreteridobject.*     @ericsnowcurrently
+**/*crossinterp*              @ericsnowcurrently
 Lib/test/support/interpreters/  @ericsnowcurrently
 Modules/_xx*interp*module.c   @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently
@@ -200,5 +247,10 @@ Lib/test/test_interpreters/   @ericsnowcurrently
 /Tools/wasm/                  @brettcannon
 
 # SBOM
+/Misc/externals.spdx.json     @sethmlarson
 /Misc/sbom.spdx.json          @sethmlarson
 /Tools/build/generate_sbom.py @sethmlarson
+
+# Config Parser
+Lib/configparser.py           @jaraco
+Lib/test/test_configparser.py @jaraco

.github/CODEOWNERS

@@ -131,17 +131,21 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
+      - name: Runner image version
+        run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
       - name: Restore config.cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: config.cache
-          key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
+          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
         run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
       - name: Configure ccache action
         uses: hendrikmuhs/[email protected]
+        with:
+          save: false
       - name: Check Autoconf and aclocal versions
         run: |
           grep "Generated by GNU Autoconf 2.71" configure
@@ -158,7 +162,7 @@ jobs:
       - name: Build CPython
         run: |
           make -j4 regen-all
-          make regen-stdlib-module-names
+          make regen-stdlib-module-names regen-sbom
       - name: Check for changes
         run: |
           git add -u
@@ -248,19 +252,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
+        openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
       OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -272,7 +278,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -284,6 +290,8 @@ jobs:
         echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
     - name: Configure ccache action
       uses: hendrikmuhs/[email protected]
+      with:
+        save: false
     - name: Configure CPython
       run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR
     - name: Build CPython
@@ -300,7 +308,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -315,7 +323,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -327,6 +335,8 @@ jobs:
         echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
     - name: Configure ccache action
       uses: hendrikmuhs/[email protected]
+      with:
+        save: false
     - name: Setup directory envs for out-of-tree builds
       run: |
         echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
@@ -335,11 +345,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -369,7 +381,7 @@ jobs:
         ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt
     - name: 'Restore Hypothesis database'
       id: cache-hypothesis-database
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./hypothesis
         key: hypothesis-database-${{ github.head_ref || github.run_id }}
@@ -409,16 +421,18 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -434,7 +448,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -446,6 +460,9 @@ jobs:
         echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
     - name: Configure ccache action
       uses: hendrikmuhs/[email protected]
+      with:
+        save: ${{ github.event_name == 'push' }}
+        max-size: "200M"
     - name: Configure CPython
       run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
     - name: Build CPython

.github/workflows/build.yml

@@ -32,6 +32,8 @@ jobs:
     strategy:
       matrix:
         type: [x86, x64, arm64]
+    env:
+      IncludeFreethreaded: true
     steps:
     - uses: actions/checkout@v4
     - name: Build CPython installer