Merge branch 'master' into randrange-index

Author: serhiy-storchaka

.azure-pipelines/ci.yml

@@ -104,7 +104,7 @@ jobs:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(Build.SourceBranchName)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/docs-steps.yml

@@ -12,11 +12,11 @@ steps:
   inputs:
     versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme
+- script: python -m pip install sphinx==3.2.1 blurb python-docs-theme
   displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:
-  - script: make check suspicious html PYTHON=python
+  - script: make check html PYTHON=python
     workingDirectory: '$(build.sourcesDirectory)/Doc'
     displayName: 'Build documentation'
 
@@ -31,7 +31,7 @@ steps:
 - ${{ if eq(parameters.upload, 'true') }}:
   - task: PublishBuildArtifacts@1
     displayName: 'Publish docs'
-  
+
     inputs:
       PathToPublish: '$(build.sourcesDirectory)/Doc/build'
       ArtifactName: docs

.azure-pipelines/pr.yml

@@ -104,7 +104,7 @@ jobs:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -120,10 +120,11 @@ jobs:
       artifactName: unsigned_msix
       downloadPath: $(Build.BinariesDirectory)
 
+  # MSIX must be signed and timestamped simultaneously
   - powershell: |
       $failed = $true
       foreach ($retry in 1..3) {
-          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix)
+          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix)
           if ($?) {
               $failed = $false
               break

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmImage: windows-2019
+    name: 'Windows Release'
 
   workspace:
     clean: all
@@ -36,6 +36,14 @@ jobs:
       nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive
     displayName: 'Create nuget package'
 
+  - powershell: |
+      gci *.nupkg | %{
+        nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite
+      }
+    displayName: 'Sign nuget package'
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), variables['SigningCertificate'])
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: nuget'
     inputs:

.azure-pipelines/windows-release/stage-sign.yml

@@ -57,7 +57,7 @@ jobs:
       $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }})
       $failed = $true
       foreach ($retry in 1..10) {
-          signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files
+          signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files
           if ($?) {
               $failed = $false
               break

.github/dependabot.yml

@@ -7,3 +7,7 @@ updates:
     labels:
       - "skip issue"
       - "skip news"
+    target_branch:
+      - "master"
+      - "3.9"
+      - "3.8"

.github/workflows/build.yml

@@ -47,6 +47,37 @@ jobs:
             # https://github.com/python/core-workflow/issues/373
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true
           fi
+
+  check_generated_files:
+    name: 'Check if generated files are up to date'
+    runs-on: ubuntu-latest
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - name: Install Dependencies
+        run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Build CPython
+        run: |
+          # Build Python with the libpython dynamic library
+          ./configure --with-pydebug --enable-shared
+          make -j4 regen-all
+      - name: Check for changes
+        run: |
+          changes=$(git status --porcelain)
+          # Check for changes in regenerated files
+          if ! test -z "$changes"
+          then
+            echo "Generated files not up to date. Perhaps you forgot to run make regen-all ;)"
+            echo "$changes"
+            exit 1
+          fi
+      - name: Check exported libpython symbols
+        run: make smelly
+      - name: Check limited ABI symbols
+        run: make check-limited-abi
+
   build_win32:
     name: 'Windows (x86)'
     runs-on: windows-latest
@@ -59,7 +90,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p Win32 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
 
   build_win_amd64:
     name: 'Windows (x64)'
@@ -73,7 +104,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
 
   build_macos:
     name: 'macOS'
@@ -106,7 +137,7 @@ jobs:
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/[email protected].2
+      uses: actions/[email protected].3
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}

.github/workflows/coverage.yml

@@ -32,7 +32,7 @@ jobs:
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/[email protected].2
+      uses: actions/[email protected].3
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}

.github/workflows/doc.yml

@@ -36,9 +36,9 @@ jobs:
     - name: 'Install build dependencies'
       run: make -C Doc/ PYTHON=../python venv
     - name: 'Build documentation'
-      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest suspicious html
+      run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html
     - name: 'Upload'
-      uses: actions/[email protected].0
+      uses: actions/[email protected].1
       with:
         name: doc-html
         path: Doc/build/html

.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.'
+        stale-pr-label: 'stale'
+        days-before-stale: 30
+        days-before-close: -1

.travis.yml

@@ -53,7 +53,7 @@ matrix:
         - cd Doc
         - make venv PYTHON=python
       script:
-        - make check suspicious html SPHINXOPTS="-q -W -j4"
+        - make check html SPHINXOPTS="-q -W -j4"
     - name: "Documentation tests"
       os: linux
       language: c
@@ -192,6 +192,8 @@ script:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi
   # Check that all symbols exported by libpython start with "Py" or "_Py"
   - make smelly
+  # Check that all symbols in the limited abi are present
+  - make check-limited-abi
   # `-r -w` implicitly provided through `make buildbottest`.
   - |
     if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then

Doc/Makefile

@@ -202,6 +202,7 @@ dist:
 
 check:
 	$(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst
+	$(PYTHON) tools/rstlint.py ../Misc/NEWS.d/next/
 
 serve:
 	$(PYTHON) ../Tools/scripts/serve.py build/html
@@ -215,12 +216,11 @@ serve:
 
 # for development releases: always build
 autobuild-dev:
-	make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1'
-	-make suspicious
+	make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
 
 # for quick rebuilds (HTML only)
 autobuild-dev-html:
-	make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1'
+	make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
 
 # for stable releases: only build if not in pre-release stage (alpha, beta)
 # release candidate downloads are okay, since the stable tree can be in that stage

Doc/c-api/arg.rst

@@ -482,7 +482,8 @@ API Functions
    *min* and no more than *max*; *min* and *max* may be equal.  Additional
    arguments must be passed to the function, each of which should be a pointer to a
    :c:type:`PyObject*` variable; these will be filled in with the values from
-   *args*; they will contain borrowed references.  The variables which correspond
+   *args*; they will contain :term:`borrowed references <borrowed reference>`.
+   The variables which correspond
    to optional parameters not given by *args* will not be filled in; these should
    be initialized by the caller. This function returns true on success and false if
    *args* is not a tuple or contains the wrong number of elements; an exception

Doc/c-api/exceptions.rst

@@ -182,8 +182,8 @@ For convenience, some of these functions will always return a
 .. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename)
 
    Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename
-   is given as a C string.  *filename* is decoded from the filesystem encoding
-   (:func:`os.fsdecode`).
+   is given as a C string.  *filename* is decoded from the :term:`filesystem
+   encoding and error handler`.
 
 
 .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr)
@@ -266,7 +266,7 @@ For convenience, some of these functions will always return a
 .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset)
 
    Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string
-   decoded from the filesystem encoding (:func:`os.fsdecode`).
+   decoded from the :term:`filesystem encoding and error handler`.
 
    .. versionadded:: 3.2
 
@@ -343,7 +343,7 @@ an error value).
 
    Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and
    *module* are UTF-8 encoded strings, and *filename* is decoded from the
-   filesystem encoding (:func:`os.fsdecode`).
+   :term:`filesystem encoding and error handler`.
 
 
 .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...)

Doc/c-api/init.rst

@@ -151,8 +151,9 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
 .. c:var:: int Py_LegacyWindowsFSEncodingFlag
 
-   If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8
-   encoding for the filesystem encoding.
+   If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error
+   handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler,
+   for the :term:`filesystem encoding and error handler`.
 
    Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment
    variable is set to a non-empty string.
@@ -375,6 +376,12 @@ Process-wide parameters
    The returned string points into static storage; the caller should not modify its
    value.
 
+   This function should not be called before :c:func:`Py_Initialize`, otherwise
+   it returns ``NULL``.
+
+   .. versionchanged:: 3.10
+      It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+
 
 .. c:function:: wchar_t* Py_GetPrefix()
 
@@ -388,6 +395,12 @@ Process-wide parameters
    script at build time.  The value is available to Python code as ``sys.prefix``.
    It is only useful on Unix.  See also the next function.
 
+   This function should not be called before :c:func:`Py_Initialize`, otherwise
+   it returns ``NULL``.
+
+   .. versionchanged:: 3.10
+      It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+
 
 .. c:function:: wchar_t* Py_GetExecPrefix()
 
@@ -423,6 +436,12 @@ Process-wide parameters
    while having :file:`/usr/local/plat` be a different filesystem for each
    platform.
 
+   This function should not be called before :c:func:`Py_Initialize`, otherwise
+   it returns ``NULL``.
+
+   .. versionchanged:: 3.10
+      It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+
 
 .. c:function:: wchar_t* Py_GetProgramFullPath()
 
@@ -436,6 +455,12 @@ Process-wide parameters
    static storage; the caller should not modify its value.  The value is available
    to Python code as ``sys.executable``.
 
+   This function should not be called before :c:func:`Py_Initialize`, otherwise
+   it returns ``NULL``.
+
+   .. versionchanged:: 3.10
+      It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+
 
 .. c:function:: wchar_t* Py_GetPath()
 
@@ -454,8 +479,14 @@ Process-wide parameters
    can be (and usually is) modified later to change the search path for loading
    modules.
 
+   This function should not be called before :c:func:`Py_Initialize`, otherwise
+   it returns ``NULL``.
+
    .. XXX should give the exact rules
 
+   .. versionchanged:: 3.10
+      It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+
 
 .. c:function::  void Py_SetPath(const wchar_t *)
 
@@ -637,6 +668,12 @@ Process-wide parameters
    :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME`
    environment variable if it is set.
 
+   This function should not be called before :c:func:`Py_Initialize`, otherwise
+   it returns ``NULL``.
+
+   .. versionchanged:: 3.10
+      It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+
 
 .. _threads:
 
@@ -1076,7 +1113,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
 
    Get the current frame of the Python thread state *tstate*.
 
-   Return a strong reference. Return ``NULL`` if no frame is currently
+   Return a :term:`strong reference`. Return ``NULL`` if no frame is currently
    executing.
 
    See also :c:func:`PyEval_GetFrame`.