Merge pull request #279 from python/master

Sync Fork from Upstream Repo

Author: sthagen

.azure-pipelines/windows-release/stage-build.yml

@@ -3,7 +3,7 @@ jobs:
   displayName: Docs build
   pool:
     name: 'Windows Release'
-    #vmName: win2016-vs2017
+    #vmImage: windows-2019
 
   workspace:
     clean: all
@@ -45,7 +45,7 @@ jobs:
   displayName: Python build
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all
@@ -91,7 +91,7 @@ jobs:
   condition: and(succeeded(), ne(variables['DoPGO'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all
@@ -141,7 +141,7 @@ jobs:
   displayName: Publish Tcl/Tk Library
 
   pool:
-    vmName: windows-latest
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-layout-embed.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoEmbed'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-layout-full.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoLayout'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-layout-msix.yml

@@ -3,7 +3,7 @@ jobs:
   displayName: Make MSIX layout
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-layout-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-msi.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), not(variables['SigningCertificate']))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   variables:
     ReleaseUri: http://www.python.org/{arch}

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -3,7 +3,7 @@ jobs:
   displayName: Pack MSIX bundles
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-publish-nugetorg.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-publish-pythonorg.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true')))
 
   pool:
-    #vmName: win2016-vs2017
+    #vmImage: windows-2019
     name: 'Windows Release'
 
   workspace:

.azure-pipelines/windows-release/stage-publish-store.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoMSIX'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-sign.yml

@@ -114,7 +114,7 @@ jobs:
   condition: and(succeeded(), not(variables['SigningCertificate']))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   steps:
   - checkout: none

.azure-pipelines/windows-release/stage-test-embed.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoEmbed'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-test-msi.yml

@@ -3,7 +3,7 @@ jobs:
   displayName: Test MSI
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

.azure-pipelines/windows-release/stage-test-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmName: win2016-vs2017
+    vmImage: windows-2019
 
   workspace:
     clean: all

Doc/library/compileall.rst

@@ -143,7 +143,7 @@ runtime.
 Public functions
 ----------------
 
-.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, prependdir=None, limit_sl_dest=None)
+.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None)
 
    Recursively descend the directory tree named by *dir*, compiling all :file:`.py`
    files along the way. Return a true value if all the files compiled successfully,
@@ -221,7 +221,7 @@ Public functions
    .. versionchanged:: 3.9
       Added *stripdir*, *prependdir* and *limit_sl_dest* arguments.
 
-.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None)
+.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None)
 
    Compile the file with path *fullname*. Return a true value if the file
    compiled successfully, and a false value otherwise.

Doc/library/token.rst

@@ -87,7 +87,7 @@ the :mod:`tokenize` module.
    now tokenized as :data:`NAME` tokens.
 
 .. versionchanged:: 3.8
-   Added :data:`TYPE_COMMENT`.
+   Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`.
    Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed
    to support parsing older Python versions for :func:`ast.parse` with
    ``feature_version`` set to 6 or lower).

Lib/compileall.py

@@ -46,7 +46,7 @@ def _walk_dir(dir, maxlevels, quiet=0):
 
 def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                 rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
-                invalidation_mode=None, stripdir=None,
+                invalidation_mode=None, *, stripdir=None,
                 prependdir=None, limit_sl_dest=None):
     """Byte-compile all modules in the given directory tree.
 
@@ -72,6 +72,13 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                    the defined path
     """
     ProcessPoolExecutor = None
+    if ddir is not None and (stripdir is not None or prependdir is not None):
+        raise ValueError(("Destination dir (ddir) cannot be used "
+                          "in combination with stripdir or prependdir"))
+    if ddir is not None:
+        stripdir = dir
+        prependdir = ddir
+        ddir = None
     if workers < 0:
         raise ValueError('workers must be greater or equal to 0')
     if workers != 1:
@@ -111,7 +118,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
 
 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
                  legacy=False, optimize=-1,
-                 invalidation_mode=None, stripdir=None, prependdir=None,
+                 invalidation_mode=None, *, stripdir=None, prependdir=None,
                  limit_sl_dest=None):
     """Byte-compile one file.
 

Lib/test/test_compileall.py

@@ -212,6 +212,47 @@ def test_compile_dir_maxlevels(self):
         compileall.compile_dir(self.directory, quiet=True, maxlevels=depth)
         self.assertTrue(os.path.isfile(pyc_filename))
 
+    def _test_ddir_only(self, *, ddir, parallel=True):
+        """Recursive compile_dir ddir must contain package paths; bpo39769."""
+        fullpath = ["test", "foo"]
+        path = self.directory
+        mods = []
+        for subdir in fullpath:
+            path = os.path.join(path, subdir)
+            os.mkdir(path)
+            script_helper.make_script(path, "__init__", "")
+            mods.append(script_helper.make_script(path, "mod",
+                                                  "def fn(): 1/0\nfn()\n"))
+        compileall.compile_dir(
+                self.directory, quiet=True, ddir=ddir,
+                workers=2 if parallel else 1)
+        self.assertTrue(mods)
+        for mod in mods:
+            self.assertTrue(mod.startswith(self.directory), mod)
+            modcode = importlib.util.cache_from_source(mod)
+            modpath = mod[len(self.directory+os.sep):]
+            _, _, err = script_helper.assert_python_failure(modcode)
+            expected_in = os.path.join(ddir, modpath)
+            mod_code_obj = test.test_importlib.util.get_code_from_pyc(modcode)
+            self.assertEqual(mod_code_obj.co_filename, expected_in)
+            self.assertIn(f'"{expected_in}"', os.fsdecode(err))
+
+    def test_ddir_only_one_worker(self):
+        """Recursive compile_dir ddir= contains package paths; bpo39769."""
+        return self._test_ddir_only(ddir="<a prefix>", parallel=False)
+
+    def test_ddir_multiple_workers(self):
+        """Recursive compile_dir ddir= contains package paths; bpo39769."""
+        return self._test_ddir_only(ddir="<a prefix>", parallel=True)
+
+    def test_ddir_empty_only_one_worker(self):
+        """Recursive compile_dir ddir='' contains package paths; bpo39769."""
+        return self._test_ddir_only(ddir="", parallel=False)
+
+    def test_ddir_empty_multiple_workers(self):
+        """Recursive compile_dir ddir='' contains package paths; bpo39769."""
+        return self._test_ddir_only(ddir="", parallel=True)
+
     def test_strip_only(self):
         fullpath = ["test", "build", "real", "path"]
         path = os.path.join(self.directory, *fullpath)

Lib/test/test_importlib/util.py

@@ -7,6 +7,7 @@
 from importlib import machinery, util, invalidate_caches
 from importlib.abc import ResourceReader
 import io
+import marshal
 import os
 import os.path
 from pathlib import Path, PurePath
@@ -118,6 +119,16 @@ def submodule(parent, name, pkg_dir, content=''):
     return '{}.{}'.format(parent, name), path
 
 
+def get_code_from_pyc(pyc_path):
+    """Reads a pyc file and returns the unmarshalled code object within.
+
+    No header validation is performed.
+    """
+    with open(pyc_path, 'rb') as pyc_f:
+        pyc_f.seek(16)
+        return marshal.load(pyc_f)
+
+
 @contextlib.contextmanager
 def uncache(*names):
     """Uncache a module from sys.modules.

Misc/NEWS.d/next/Documentation/2020-02-21-22-05-20.bpo-39718.xtBoSi.rst

@@ -0,0 +1 @@
+Update :mod:`token` documentation to reflect additions in Python 3.8

Misc/NEWS.d/next/Library/2020-02-27-00-40-21.bpo-39769.hJmxu4.rst

@@ -0,0 +1,4 @@
+The :func:`compileall.compile_dir` function's *ddir* parameter and the
+compileall command line flag `-d` no longer write the wrong pathname to the
+generated pyc file for submodules beneath the root of the directory tree
+being compiled.  This fixes a regression introduced with Python 3.5.

Misc/NEWS.d/next/Windows/2020-02-28-22-46-09.bpo-39789.67XRoP.rst

@@ -0,0 +1 @@
+Update Windows release build machines to Visual Studio 2019 (MSVC 14.2).

PC/layout/support/nuspec.py

@@ -14,7 +14,7 @@
 NUSPEC_DATA = {
     "PYTHON_TAG": VER_DOT,
     "PYTHON_VERSION": os.getenv("PYTHON_NUSPEC_VERSION"),
-    "FILELIST": r'    <file src="**\*" target="tools" />',
+    "FILELIST": r'    <file src="**\*" exclude="python.png" target="tools" />',
     "GIT": sys._git,
 }
 
@@ -31,7 +31,7 @@
         VER_DOT, VER_MICRO, "-" if VER_SUFFIX else "", VER_SUFFIX
     )
 
-FILELIST_WITH_PROPS = r"""    <file src="**\*" exclude="python.props" target="tools" />
+FILELIST_WITH_PROPS = r"""    <file src="**\*" exclude="python.png;python.props" target="tools" />
     <file src="python.props" target="build\native" />"""
 
 NUSPEC_TEMPLATE = r"""<?xml version="1.0"?>
@@ -44,13 +44,13 @@
     <license type="file">tools\LICENSE.txt</license>
     <projectUrl>https://www.python.org/</projectUrl>
     <description>Installs {PYTHON_BITNESS} Python for use in build scenarios.</description>
-    <icon>images\logox128.png</icon>
+    <icon>images\python.png</icon>
     <iconUrl>https://www.python.org/static/favicon.ico</iconUrl>
     <tags>python</tags>
     <repository type="git" url="https://github.com/Python/CPython.git" commit="{GIT[2]}" />
   </metadata>
   <files>
-    <file src="{LOGO}" target="images" />
+    <file src="python.png" target="images" />
 {FILELIST}
   </files>
 </package>
@@ -73,6 +73,6 @@ def get_nuspec_layout(ns):
                 data[k] = v
         if ns.include_all or ns.include_props:
             data["FILELIST"] = FILELIST_WITH_PROPS
-        data["LOGO"] = ns.source / "PC" / "icons" / "logox128.png"
         nuspec = NUSPEC_TEMPLATE.format_map(data)
         yield "python.nuspec", ("python.nuspec", nuspec.encode("utf-8"))
+        yield "python.png", ns.source / "PC" / "icons" / "logox128.png"

Tools/nuget/make_pkg.proj

@@ -33,7 +33,7 @@
 
         <PackageArguments Condition="$(Packages) != ''">"$(IntermediateOutputPath)pkg\pip.exe" -B -m pip install -U $(Packages)</PackageArguments>
 
-        <NugetPackCommand>"$(Nuget)" pack "$(MSBuildThisFileDirectory)\$(OutputName).nuspec" -BasePath "$(IntermediateOutputPath)pkg"</NugetPackCommand>
+        <NugetPackCommand>"$(Nuget)" pack "$(IntermediateOutputPath)pkg\python.nuspec" -BasePath "$(IntermediateOutputPath)pkg"</NugetPackCommand>
         <NugetPackSymbolsCommand Condition="Exists('$(MSBuildThisFileDirectory)\$(OutputName).symbols.nuspec')">"$(Nuget)" pack "$(MSBuildThisFileDirectory)\$(OutputName).symbols.nuspec" -BasePath "$(BuildPath.TrimEnd(`\`))"</NugetPackSymbolsCommand>
         <NugetArguments>$(NugetArguments) -OutputDirectory "$(OutputPath.Trim(`\`))"</NugetArguments>
         <NugetArguments>$(NugetArguments) -Version "$(NuspecVersion)"</NugetArguments>