bpo-35596: Use unchecked PYCs for the embeddable distro to avoid zipimport restrictions (GH-11465)

Also adds extra steps to the CI build for Windows on Azure Pipelines to validate that the various layouts at least execute.
(cherry picked from commit 872bd2b)

Co-authored-by: Steve Dower <[email protected]>

Author: miss-islington

.azure-pipelines/ci.yml

@@ -134,3 +134,13 @@ jobs:
 
   steps:
   - template: ./windows-steps.yml
+
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: nuget
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: embed
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: appx

.azure-pipelines/windows-layout-steps.yml

@@ -0,0 +1,11 @@
+parameters:
+  kind: nuget
+  extraOpts: --precompile
+
+steps:
+- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
+  displayName: Create ${{ parameters['kind'] }} layout
+
+- script: .\python.exe -m test.pythoninfo
+  workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
+  displayName: Show layout info (${{ parameters['kind'] }})

Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst

@@ -0,0 +1,2 @@
+Use unchecked PYCs for the embeddable distro to avoid zipimport
+restrictions.

PC/layout/main.py

@@ -240,12 +240,18 @@ def _c(d):
             yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
 
 
-def _compile_one_py(src, dest, name, optimize):
+def _compile_one_py(src, dest, name, optimize, checked=True):
     import py_compile
 
     if dest is not None:
         dest = str(dest)
 
+    mode = (
+        py_compile.PycInvalidationMode.CHECKED_HASH
+        if checked
+        else py_compile.PycInvalidationMode.UNCHECKED_HASH
+    )
+
     try:
         return Path(
             py_compile.compile(
@@ -254,24 +260,24 @@ def _compile_one_py(src, dest, name, optimize):
                 str(name),
                 doraise=True,
                 optimize=optimize,
-                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+                invalidation_mode=mode,
             )
         )
     except py_compile.PyCompileError:
         log_warning("Failed to compile {}", src)
         return None
 
 
-def _py_temp_compile(src, ns, dest_dir=None):
+def _py_temp_compile(src, ns, dest_dir=None, checked=True):
     if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
         return None
 
     dest = (dest_dir or ns.temp) / (src.stem + ".py")
-    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2)
+    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
 
 
-def _write_to_zip(zf, dest, src, ns):
-    pyc = _py_temp_compile(src, ns)
+def _write_to_zip(zf, dest, src, ns, checked=True):
+    pyc = _py_temp_compile(src, ns, checked=checked)
     if pyc:
         try:
             zf.write(str(pyc), dest.with_suffix(".pyc"))
@@ -321,7 +327,7 @@ def generate_source_files(ns):
         ns.temp.mkdir(parents=True, exist_ok=True)
         with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
             for dest, src in get_lib_layout(ns):
-                _write_to_zip(zf, dest, src, ns)
+                _write_to_zip(zf, dest, src, ns, checked=False)
 
     if ns.include_underpth:
         log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)