bpo-33125: Add support for building and releasing Windows ARM64 packages

Author: zooba

.azure-pipelines/ci.yml

@@ -145,7 +145,7 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(Build.SourceBranchName)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml

.azure-pipelines/pr.yml

@@ -145,7 +145,10 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+      winarm64:
+        arch: arm64
+        buildOpt: '-p arm64'
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml

.azure-pipelines/windows-release/build-steps.yml

@@ -43,7 +43,7 @@ steps:
 
 - powershell: |
     $env:SigningCertificate = $null
-    .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default
+    python PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch)
     makecat "${env:CAT}.cdf"
     del "${env:CAT}.cdf"
     if (-not (Test-Path "${env:CAT}.cat")) {
@@ -52,6 +52,7 @@ steps:
   displayName: 'Generate catalog'
   env:
     CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+    PYTHON_HEXVERSION: $(VersionHex)
 
 - task: PublishPipelineArtifact@0
   displayName: 'Publish binaries'

.azure-pipelines/windows-release/layout-command.yml

@@ -1,12 +1,20 @@
 steps:
+- task: DownloadPipelineArtifact@1
+  displayName: 'Download artifact: bin_$(HostArch)'
+  condition: and(succeeded(), variables['HostArch'])
+  inputs:
+    artifactName: bin_$(HostArch)
+    targetPath: $(Build.BinariesDirectory)\bin_$(HostArch)
+
 - powershell: >
     Write-Host (
     '##vso[task.setvariable variable=LayoutCmd]&
-    "{0}\bin\python.exe"
+    "$(Python)"
     "{1}\PC\layout"
     -vv
     --source "{1}"
     --build "{0}\bin"
+    --arch "$(Name)"
     --temp "{0}\layout-temp"
     --include-cat "{0}\bin\python.cat"
     --doc-build "{0}\doc"'

.azure-pipelines/windows-release/msi-steps.yml

@@ -54,6 +54,7 @@ steps:
   - powershell: |
       copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - script: |
       call Tools\msi\get_externals.bat

.azure-pipelines/windows-release/stage-build.yml

@@ -16,14 +16,16 @@ jobs:
     env:
       BUILDDIR: $(Build.BinariesDirectory)\Doc
 
-  #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm"
-  #  displayName: 'Cheat at building CHM docs'
-
   - script: Doc\make.bat htmlhelp
     displayName: 'Build CHM docs'
     env:
       BUILDDIR: $(Build.BinariesDirectory)\Doc
 
+  #- powershell: |
+  #    mkdir -Force "$(Build.BinariesDirectory)\Doc\htmlhelp"
+  #    iwr "https://www.python.org/ftp/python/3.8.0/python380.chm" -OutFile "$(Build.BinariesDirectory)\Doc\htmlhelp\python390a0.chm"
+  #  displayName: 'Cheat at building CHM docs'
+
   - task: CopyFiles@2
     displayName: 'Assemble artifact: Doc'
     inputs:
@@ -65,6 +67,16 @@ jobs:
         Arch: amd64
         Platform: x64
         Configuration: Debug
+      arm64:
+        Name: arm64
+        Arch: arm64
+        Platform: ARM64
+        Configuration: Release
+      arm64_d:
+        Name: arm64_d
+        Arch: arm64
+        Platform: ARM64
+        Configuration: Debug
 
   steps:
     - template: ./build-steps.yml

.azure-pipelines/windows-release/stage-layout-embed.yml

@@ -19,6 +19,11 @@ jobs:
         Name: amd64
         Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
   - template: ./checkout.yml

.azure-pipelines/windows-release/stage-layout-full.yml

@@ -13,11 +13,18 @@ jobs:
     matrix:
       win32:
         Name: win32
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -43,21 +50,23 @@ jobs:
 
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: tcltk_lib_$(Name)'
+    condition: and(succeeded(), variables['TclLibrary'])
     inputs:
       artifactName: tcltk_lib_$(Name)
       targetPath: $(Build.BinariesDirectory)\tcltk_lib
 
   - powershell: |
-      copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
+      copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
   - powershell: |
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default
     displayName: 'Generate full layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_full_$(Name)'

.azure-pipelines/windows-release/stage-layout-msix.yml

@@ -12,11 +12,18 @@ jobs:
     matrix:
       #win32:
       #  Name: win32
-      #  Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+      #  Python: $(Build.BinariesDirectory)\bin\python.exe
       #  PYTHONHOME: $(Build.SourcesDirectory)
+      #  TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -36,13 +43,15 @@ jobs:
 
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: tcltk_lib_$(Name)'
+    condition: and(succeeded(), variables['TclLibrary'])
     inputs:
       artifactName: tcltk_lib_$(Name)
       targetPath: $(Build.BinariesDirectory)\tcltk_lib
 
   - powershell: |
-      copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
+      copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
@@ -51,7 +60,7 @@ jobs:
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile
     displayName: 'Generate store APPX layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_appxstore_$(Name)'
@@ -79,7 +88,7 @@ jobs:
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests
     displayName: 'Generate sideloading APPX layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_appx_$(Name)'

.azure-pipelines/windows-release/stage-layout-nuget.yml

@@ -13,11 +13,16 @@ jobs:
     matrix:
       win32:
         Name: win32
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -32,14 +37,13 @@ jobs:
   - powershell: |
       copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
   - powershell: |
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget
     displayName: 'Generate nuget layout'
-    env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_nuget_$(Name)'

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -20,6 +20,16 @@ jobs:
         Artifact: appxstore
         Suffix: -store
         Upload: true
+      arm64:
+        Name: arm64
+        Artifact: appx
+        Suffix:
+        ShouldSign: true
+      arm64_store:
+        Name: arm64
+        Artifact: appxstore
+        Suffix: -store
+        Upload: true
 
   steps:
   - template: ./checkout.yml

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -15,6 +15,8 @@ jobs:
         Name: amd64
       win32:
         Name: win32
+      arm64:
+        Name: arm64
 
   steps:
   - checkout: none

.azure-pipelines/windows-release/stage-sign.yml

@@ -19,6 +19,8 @@ jobs:
         Name: win32
       amd64:
         Name: amd64
+      arm64:
+        Name: arm64
 
   steps:
   - template: ./checkout.yml

.azure-pipelines/windows-steps.yml

@@ -19,9 +19,11 @@ steps:
 
 - script: python.bat -m test.pythoninfo
   displayName: 'Display build info'
+  condition: and(succeeded(), variables['testRunPlatform'])
 
 - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test"
   displayName: 'Tests'
+  condition: and(succeeded(), variables['testRunPlatform'])
   env:
     PREFIX: $(Py_OutDir)\$(arch)
 
@@ -32,4 +34,4 @@ steps:
     mergeTestResults: true
     testRunTitle: $(testRunTitle)
     platform: $(testRunPlatform)
-  condition: succeededOrFailed()
+  condition: and(succeededOrFailed(), variables['testRunPlatform'])

Misc/NEWS.d/next/Windows/2019-11-14-08-57-50.bpo-33125.EN5MWS.rst

@@ -0,0 +1 @@
+Add support for building and releasing Windows ARM64 packages.

PC/layout/main.py

@@ -285,14 +285,13 @@ def _compile_one_py(src, dest, name, optimize, checked=True):
         log_warning("Failed to compile {}", src)
         return None
 
+
 # name argument added to address bpo-37641
 def _py_temp_compile(src, name, ns, dest_dir=None, checked=True):
     if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
         return None
     dest = (dest_dir or ns.temp) / (src.stem + ".pyc")
-    return _compile_one_py(
-        src, dest, name, optimize=2, checked=checked
-    )
+    return _compile_one_py(src, dest, name, optimize=2, checked=checked)
 
 
 def _write_to_zip(zf, dest, src, ns, checked=True):
@@ -496,6 +495,13 @@ def main():
     parser.add_argument(
         "-b", "--build", metavar="dir", help="Specify the build directory", type=Path
     )
+    parser.add_argument(
+        "--arch",
+        metavar="architecture",
+        help="Specify the target architecture",
+        type=str,
+        default=None,
+    )
     parser.add_argument(
         "--doc-build",
         metavar="dir",
@@ -587,6 +593,8 @@ def main():
         ns.doc_build = (Path.cwd() / ns.doc_build).resolve()
     if ns.include_cat and not ns.include_cat.is_absolute():
         ns.include_cat = (Path.cwd() / ns.include_cat).resolve()
+    if not ns.arch:
+        ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32"
 
     if ns.copy and not ns.copy.is_absolute():
         ns.copy = (Path.cwd() / ns.copy).resolve()
@@ -602,6 +610,7 @@ def main():
 Source: {ns.source}
 Build:  {ns.build}
 Temp:   {ns.temp}
+Arch:   {ns.arch}
 
 Copy to: {ns.copy}
 Zip to:  {ns.zip}
@@ -613,6 +622,10 @@ def main():
         log_warning("Assuming --include-tcltk to support --include-idle")
         ns.include_tcltk = True
 
+    if ns.arch not in ("win32", "amd64", "arm32", "arm64"):
+        log_error("--arch is not a valid value (win32, amd64, arm32, arm64)")
+        return 4
+
     try:
         generate_source_files(ns)
         files = list(get_layout(ns))