Several cherrypicks for the windows build

cmake/caches/Windows-aarch64.cmake

@@ -50,7 +50,7 @@ set(LLVM_TARGETS_TO_BUILD AArch64 ARM WebAssembly X86 CACHE STRING "")
 set(LLVM_BUILD_LLVM_DYLIB NO CACHE BOOL "")
 set(LLVM_BUILD_LLVM_C_DYLIB NO CACHE BOOL "")
 set(LLVM_ENABLE_LIBEDIT NO CACHE BOOL "")
-set(LLVM_ENABLE_LIBXML2 NO CACHE BOOL "")
+set(LLVM_ENABLE_LIBXML2 YES CACHE BOOL "")
 set(LLVM_ENABLE_OCAMLDOC NO CACHE BOOL "")
 set(LLVM_ENABLE_TERMINFO NO CACHE BOOL "")
 set(LLVM_ENABLE_Z3_SOLVER NO CACHE BOOL "")
@@ -62,12 +62,14 @@ set(LLVM_INCLUDE_GO_TESTS NO CACHE BOOL "")
 set(LLVM_TOOL_GOLD_BUILD NO CACHE BOOL "")
 set(LLVM_TOOL_LLVM_SHLIB_BUILD NO CACHE BOOL "")
 
+set(CLANG_ENABLE_LIBXML2 NO CACHE BOOL "")
+
 # Avoid swig dependency for lldb
 set(LLDB_ALLOW_STATIC_BINDINGS YES CACHE BOOL "")
 set(LLDB_USE_STATIC_BINDINGS YES CACHE BOOL "")
 set(LLDB_ENABLE_PYTHON YES CACHE BOOL "")
 set(LLDB_EMBED_PYTHON_HOME NO CACHE BOOL "")
-set(LLDB_ENABLE_LIBXML2 NO CACHE BOOL "")
+set(LLDB_ENABLE_LIBXML2 YES CACHE BOOL "")
 
 # This requires perl which may not be available on Windows
 set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "")

cmake/caches/Windows-x86_64.cmake

@@ -90,7 +90,7 @@ set(LLVM_TARGETS_TO_BUILD AArch64 ARM WebAssembly X86 CACHE STRING "")
 set(LLVM_BUILD_LLVM_DYLIB NO CACHE BOOL "")
 set(LLVM_BUILD_LLVM_C_DYLIB NO CACHE BOOL "")
 set(LLVM_ENABLE_LIBEDIT NO CACHE BOOL "")
-set(LLVM_ENABLE_LIBXML2 NO CACHE BOOL "")
+set(LLVM_ENABLE_LIBXML2 YES CACHE BOOL "")
 set(LLVM_ENABLE_OCAMLDOC NO CACHE BOOL "")
 set(LLVM_ENABLE_TERMINFO NO CACHE BOOL "")
 set(LLVM_ENABLE_Z3_SOLVER NO CACHE BOOL "")
@@ -102,12 +102,14 @@ set(LLVM_INCLUDE_GO_TESTS NO CACHE BOOL "")
 set(LLVM_TOOL_GOLD_BUILD NO CACHE BOOL "")
 set(LLVM_TOOL_LLVM_SHLIB_BUILD NO CACHE BOOL "")
 
+set(CLANG_ENABLE_LIBXML2 NO CACHE BOOL "")
+
 # Avoid swig dependency for lldb
 set(LLDB_ALLOW_STATIC_BINDINGS YES CACHE BOOL "")
 set(LLDB_USE_STATIC_BINDINGS YES CACHE BOOL "")
 set(LLDB_ENABLE_PYTHON YES CACHE BOOL "")
 set(LLDB_EMBED_PYTHON_HOME NO CACHE BOOL "")
-set(LLDB_ENABLE_LIBXML2 NO CACHE BOOL "")
+set(LLDB_ENABLE_LIBXML2 YES CACHE BOOL "")
 
 # This requires perl which may not be available on Windows
 set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "")

tools/swift-ide-test/CMakeLists.txt

@@ -17,12 +17,8 @@ target_link_libraries(swift-ide-test
 
 # If libxml2 is available, make it available for swift-ide-test.
 if(LLVM_ENABLE_LIBXML2)
-  include_directories(SYSTEM ${LIBXML2_INCLUDE_DIR})
-  target_link_libraries(swift-ide-test PRIVATE ${LIBXML2_LIBRARIES})
+  target_link_libraries(swift-ide-test PRIVATE LibXml2::LibXml2)
   target_compile_definitions(swift-ide-test PRIVATE SWIFT_HAVE_LIBXML=1)
-  if("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "OpenBSD" AND NOT CMAKE_CROSSCOMPILING)
-    include_directories(SYSTEM "/usr/local/include")
-  endif()
 endif()
 
 # Create a symlink for swift-api-dump.py in the bin directory

utils/build.ps1

@@ -194,14 +194,6 @@ $WiXVersion = "4.0.5"
 # Avoid $env:ProgramFiles in case this script is running as x86
 $UnixToolsBinDir = "$env:SystemDrive\Program Files\Git\usr\bin"
 
-$python = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Shared\Python39_64\python.exe"
-if (-not (Test-Path $python)) {
-  $python = (where.exe python) | Select-Object -First 1
-  if (-not (Test-Path $python)) {
-    throw "Python.exe not found"
-  }
-}
-
 if ($Android -and ($AndroidSDKs.Length -eq 0)) {
   # Enable all android SDKs by default.
   $AndroidSDKs = @("aarch64","armv7","i686","x86_64")
@@ -358,6 +350,10 @@ function Get-BisonExecutable {
   return Join-Path -Path $BinaryCache -ChildPath "win_flex_bison\win_bison.exe"
 }
 
+function Get-PythonExecutable {
+  return Join-Path -Path $BinaryCache -ChildPath "Python$($BuildArch.CMakeName)-$PythonVersion\tools\python.exe"
+}
+
 function Get-InstallDir($Arch) {
   if ($Arch -eq $HostArch) {
     $ProgramFilesName = "Program Files"
@@ -747,10 +743,43 @@ function Fetch-Dependencies {
     }
   }
 
+  function Ensure-PythonModules($Python) {
+    # First ensure pip is installed, else bootstrap it
+    try {
+      Invoke-Program -OutNull $Python -m pip *> $null
+    } catch {
+      Write-Output "Installing pip ..."
+      Invoke-Program -OutNull $Python '-I' -m ensurepip -U --default-pip
+    }
+    # 'packaging' is required for building LLVM 18+
+    try {
+      Invoke-Program -OutNull $Python -c 'import packaging' *> $null
+    } catch {
+      $WheelURL = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl"
+      $WheelHash = "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
+      DownloadAndVerify $WheelURL "$BinaryCache\python\packaging-24.1-py3-none-any.whl" $WheelHash
+      Write-Output "Installing 'packaging-24.1-py3-none-any.whl' ..."
+      Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\packaging-24.1-py3-none-any.whl" --disable-pip-version-check
+    }
+    # 'setuptools' provides 'distutils' module for Python 3.12+, required for SWIG support
+    # https://github.com/swiftlang/llvm-project/issues/9289
+    try {
+      Invoke-Program -OutNull $Python -c 'import distutils' *> $null
+    } catch {
+      $WheelURL = "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl"
+      $WheelHash = "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"
+      DownloadAndVerify $WheelURL "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" $WheelHash
+      Write-Output "Installing 'setuptools-75.1.0-py3-none-any.whl' ..."
+      Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" --disable-pip-version-check
+    }
+  }
+
   Download-Python $HostArchName
   if ($IsCrossCompiling) {
     Download-Python $BuildArchName
   }
+  # Ensure Python modules that are required as host build tools
+  Ensure-PythonModules "$(Get-PythonExecutable)"
 
   if ($Android) {
     # Only a specific NDK version is supported right now.
@@ -959,7 +988,6 @@ function Build-CMakeProject {
     }
 
     TryAdd-KeyValue $Defines CMAKE_BUILD_TYPE Release
-    TryAdd-KeyValue $Defines CMAKE_MT "mt"
 
     $CFlags = @()
     switch ($Platform) {
@@ -1382,6 +1410,8 @@ function Build-BuildTools($Arch) {
     -BuildTargets llvm-tblgen,clang-tblgen,clang-pseudo-gen,clang-tidy-confusable-chars-gen,lldb-tblgen,llvm-config,swift-def-to-strings-converter,swift-serialize-diagnostics,swift-compatibility-symbols `
     -Defines @{
       CMAKE_CROSSCOMPILING = "NO";
+      CLANG_ENABLE_LIBXML2 = "NO";
+      LLDB_ENABLE_LIBXML2 = "NO";
       LLDB_ENABLE_PYTHON = "NO";
       LLDB_INCLUDE_TESTS = "NO";
       LLDB_ENABLE_SWIFT_SUPPORT = "NO";
@@ -1456,6 +1486,9 @@ function Build-Compilers() {
       }
     }
 
+    $PythonRoot = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools"
+    $PythonLibName = "python{0}{1}" -f ([System.Version]$PythonVersion).Major, ([System.Version]$PythonVersion).Minor
+
     # The STL in VS 17.10 requires Clang 17 or higher, but Swift toolchains prior to version 6 include older versions
     # of Clang. If bootstrapping with an older toolchain, we need to relax to relax this requirement with
     # ALLOW_COMPILER_AND_STL_VERSION_MISMATCH.
@@ -1477,7 +1510,9 @@ function Build-Compilers() {
       -Defines ($TestingDefines + @{
         CLANG_TABLEGEN = (Join-Path -Path $BuildTools -ChildPath "clang-tblgen.exe");
         CLANG_TIDY_CONFUSABLE_CHARS_GEN = (Join-Path -Path $BuildTools -ChildPath "clang-tidy-confusable-chars-gen.exe");
+        CMAKE_FIND_PACKAGE_PREFER_CONFIG = "YES";
         CMAKE_Swift_FLAGS = $SwiftFlags;
+        LibXml2_DIR = "$LibraryRoot\libxml2-2.11.5\usr\lib\Windows\$($Arch.LLVMName)\cmake\libxml2-2.11.5";
         LLDB_PYTHON_EXE_RELATIVE_PATH = "python.exe";
         LLDB_PYTHON_EXT_SUFFIX = ".pyd";
         LLDB_PYTHON_RELATIVE_PATH = "lib/site-packages";
@@ -1487,10 +1522,10 @@ function Build-Compilers() {
         LLVM_NATIVE_TOOL_DIR = $BuildTools;
         LLVM_TABLEGEN = (Join-Path $BuildTools -ChildPath "llvm-tblgen.exe");
         LLVM_USE_HOST_TOOLS = "NO";
-        Python3_EXECUTABLE = "$python";
-        Python3_INCLUDE_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\include";
-        Python3_LIBRARY = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\libs\python39.lib";
-        Python3_ROOT_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools";
+        Python3_EXECUTABLE = (Get-PythonExecutable);
+        Python3_INCLUDE_DIR = "$PythonRoot\include";
+        Python3_LIBRARY = "$PythonRoot\libs\$PythonLibName.lib";
+        Python3_ROOT_DIR = $PythonRoot;
         SWIFT_BUILD_SWIFT_SYNTAX = "YES";
         SWIFT_CLANG_LOCATION = (Get-PinnedToolchainTool);
         SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY = "YES";
@@ -1521,47 +1556,52 @@ function Build-Mimalloc() {
     [hashtable]$Arch
   )
 
-  if ($Arch -eq $ArchX64) {
-    $Args = @()
-    Isolate-EnvVars {
-      Invoke-VsDevShell $Arch
-      # Avoid hard-coding the VC tools version number
-      $VCRedistDir = (Get-ChildItem "${env:VCToolsRedistDir}\$($HostArch.ShortName)" -Filter "Microsoft.VC*.CRT").FullName
-      if ($VCRedistDir) {
-        $Args += "-p:VCRedistDir=$VCRedistDir\"
-      }
-    }
-    $Args += "$SourceCache\mimalloc\ide\vs2022\mimalloc.sln"
-    $Args += "-p:Configuration=Release"
-    $Args += "-p:ProductArchitecture=$($Arch.VSName)"
-    Invoke-Program $msbuild @Args
-    $Dest = "$($Arch.ToolchainInstallRoot)\usr\bin"
-    Copy-Item -Path "$SourceCache\mimalloc\out\msvc-$($Arch.ShortName)\Release\mimalloc-override.dll" `
-      -Destination "$Dest"
-    Copy-Item -Path "$SourceCache\mimalloc\out\msvc-$($Arch.ShortName)\Release\mimalloc-redirect.dll" `
-      -Destination "$Dest"
-    $MimallocExecutables = @("swift.exe","swiftc.exe","swift-driver.exe","swift-frontend.exe")
-    $MimallocExecutables += @("clang.exe","clang++.exe","clang-cl.exe")
-    $MimallocExecutables += @("lld.exe","lld-link.exe","ld.lld.exe","ld64.lld.exe")
-    foreach ($Exe in $MimallocExecutables) {
-      $ExePath = [IO.Path]::Combine($Dest, $Exe)
-      # Binary-patch in place
-      $Args = @()
-      $Args += "-f"
-      $Args += "-i"
-      $Args += "-v"
-      $Args += $ExePath
-      Invoke-Program "$SourceCache\mimalloc\bin\minject" @Args
-      # Log the import table
-      $Args = @()
-      $Args += "-l"
-      $Args += $ExePath
-      Invoke-Program "$SourceCache\mimalloc\bin\minject" @Args
-      dir "$ExePath"
-    }
-  } else {
+  if ($Arch -ne $ArchX64) {
     throw "mimalloc is currently supported for X64 only"
   }
+
+  $MSBuildArgs = @("$SourceCache\mimalloc\ide\vs2022\mimalloc.sln")
+  $MSBuildArgs += "-noLogo"
+  $MSBuildArgs += "-maxCpuCount"
+  $MSBuildArgs += "-p:Configuration=Release"
+  $MSBuildArgs += "-p:ProductArchitecture=$($Arch.VSName)"
+
+  Isolate-EnvVars {
+    Invoke-VsDevShell $Arch
+    # Avoid hard-coding the VC tools version number
+    $VCRedistDir = (Get-ChildItem "${env:VCToolsRedistDir}\$($HostArch.ShortName)" -Filter "Microsoft.VC*.CRT").FullName
+    if ($VCRedistDir) {
+      $MSBuildArgs += "-p:VCRedistDir=$VCRedistDir\"
+    }
+  }
+
+  Invoke-Program $msbuild @MSBuildArgs
+
+  $Products = @( "mimalloc-override.dll", "mimalloc-redirect.dll" )
+  foreach ($Product in $Products) {
+    Copy-Item -Path "$SourceCache\mimalloc\out\msvc-$($Arch.ShortName)\Release\$Product" -Destination "$(Arch.ToolchainInstallRoot)\usr\bin"
+  }
+
+  $Tools = @(
+    "swift.exe",
+    "swiftc.exe",
+    "swift-driver.exe",
+    "swift-frontend.exe",
+    "clang.exe",
+    "clang++.exe",
+    "clang-cl.exe",
+    "lld.exe",
+    "lld-link.exe",
+    "ld.lld.exe",
+    "ld64.lld.exe"
+  )
+  foreach ($Tool in $Tools) {
+    $Binary = [IO.Path]::Combine($Dest, $Tool)
+    # Binary-patch in place
+    Invoke-Program "$SourceCache\mimalloc\bin\minject" @("-f", "-i", "-v", $Binary)
+    # Log the import table
+    Invoke-Program "$SourceCache\mimalloc\bin\minject" @("-l", $Binary)
+  }
 }
 
 function Build-LLVM([Platform]$Platform, $Arch) {
@@ -1815,7 +1855,7 @@ function Build-Runtime([Platform]$Platform, $Arch) {
       })
   }
 
-  Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" `
+  Invoke-Program "$(Get-PythonExecutable)" -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" `
     -OutFile "$($Arch.SDKInstallRoot)\SDKSettings.plist"
 }
 
@@ -1900,14 +1940,9 @@ function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) {
         -Defines (@{
           ENABLE_TESTING = "NO";
           FOUNDATION_BUILD_TOOLS = if ($Platform -eq "Windows") { "YES" } else { "NO" };
+          CMAKE_FIND_PACKAGE_PREFER_CONFIG = "YES";
           CURL_DIR = "$LibraryRoot\curl-8.9.1\usr\lib\$Platform\$ShortArch\cmake\CURL";
-          LIBXML2_LIBRARY = if ($Platform -eq "Windows") {
-            "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2s.lib";
-          } else {
-            "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2.a";
-          };
-          LIBXML2_INCLUDE_DIR = "$LibraryRoot\libxml2-2.11.5\usr\include\libxml2";
-          LIBXML2_DEFINITIONS = "-DLIBXML_STATIC";
+          LibXml2_DIR = "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\cmake\libxml2-2.11.5";
           ZLIB_LIBRARY = if ($Platform -eq "Windows") {
             "$LibraryRoot\zlib-1.3.1\usr\lib\$Platform\$ShortArch\zlibstatic.lib"
           } else {
@@ -2046,7 +2081,7 @@ function Build-Testing([Platform]$Platform, $Arch, [switch]$Test = $false) {
 
 function Write-PlatformInfoPlist($Arch) {
     $PList = Join-Path -Path $Arch.PlatformInstallRoot -ChildPath "Info.plist"
-    Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" `
+    Invoke-Program "$(Get-PythonExecutable)" -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" `
       -OutFile "$PList"
 }
 
@@ -2746,13 +2781,15 @@ if (-not $SkipBuild) {
   Invoke-BuildStep Build-CMark $BuildArch
   Invoke-BuildStep Build-BuildTools $BuildArch
   if ($IsCrossCompiling) {
+    Invoke-BuildStep Build-XML2 Windows $BuildArch
     Invoke-BuildStep Build-Compilers -Build $BuildArch
   }
   if ($IncludeDS2) {
     Invoke-BuildStep Build-RegsGen2 $BuildArch
   }
 
   Invoke-BuildStep Build-CMark $HostArch
+  Invoke-BuildStep Build-XML2 Windows $HostArch
   Invoke-BuildStep Build-Compilers $HostArch
 }