Install java via native tools in CI

.azure/pipelines/ci.yml

@@ -200,7 +200,8 @@ stages:
         steps:
         - script: ./eng/build.cmd
                   -ci
-                  -prepareMachine
+                  -prepareMchine
+                  -nativeToolsOnMachine
                   -arch x64
                   -all
                   $(_BuildArgs)
@@ -252,6 +253,7 @@ stages:
         - script: ./eng/build.cmd
                   -ci
                   -prepareMachine
+                  -nativeToolsOnMachine
                   -arch x64
                   -pack
                   -all
@@ -737,11 +739,11 @@ stages:
           timeoutInMinutes: 240
           steps:
           # Build the shared framework
-          - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -pack -arch x64
+          - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64
                     /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
             displayName: Build shared fx
           # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
-          - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
+          - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
                     -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
                     /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
             displayName: Run build.cmd helix target

.azure/pipelines/jobs/codesign-xplat.yml

@@ -13,7 +13,6 @@ jobs:
     jobDisplayName: "Code-sign ${{ parameters.inputName }} packages"
     agentOs: Windows
     installNodeJs: false
-    installJdk: false
     steps:
     - task: DownloadBuildArtifacts@0
       displayName: Download ${{ parameters.inputName }} artifacts

.azure/pipelines/jobs/default-build.yml

@@ -68,7 +68,6 @@ parameters:
   container: ''
   enableRichCodeNavigation: ''
   installNodeJs: true
-  installJdk: true # Ignored unless agentOs == Windows.
   timeoutInMinutes: 180
   testRunTitle: $(AgentOsName)-$(BuildConfiguration)
   useHostedUbuntu: true
@@ -147,8 +146,6 @@ jobs:
       - LC_ALL: 'en_US.UTF-8'
       - LANG: 'en_US.UTF-8'
       - LANGUAGE: 'en_US.UTF-8'
-    - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-      - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64
     - ${{ if or(ne(parameters.codeSign, true), ne(variables['System.TeamProject'], 'internal')) }}:
       - _SignType: ''
     - ${{ if and(eq(parameters.codeSign, true), eq(variables['System.TeamProject'], 'internal')) }}:
@@ -202,14 +199,11 @@ jobs:
         displayName: Install Node 20.x
         inputs:
           versionSpec: 20.x
-    - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-      - powershell: ./eng/scripts/InstallJdk.ps1
-        displayName: Install JDK 11
-      - ${{ if eq(parameters.isAzDOTestingJob, true) }}:
-        - powershell: |
-            Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
-            ./eng/scripts/InstallGoogleChrome.ps1
-          displayName: Install Chrome
+    - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}:
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
+          ./eng/scripts/InstallGoogleChrome.ps1
+        displayName: Install Chrome
     - ${{ if eq(parameters.agentOs, 'Windows') }}:
       - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools"
         displayName: Add dotnet tools to path
@@ -256,7 +250,7 @@ jobs:
                 ${{ step.env }}
     - ${{ if eq(parameters.steps, '')}}:
       - ${{ if eq(parameters.agentOs, 'Windows') }}:
-        - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
+        - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
             /p:DotNetSignType=$(_SignType)
           displayName: Run build.cmd
           env:

eng/build.ps1

@@ -15,6 +15,9 @@ Sets up CI specific settings and variables.
 .PARAMETER PrepareMachine
 In CI, Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
 
+.PARAMETER NativeToolsOnMachine
+Turns on native tooling handling. On CI machines, promotes native tools listed in global.json to the path.
+
 .PARAMETER Restore
 Run restore.
 
@@ -128,6 +131,7 @@ Online version: https://github.com/dotnet/aspnetcore/blob/main/docs/BuildFromSou
 param(
     [switch]$CI,
     [switch]$PrepareMachine,
+    [switch]$NativeToolsOnMachine,
 
     # Build lifecycle options
     [switch]$Restore,
@@ -312,58 +316,6 @@ $performDesktopBuild = ($BuildInstallers -and $Architecture -ne "arm") -or `
 $performDotnetBuild = $BuildJava -or $BuildManaged -or $BuildNodeJS -or `
     ($All -and -not ($NoBuildJava -and $NoBuildManaged -and $NoBuildNodeJS)) -or `
     ($Projects -and -not ($BuildInstallers -or $specifiedBuildNative))
-$foundJdk = $false
-$javac = Get-Command javac -ErrorAction Ignore -CommandType Application
-$localJdkPath = "$PSScriptRoot\..\.tools\jdk\win-x64\"
-if (Test-Path "$localJdkPath\bin\javac.exe") {
-    $foundJdk = $true
-    Write-Host -f Magenta "Detected JDK in $localJdkPath (via local repo convention)"
-    $env:JAVA_HOME = $localJdkPath
-}
-elseif ($env:JAVA_HOME) {
-    if (-not (Test-Path "${env:JAVA_HOME}\bin\javac.exe")) {
-        Write-Error "The environment variable JAVA_HOME was set, but ${env:JAVA_HOME}\bin\javac.exe does not exist. Remove JAVA_HOME or update it to the correct location for the JDK. See https://www.bing.com/search?q=java_home for details."
-    }
-    else {
-        Write-Host -f Magenta "Detected JDK in ${env:JAVA_HOME} (via JAVA_HOME)"
-        $foundJdk = $true
-    }
-}
-elseif ($javac) {
-    $foundJdk = $true
-    $javaHome = Split-Path -Parent (Split-Path -Parent $javac.Path)
-    $env:JAVA_HOME = $javaHome
-    Write-Host -f Magenta "Detected JDK in $javaHome (via PATH)"
-}
-else {
-    try {
-        $jdkRegistryKeys = @(
-            "HKLM:\SOFTWARE\JavaSoft\JDK",  # for JDK 10+
-            "HKLM:\SOFTWARE\JavaSoft\Java Development Kit"  # fallback for JDK 8
-        )
-        $jdkRegistryKey = $jdkRegistryKeys | Where-Object { Test-Path $_ } | Select-Object -First 1
-        if ($jdkRegistryKey) {
-            $jdkVersion = (Get-Item $jdkRegistryKey | Get-ItemProperty -name CurrentVersion).CurrentVersion
-            $javaHome = (Get-Item $jdkRegistryKey\$jdkVersion | Get-ItemProperty -Name JavaHome).JavaHome
-            if (Test-Path "${javaHome}\bin\javac.exe") {
-                $env:JAVA_HOME = $javaHome
-                Write-Host -f Magenta "Detected JDK $jdkVersion in $env:JAVA_HOME (via registry)"
-                $foundJdk = $true
-            }
-        }
-    }
-    catch {
-        Write-Verbose "Failed to detect Java: $_"
-    }
-}
-
-if ($env:PATH -notlike "*${env:JAVA_HOME}*") {
-    $env:PATH = "$(Join-Path $env:JAVA_HOME bin);${env:PATH}"
-}
-
-if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJava) {
-    Write-Error "Could not find the JDK. Either run $PSScriptRoot\scripts\InstallJdk.ps1 to install for this repo, or install the JDK globally on your machine (see $PSScriptRoot\..\docs\BuildFromSource.md for details)."
-}
 
 # Initialize global variables need to be set before the import of Arcade is imported
 $restore = $RunRestore
@@ -392,6 +344,61 @@ Remove-Item variable:global:_MSBuildExe -ea Ignore
 # Import Arcade
 . "$PSScriptRoot/common/tools.ps1"
 
+function LocateJava {
+    $foundJdk = $false
+    $javac = Get-Command javac -ErrorAction Ignore -CommandType Application
+    $localJdkPath = "$PSScriptRoot\..\.tools\jdk\win-x64\"
+    if (Test-Path "$localJdkPath\bin\javac.exe") {
+        $foundJdk = $true
+        Write-Host -f Magenta "Detected JDK in $localJdkPath (via local repo convention)"
+        $env:JAVA_HOME = $localJdkPath
+    }
+    elseif ($env:JAVA_HOME) {
+        if (-not (Test-Path "${env:JAVA_HOME}\bin\javac.exe")) {
+            Write-Error "The environment variable JAVA_HOME was set, but ${env:JAVA_HOME}\bin\javac.exe does not exist. Remove JAVA_HOME or update it to the correct location for the JDK. See https://www.bing.com/search?q=java_home for details."
+        }
+        else {
+            Write-Host -f Magenta "Detected JDK in ${env:JAVA_HOME} (via JAVA_HOME)"
+            $foundJdk = $true
+        }
+    }
+    elseif ($javac) {
+        $foundJdk = $true
+        $javaHome = Split-Path -Parent (Split-Path -Parent $javac.Path)
+        $env:JAVA_HOME = $javaHome
+        Write-Host -f Magenta "Detected JDK in $javaHome (via PATH)"
+    }
+    else {
+        try {
+            $jdkRegistryKeys = @(
+                "HKLM:\SOFTWARE\JavaSoft\JDK",  # for JDK 10+
+                "HKLM:\SOFTWARE\JavaSoft\Java Development Kit"  # fallback for JDK 8
+            )
+            $jdkRegistryKey = $jdkRegistryKeys | Where-Object { Test-Path $_ } | Select-Object -First 1
+            if ($jdkRegistryKey) {
+                $jdkVersion = (Get-Item $jdkRegistryKey | Get-ItemProperty -name CurrentVersion).CurrentVersion
+                $javaHome = (Get-Item $jdkRegistryKey\$jdkVersion | Get-ItemProperty -Name JavaHome).JavaHome
+                if (Test-Path "${javaHome}\bin\javac.exe") {
+                    $env:JAVA_HOME = $javaHome
+                    Write-Host -f Magenta "Detected JDK $jdkVersion in $env:JAVA_HOME (via registry)"
+                    $foundJdk = $true
+                }
+            }
+        }
+        catch {
+            Write-Verbose "Failed to detect Java: $_"
+        }
+    }
+
+    if ($env:PATH -notlike "*${env:JAVA_HOME}*") {
+        $env:PATH = "$(Join-Path $env:JAVA_HOME bin);${env:PATH}"
+    }
+
+    if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJava) {
+        Write-Error "Could not find the JDK. Either run $PSScriptRoot\scripts\InstallJdk.ps1 to install for this repo, or install the JDK globally on your machine (see $PSScriptRoot\..\docs\BuildFromSource.md for details)."
+    }
+}
+
 # Add default .binlog location if not already on the command line. tools.ps1 does not handle this; it just checks
 # $BinaryLog, $CI and $ExcludeCIBinarylog values for an error case. But tools.ps1 provides a nice function to help.
 if ($BinaryLog) {
@@ -424,6 +431,17 @@ try {
     $tmpRestore = $restore
     $restore = $true
 
+    # Initialize the native tools before locating java.
+    if ($NativeToolsOnMachine) {
+        $env:NativeToolsOnMachine=$true
+        # Do not promote native tools except in cases where -NativeToolsOnMachine is passed.
+        # Currently the JDK is laid out in an incorrect pattern: https://github.com/dotnet/dnceng/issues/2185
+        InitializeNativeTools
+    }
+
+    # Locate java, now that we may have java available after initializing native tools.
+    LocateJava
+
     $toolsetBuildProj = InitializeToolset
 
     $restore = $tmpRestore

eng/common/tools.ps1

@@ -296,7 +296,7 @@ function InstallDotNet([string] $dotnetRoot,
     if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" }
     if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" }
     $runtimePath = $runtimePath + "\" + $version
-  
+
     $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'"
 
     if (Test-Path $runtimePath) {
@@ -608,7 +608,7 @@ function InitializeBuildTool() {
     } else {
       $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
     }
-    
+
     $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
   } elseif ($msbuildEngine -eq "vs") {
     try {

eng/configure-toolset.sh

@@ -5,3 +5,6 @@
 if [ "${DotNetBuildFromSource:-false}" = false ]; then
     use_installed_dotnet_cli="false"
 fi
+
+# Working around issue https://github.com/dotnet/arcade/issues/2673
+DisableNativeToolsetInstalls=true

eng/scripts/InstallJdk.ps1

@@ -23,7 +23,7 @@ $javacExe = "$installDir\bin\javac.exe"
 $tempDir = "$repoRoot\obj"
 if (-not $JdkVersion) {
     $globalJson = Get-Content "$repoRoot\global.json" | ConvertFrom-Json
-    $JdkVersion = $globalJson.tools.jdk
+    $JdkVersion = $globalJson.'native-tools'.jdk
 }
 
 if (Test-Path $javacExe) {
@@ -40,12 +40,13 @@ Remove-Item -Force -Recurse $tempDir -ErrorAction Ignore | out-null
 mkdir $tempDir -ea Ignore | out-null
 mkdir $installDir -ea Ignore | out-null
 Write-Host "Starting download of JDK ${JdkVersion}"
-& $PSScriptRoot\Download.ps1 "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/java/jdk-${JdkVersion}_windows-x64_bin.zip" "$tempDir/jdk.zip"
+& $PSScriptRoot\Download.ps1 "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/java/microsoft-jdk-${JdkVersion}-windows-x64.zip" "$tempDir/jdk.zip"
 Write-Host "Done downloading JDK ${JdkVersion}"
 Expand-Archive "$tempDir/jdk.zip" -d "$tempDir/jdk/"
 Write-Host "Expanded JDK to $tempDir"
 Write-Host "Installing JDK to $installDir"
-Move-Item "$tempDir/jdk/jdk-${JdkVersion}/*" $installDir
+# The name of the file directory within the zip is based on the version, but may contain a +N for build number.
+Move-Item "$(Get-ChildItem -Path "$tempDir/jdk" | Select-Object -First 1)/*" $installDir
 Write-Host "Done installing JDK to $installDir"
 
 if ($env:TF_BUILD) {

global.json

@@ -12,8 +12,6 @@
         "$(MicrosoftNETCoreBrowserDebugHostTransportVersion)"
       ]
     },
-    "Git": "2.22.0",
-    "jdk": "11.0.3",
     "vs": {
       "version": "17.2",
       "components": [
@@ -25,6 +23,9 @@
     },
     "xcopy-msbuild": "17.1.0"
   },
+  "native-tools": {
+    "jdk": "11.0.22"
+  },
   "msbuild-sdks": {
     "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24159.1",
     "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24159.1"