[release/3.0] Update dependencies from 4 repositories

NuGet.config

@@ -1,13 +1,18 @@
 <?xml version="1.0" encoding="utf-8"?>
+<!-- Keep this file in sync with src/ProjectTemplates/test/Infrastructure/NuGet.config.in. -->
 <configuration>
   <packageSources>
     <clear />
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="aspnet-blazor" value="https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json" />
     <add key="aspnet-extensions" value="https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json" />
     <add key="aspnet-entityframeworkcore" value="https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json" />
     <add key="aspnet-aspnetcore-tooling" value="https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json" />
     <add key="grpc-nuget-dev" value="https://grpc.jfrog.io/grpc/api/nuget/v3/grpc-nuget-dev" />
+    <add key="roslyn" value="https://dotnet.myget.org/F/roslyn/api/v3/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <add key="aspnetcore-dev" value="https://dotnet.myget.org/F/aspnetcore-dev/api/v3/index.json" />
+    <add key="aspnetcore-tools" value="https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json" />
+    <add key="roslyn-tools" value="https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json" />
   </packageSources>
 </configuration>

eng/Workarounds.props

@@ -8,25 +8,6 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
 
-  <!--
-    Workaround for error when running build.cmd from subfolders. It appears RepoRoot is not set correctly.
-    C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\IDE\CommonExtensions\Microsoft\NuGet\NuGet.targets(523,5): error : File 'C:\src\aspnet\AspNetCore\src\..NuGet.config' does not exist.
-  -->
-  <PropertyGroup>
-    <RestoreConfigFile>$(RepoRoot)NuGet.config</RestoreConfigFile>
-  </PropertyGroup>
-
-  <!--
-    Required to restore the RoslynTools.ModifyVsixManifest package.
-    This isn't auto-detected by Arcade like other feeds are.
-  -->
-  <PropertyGroup>
-    <RestoreSources>
-      $(RestoreSources);
-      https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json;
-    </RestoreSources>
-  </PropertyGroup>
-
   <!-- Prevent pdb2pdb.exe from running because it is currently causing build failures in Blazor. -->
   <PropertyGroup>
     <PublishWindowsPdb>false</PublishWindowsPdb>

eng/Workarounds.targets

@@ -29,12 +29,6 @@
   <ItemGroup>
     <PackageReference Include="Internal.AspNetCore.BuildTasks" PrivateAssets="All" Version="$(InternalAspNetCoreBuildTasksPackageVersion)" IsImplicitlyDefined="true" />
   </ItemGroup>
-  <PropertyGroup>
-    <RestoreSources>
-      $(RestoreSources);
-      https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json;
-    </RestoreSources>
-  </PropertyGroup>
 
   <!-- Workaround for netstandard2.1 projects until we can get a preview 8 SDK containing https://github.com/dotnet/sdk/pull/3463 fix. -->
   <ItemGroup>

eng/common/init-tools-native.ps1

@@ -98,12 +98,18 @@ try {
       }
 
       Write-Verbose "Installing $ToolName version $ToolVersion"
-      Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'"
+      Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
       & $InstallerPath @LocalInstallerArguments
       if ($LASTEXITCODE -Ne "0") {
         $errMsg = "$ToolName installation failed"
         if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
-            Write-Warning $errMsg
+            $showNativeToolsWarning = $true
+            if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+                $showNativeToolsWarning = $false
+            }
+            if ($showNativeToolsWarning) {
+                Write-Warning $errMsg
+            }
             $toolInstallationFailure = $true
         } else {
             Write-Error $errMsg

eng/common/init-tools-native.sh

@@ -70,8 +70,7 @@ function ReadGlobalJsonNativeTools {
   # Only extract the contents of the object.
   local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
   native_tools_list=${native_tools_list//[\" ]/}
-  native_tools_list=${native_tools_list//,/$'\n'}
-  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[[:space:]]')"
+  native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
 
   local old_IFS=$IFS
   while read -r line; do
@@ -108,6 +107,7 @@ else
     installer_command+=" --baseuri $base_uri"
     installer_command+=" --installpath $install_bin"
     installer_command+=" --version $tool_version"
+    echo $installer_command
 
     if [[ $force = true ]]; then
       installer_command+=" --force"

eng/common/native/CommonLibrary.psm1

@@ -59,9 +59,38 @@ function DownloadAndExtract {
                                           -Verbose:$Verbose
 
   if ($UnzipStatus -Eq $False) {
-    Write-Error "Unzip failed"
-    return $False
+    # Retry Download one more time with Force=true
+    $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri `
+                                             -Path $TempToolPath `
+                                             -DownloadRetries 1 `
+                                             -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                             -Force:$True `
+                                             -Verbose:$Verbose
+
+    if ($DownloadRetryStatus -Eq $False) {
+      Write-Error "Last attempt of download failed as well"
+      return $False
+    }
+
+    # Retry unzip again one more time with Force=true
+    $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+                                            -OutputDirectory $InstallDirectory `
+                                            -Force:$True `
+                                            -Verbose:$Verbose
+    if ($UnzipRetryStatus -Eq $False)
+    {
+      Write-Error "Last attempt of unzip failed as well"
+      # Clean up partial zips and extracts
+      if (Test-Path $TempToolPath) {
+        Remove-Item $TempToolPath -Force
+      }
+      if (Test-Path $InstallDirectory) {
+        Remove-Item $InstallDirectory -Force -Recurse
+      }
+      return $False
+    }
   }
+
   return $True
 }
 

eng/common/native/install-cmake-test.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  case $lowerI in
+    --baseuri)
+      base_uri=$2
+      shift 2
+      ;;
+    --installpath)
+      install_path=$2
+      shift 2
+      ;;
+    --version)
+      version=$2
+      shift 2
+      ;;
+    --clean)
+      clean=true
+      shift 1
+      ;;
+    --force)
+      force=true
+      shift 1
+      ;;
+    --downloadretries)
+      download_retries=$2
+      shift 2
+      ;;
+    --retrywaittimeseconds)
+      retry_wait_time_seconds=$2
+      shift 2
+      ;;
+    --help)
+      echo "Common settings:"
+      echo "  --baseuri <value>        Base file directory or Url wrom which to acquire tool archives"
+      echo "  --installpath <value>    Base directory to install native tool to"
+      echo "  --clean                  Don't install the tool, just clean up the current install of the tool"
+      echo "  --force                  Force install of tools even if they previously exist"
+      echo "  --help                   Print help and exit"
+      echo ""
+      echo "Advanced settings:"
+      echo "  --downloadretries        Total number of retry attempts"
+      echo "  --retrywaittimeseconds   Wait time between retry attempts in seconds"
+      echo ""
+      exit 0
+      ;;
+  esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+  echo "Cleaning $tool_install_directory"
+  if [[ -d $tool_install_directory ]]; then
+    rm -rf $tool_install_directory
+  fi
+
+  echo "Cleaning $shim_path"
+  if [[ -f $shim_path ]]; then
+    rm -rf $shim_path
+  fi
+
+  tool_temp_path=$(GetTempPathFileName $uri)
+  echo "Cleaning $tool_temp_path"
+  if [[ -f $tool_temp_path ]]; then
+    rm -rf $tool_temp_path
+  fi
+
+  exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+  echo "$tool_name ($version) already exists, skipping install"
+  exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+  echo "Installation failed" >&2
+  exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+  echo "Shim generation failed" >&2
+  exit 1
+fi
+
+exit 0

eng/common/native/install-cmake.sh

@@ -69,7 +69,7 @@ tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
 tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
 shim_path="$install_path/$tool_name.sh"
-uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
 
 # Clean up tool and installers
 if [[ $clean = true ]]; then

eng/common/post-build/darc-gather-drop.ps1

@@ -0,0 +1,35 @@
+param(
+  [Parameter(Mandatory=$true)][int] $BarBuildId,                # ID of the build which assets should be downloaded
+  [Parameter(Mandatory=$true)][string] $DropLocation,           # Where the assets should be downloaded to
+  [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,  # Token used to access Maestro API
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com",     # Maestro API URL
+  [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16"                                            # Version of Maestro API to use
+)
+
+. $PSScriptRoot\post-build-utils.ps1
+
+try {
+  Write-Host "Installing DARC ..."
+
+  . $PSScriptRoot\..\darc-init.ps1
+  $exitCode = $LASTEXITCODE
+
+  if ($exitCode -ne 0) {
+    Write-PipelineTaskError "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..."
+    ExitWithExitCode $exitCode
+  }
+
+  darc gather-drop --non-shipping `
+    --continue-on-error `
+    --id $BarBuildId `
+    --output-dir $DropLocation `
+    --bar-uri $MaestroApiEndpoint `
+    --password $MaestroApiAccessToken `
+    --latest-location
+}
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/post-build/nuget-validation.ps1

@@ -6,10 +6,7 @@ param(
   [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
 )
 
-$ErrorActionPreference = "Stop"
-Set-StrictMode -Version 2.0
-
-. $PSScriptRoot\..\tools.ps1
+. $PSScriptRoot\post-build-utils.ps1
 
 try {
   $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1"