[automated] Merge branch 'release/2.1' => 'release/3.1'

.azure/pipelines/ci.yml

@@ -38,6 +38,19 @@ variables:
     value: ''
   - name: _SignType
     value: ''
+  - name: _InternalRuntimeDownloadArgs
+    value: ''
+  - name: _InternalRuntimeDownloadCodeSignArgs
+    value: ''
+- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - group: DotNet-MSRC-Storage
+  - name: _InternalRuntimeDownloadArgs
+    value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
+  # The code signing doesn't use the aspnet build scripts, so the msbuild parameers have
+  # to be passed directly. This is awkward, since we pass the same info above, but we have
+  # to have it in two different forms
+  - name: _InternalRuntimeDownloadCodeSignArgs
+    value: /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64)
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
@@ -81,7 +94,15 @@ stages:
       jobDisplayName: Code check
       agentOs: Windows
       steps:
-      - powershell: ./eng/scripts/CodeCheck.ps1 -ci
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+      - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs)
         displayName: Run eng/scripts/CodeCheck.ps1
       artifacts:
       - name: Code_Check_Logs
@@ -108,6 +129,14 @@ stages:
       # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even
       # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing.
       # The sign settings have been configured to
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
       - script: ./build.cmd
                 -ci
@@ -117,6 +146,7 @@ stages:
                 -buildNative
                 /bl:artifacts/log/build.x64.binlog
                 $(_BuildArgs)
+                $(_InternalRuntimeDownloadArgs)
         displayName: Build x64
 
       # Build the x86 shared framework
@@ -132,6 +162,7 @@ stages:
                 /p:OnlyPackPlatformSpecificPackages=true
                 /bl:artifacts/log/build.x86.binlog
                 $(_BuildArgs)
+                $(_InternalRuntimeDownloadArgs)
         displayName: Build x86
 
       # This is in a separate build step with -forceCoreMsbuild to workaround MAX_PATH limitations - https://github.com/Microsoft/msbuild/issues/53
@@ -140,6 +171,7 @@ stages:
                 -pack
                 -noBuildDeps
                 $(_BuildArgs)
+                $(_InternalRuntimeDownloadArgs)
         displayName: Build SiteExtension
 
       # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If https://github.com/dotnet/arcade/issues/1957 is resolved,
@@ -165,6 +197,7 @@ stages:
                 /p:AssetManifestFileName=aspnetcore-win-x64-x86.xml
                 $(_BuildArgs)
                 $(_PublishArgs)
+                $(_InternalRuntimeDownloadArgs)
                 /p:PublishInstallerBaseVersion=true
         displayName: Build Installers
 
@@ -205,6 +238,7 @@ stages:
         /p:AssetManifestFileName=aspnetcore-win-arm.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -231,6 +265,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -251,6 +286,14 @@ stages:
       jobDisplayName: "Build: Linux x64"
       agentOs: Linux
       steps:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: Bash@3
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
       - script: ./build.sh
             --ci
             --arch x64
@@ -261,6 +304,7 @@ stages:
             -p:OnlyPackPlatformSpecificPackages=true
             -bl:artifacts/log/build.linux-x64.binlog
             $(_BuildArgs)
+            $(_InternalRuntimeDownloadArgs)
         displayName: Run build.sh
       - script: |
           git clean -xfd src/**/obj/
@@ -274,7 +318,8 @@ stages:
             -p:BuildRuntimeArchive=false \
             -p:LinuxInstallerType=deb \
             -bl:artifacts/log/build.deb.binlog \
-            $(_BuildArgs)
+            $(_BuildArgs) \
+            $(_InternalRuntimeDownloadArgs)
         displayName: Build Debian installers
       - script: |
           git clean -xfd src/**/obj/
@@ -290,7 +335,8 @@ stages:
             -bl:artifacts/log/build.rpm.binlog \
             -p:AssetManifestFileName=aspnetcore-Linux_x64.xml \
             $(_BuildArgs) \
-            $(_PublishArgs)
+            $(_PublishArgs) \
+            $(_InternalRuntimeDownloadArgs)
         displayName: Build RPM installers
       installNodeJs: false
       installJdk: false
@@ -322,6 +368,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_arm.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -352,6 +399,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -385,6 +433,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -418,6 +467,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -439,7 +489,7 @@ stages:
       jobDisplayName: "Test: Windows Server 2016 x64"
       agentOs: Windows
       isTestingJob: true
-      buildArgs: -all -pack -test -BuildNative "/p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false"
+      buildArgs: -all -pack -test -BuildNative "/p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
         displayName: Setup IISExpress test certificates and schema
@@ -475,7 +525,15 @@ stages:
       agentOs: Windows
       isTestingJob: true
       steps:
-      - script: ./build.cmd -ci -all -pack
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+      - script: ./build.cmd -ci -all -pack $(_InternalRuntimeDownloadArgs)
         displayName: Build Repo
       - script: ./src/ProjectTemplates/build.cmd -ci -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.pack.binlog"
         displayName: Pack Templates
@@ -502,7 +560,7 @@ stages:
       jobDisplayName: "Test: macOS 10.13"
       agentOs: macOS
       isTestingJob: true
-      buildArgs: --all --test "/p:RunTemplateTests=false"
+      buildArgs: --all --test "/p:RunTemplateTests=false" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-mac.sh"
         displayName: Installing Nginx
@@ -537,7 +595,7 @@ stages:
       jobDisplayName: "Test: Ubuntu 16.04 x64"
       agentOs: Linux
       isTestingJob: true
-      buildArgs: --all --test "/p:RunTemplateTests=false"
+      buildArgs: --all --test "/p:RunTemplateTests=false"  $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-linux.sh"
         displayName: Installing Nginx
@@ -584,6 +642,25 @@ stages:
         chmod +x $HOME/bin/jq
         echo "##vso[task.prependpath]$HOME/bin"
       displayName: Install jq
+    - task: UseDotNet@2
+      displayName: 'Use .NET Core sdk'
+      inputs:
+        packageType: sdk
+        # The SDK version selected here is intentionally supposed to use the latest release
+        # For the purpose of building Linux distros, we can't depend on features of the SDK
+        # which may not exist in pre-built versions of the SDK
+        # Pinning to preview 8 since preview 9 has breaking changes
+        version: 3.1.100
+        installationPath: $(DotNetCoreSdkDir)
+        includePreviewVersions: true
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: Bash@3
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
     - script: ./eng/scripts/ci-source-build.sh --ci --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false
       displayName: Run ci-source-build.sh
     - task: PublishBuildArtifacts@1

.azure/pipelines/jobs/codesign-xplat.yml

@@ -28,6 +28,14 @@ jobs:
         contents: '**/*.nupkg'
         targetFolder: $(Build.SourcesDirectory)/artifacts/packages/$(BuildConfiguration)/shipping/
         flattenFolders: true
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - task: PowerShell@2
+        displayName: Setup Private Feeds Credentials
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+          arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+        env:
+          Token: $(dn-bot-dnceng-artifact-feeds-rw)
     - powershell: .\eng\common\build.ps1
         -ci
         -restore
@@ -39,6 +47,7 @@ jobs:
         /p:DotNetSignType=$(_SignType)
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadCodeSignArgs)
       displayName: Sign and publish packages
     artifacts:
     - name: CodeSign_Xplat_${{ parameters.inputName }}_Logs

.azure/pipelines/jobs/default-build.yml

@@ -161,6 +161,24 @@ jobs:
     - ${{ if ne(parameters.steps, '')}}:
       - ${{ parameters.steps }}
     - ${{ if eq(parameters.steps, '')}}:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - ${{ if eq(parameters.agentOs, 'Windows') }}:
+          - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+            - task: PowerShell@2
+              displayName: Setup Private Feeds Credentials
+              inputs:
+                filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+                arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+              env:
+                Token: $(dn-bot-dnceng-artifact-feeds-rw)
+        - ${{ if ne(parameters.agentOs, 'Windows') }}:
+          - task: Bash@3
+            displayName: Setup Private Feeds Credentials
+            inputs:
+              filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+              arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+            env:
+              Token: $(dn-bot-dnceng-artifact-feeds-rw)
       - ${{ if eq(parameters.buildScript, '') }}:
         - ${{ if eq(parameters.agentOs, 'Windows') }}:
           - script: .\$(BuildDirectory)\build.cmd -ci /p:DotNetSignType=$(_SignType) -Configuration $(BuildConfiguration) $(BuildScriptArgs)

Directory.Build.props

@@ -84,11 +84,11 @@
     <RuntimeInstallerBaseName>aspnetcore-runtime</RuntimeInstallerBaseName>
     <TargetingPackInstallerBaseName>aspnetcore-targeting-pack</TargetingPackInstallerBaseName>
 
-    <!-- Produce targeting pack installers/packages once per major.minor except in extraordinary cases i.e. 3.0.1. -->
-    <!-- We can remove the 3.0.1 line from any branch other than release/3.0 and from here after 3.0.1 is released. -->
+    <!-- Produce targeting pack installers/packages once per major.minor except in extraordinary cases i.e. 3.1.2. -->
+    <!-- We can remove the 3.1.2 line from any branch other than release/3.1 and from here after 3.1.2 is released. -->
     <IsTargetingPackBuilding Condition=" '$(DotNetBuildFromSource)' == 'true' ">false</IsTargetingPackBuilding>
     <IsTargetingPackBuilding
-        Condition=" '$(IsTargetingPackBuilding)' == '' AND '$(VersionPrefix)' == '3.0.1' ">true</IsTargetingPackBuilding>
+        Condition=" '$(IsTargetingPackBuilding)' == '' AND '$(VersionPrefix)' == '3.1.2' ">true</IsTargetingPackBuilding>
     <IsTargetingPackBuilding
         Condition=" '$(IsTargetingPackBuilding)' == '' AND '$(AspNetCorePatchVersion)' != '0' ">false</IsTargetingPackBuilding>
     <IsTargetingPackBuilding Condition=" '$(IsTargetingPackBuilding)' == '' ">true</IsTargetingPackBuilding>

Directory.Build.targets

@@ -1,8 +1,6 @@
 <Project>
 
   <PropertyGroup>
-    <!-- When OnlyPackPlatformSpecificPackages is set, only produce packages for projects which set RuntimeIdentifier. -->
-    <IsPackable Condition=" '$(OnlyPackPlatformSpecificPackages)' == 'true' AND '$(IsPackable)' != 'false' AND '$(RuntimeIdentifier)' == '' ">false</IsPackable>
 
     <!--
       By default, assemblies which are only in the Microsoft.AspNetCore.App shared framework are not available as NuGet packages.
@@ -67,6 +65,12 @@
     <IsPackable Condition=" '$(IsPackageInThisPatch)' != 'true' ">false</IsPackable>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <!-- When OnlyPackPlatformSpecificPackages is set, only produce packages for projects which set RuntimeIdentifier. -->
+    <!-- Keep this below where we set "IsPackageInThisPatch" -->
+    <IsPackable Condition=" '$(OnlyPackPlatformSpecificPackages)' == 'true' AND '$(RuntimeIdentifier)' == '' ">false</IsPackable>
+  </PropertyGroup>
+
   <PropertyGroup>
     <PackageVersionForPackageVersionInfo>$(PackageVersion)</PackageVersionForPackageVersionInfo>
   </PropertyGroup>

build.ps1

@@ -77,6 +77,12 @@ MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]
 .PARAMETER MSBuildArguments
 Additional MSBuild arguments to be passed through.
 
+.PARAMETER DotNetRuntimeSourceFeed
+Additional feed that can be used when downloading .NET runtimes
+
+.PARAMETER DotNetRuntimeSourceFeedKey
+Key for feed that can be used when downloading .NET runtimes
+
 .EXAMPLE
 Building both native and managed projects.
 
@@ -151,6 +157,11 @@ param(
 
     # Other lifecycle targets
     [switch]$Help, # Show help
+
+    # Optional arguments that enable downloading an internal
+    # runtime or runtime from a non-default location
+    [string]$DotNetRuntimeSourceFeed,
+    [string]$DotNetRuntimeSourceFeedKey,
 
     # Capture the rest
     [Parameter(ValueFromRemainingArguments = $true)]
@@ -251,6 +262,16 @@ if (-not $Configuration) {
 }
 $MSBuildArguments += "/p:Configuration=$Configuration"
 
+[string[]]$ToolsetBuildArguments = @()
+if ($DotNetRuntimeSourceFeed -or $DotNetRuntimeSourceFeedKey) {
+    $runtimeFeedArg = "/p:DotNetRuntimeSourceFeed=$DotNetRuntimeSourceFeed"
+    $runtimeFeedKeyArg = "/p:DotNetRuntimeSourceFeedKey=$DotNetRuntimeSourceFeedKey"
+    $MSBuildArguments += $runtimeFeedArg
+    $MSBuildArguments += $runtimeFeedKeyArg
+    $ToolsetBuildArguments += $runtimeFeedArg
+    $ToolsetBuildArguments += $runtimeFeedKeyArg
+}
+
 $foundJdk = $false
 $javac = Get-Command javac -ErrorAction Ignore -CommandType Application
 $localJdkPath = "$PSScriptRoot\.tools\jdk\win-x64\"
@@ -375,7 +396,8 @@ try {
             /p:Configuration=Release `
             /p:Restore=$RunRestore `
             /p:Build=true `
-            /clp:NoSummary
+            /clp:NoSummary `
+            @ToolsetBuildArguments
     }
 
     MSBuild $toolsetBuildProj `