Use stages pipeline

.azure/pipelines/jobs/codesign-xplat.yml

@@ -19,13 +19,30 @@ jobs:
       displayName: Download ${{ parameters.inputName }} artifacts
       inputs:
         artifactName: ${{ parameters.inputName }}_Packages
-        downloadPath: $(Build.StagingDirectory)/deps/
+        downloadPath: $(Build.StagingDirectory)/downloaded_packages/
         itemPattern: '**/*.nupkg'
-    - task: MSBuild@1
-      displayName: Code-sign .nupkg files
+    - task: CopyFiles@2
+      displayName: Copy packages to ArtifactsShippingPackagesDir
       inputs:
-        solution: eng\tools\XplatPackageSigner\XplatPackageSigner.proj
-        msbuildArguments: /p:SignType=$(_SignType) /p:DirectoryToSign=$(Build.StagingDirectory)\deps\${{ parameters.inputName }}_Packages\
+        sourceFolder: $(Build.StagingDirectory)/downloaded_packages/
+        contents: '**/*.nupkg'
+        targetFolder: $(Build.SourcesDirectory)/artifacts/packages/$(BuildConfiguration)/shipping/
+        flattenFolders: true
+    - powershell: .\eng\common\build.ps1
+        -ci
+        -restore
+        -sign
+        -publish
+        -configuration $(BuildConfiguration)
+        -projects $(Build.SourcesDirectory)/eng/empty.proj
+        /p:AssetManifestFileName=aspnetcore-${{ parameters.inputName }}-signed.xml
+        /p:DotNetSignType=$(_SignType)
+        $(_BuildArgs)
+        $(_PublishArgs)
+      displayName: Sign and publish packages
     artifacts:
+    - name: CodeSign_Xplat_${{ parameters.inputName }}_Logs
+      path: artifacts/log/
+      publishOnError: true
     - name: ${{ parameters.inputName }}_Packages_Signed
-      path: $(Build.StagingDirectory)\deps\${{ parameters.inputName }}_Packages\
+      path: artifacts/packages/

Directory.Build.props

@@ -21,6 +21,15 @@
     <IsSampleProject Condition="$(RepoRelativeProjectDir.Contains('sample'))">true</IsSampleProject>
     <IsAnalyzersProject Condition="$(MSBuildProjectName.EndsWith('.Analyzers'))">true</IsAnalyzersProject>
     <IsShipping Condition="'$(IsSampleProject)' == 'true' or '$(IsTestAssetProject)' == 'true' or '$(IsBenchmarkProject)' == 'true' or '$(IsUnitTestProject)' == 'true'">false</IsShipping>
+
+    <!--
+      Following logic mimics core-setup approach as well as
+      https://github.com/dotnet/arcade/blob/694d59f090b743f894779d04a7ffe11cbaf352e7/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj#L30-L31
+      $(DotNetFinalVersionKind) is set globally when doing final aka stable builds. Arcade infrastructure should pick
+      up $(IsStableBuild) automatically; property is also used to control prerelease branding.
+    -->
+    <IsStableBuild>false</IsStableBuild>
+    <IsStableBuild Condition="'$(DotNetFinalVersionKind)' == 'release'">true</IsStableBuild>
   </PropertyGroup>
 
   <Import Project="eng\FlakyTests.BeforeArcade.props" />
@@ -119,7 +128,7 @@
     <BuildProjectReferences Condition=" '$(NoBuild)' == 'true' ">false</BuildProjectReferences>
   </PropertyGroup>
 
-  <!-- Artifacts layout -->
+  <!-- Artifacts layout. Keep these values consistent with items defined in eng/Publishing.props. -->
   <PropertyGroup>
     <InstallersOutputPath>$(ArtifactsDir)installers\$(Configuration)\</InstallersOutputPath>
     <SymbolsOutputPath>$(ArtifactsDir)symbols\$(Configuration)\</SymbolsOutputPath>

Directory.Build.targets

@@ -30,7 +30,8 @@
 
   <PropertyGroup Label="Versioning settings">
     <!-- The 'human friendly' version to display in installers. In pre-release builds, this might be "2.0.7 Preview 2 Build 12356". In final builds, it should be "2.0.7" -->
-    <BrandingVersionSuffix>$(PreReleaseBrandingLabel) Build $(VersionSuffix)</BrandingVersionSuffix>
+    <BrandingVersionSuffix />
+    <BrandingVersionSuffix Condition="$(IncludePreReleaseLabelInPackageVersion)">$(PreReleaseBrandingLabel) Build $(VersionSuffix)</BrandingVersionSuffix>
     <PackageBrandingVersion>$(VersionPrefix)</PackageBrandingVersion>
     <PackageBrandingVersion Condition=" '$(VersionSuffix)' != '' ">$(PackageBrandingVersion) $(BrandingVersionSuffix.Trim())</PackageBrandingVersion>
 

build.sh

@@ -221,7 +221,7 @@ if [ "$build_deps" = false ]; then
     msbuild_args[${#msbuild_args[*]}]="-p:BuildProjectReferences=false"
 fi
 
-if [ "$build_managed" = true ] || (["$build_all" = true ] && [ "$build_managed" != false ]); then
+if [ "$build_managed" = true ] || ([ "$build_all" = true ] && [ "$build_managed" != false ]); then
     if [ -z "$build_nodejs" ]; then
         if [ -x "$(command -v node)" ]; then
             __warn "Building of C# project is enabled and has dependencies on NodeJS projects. Building of NodeJS projects is enabled since node is detected on PATH."
@@ -266,6 +266,10 @@ if [ -z "$configuration" ]; then
 fi
 msbuild_args[${#msbuild_args[*]}]="-p:Configuration=$configuration"
 
+# Set verbosity
+echo "Setting msbuild verbosity to $verbosity"
+msbuild_args[${#msbuild_args[*]}]="-verbosity:$verbosity"
+
 # Initialize global variables need to be set before the import of Arcade is imported
 restore=$run_restore
 

dockerbuild.sh

@@ -124,6 +124,7 @@ docker build "$(dirname "$dockerfile")" \
     --build-arg "USER=$(whoami)" \
     --build-arg "USER_ID=$(id -u)" \
     --build-arg "GROUP_ID=$(id -g)" \
+    --build-arg "WORKDIR=$DIR" \
     --tag $tagname \
     -f "$dockerfile"
 
@@ -138,7 +139,7 @@ docker run \
     -e BUILD_SOURCEBRANCH \
     -e DOTNET_CLI_TELEMETRY_OPTOUT \
     -e Configuration \
-    -v "$DIR:/code/build" \
+    -v "$DIR:$DIR" \
     ${docker_args[@]+"${docker_args[@]}"} \
     $tagname \
     ./build.sh \

docs/Artifacts.md

@@ -3,22 +3,36 @@ Artifacts
 
 Building this repo produces build artifacts in the directory structure described below. Build outputs are organized into logical groups based on artifact type and the intended usage of the artifacts.
 
+See also https://github.com/dotnet/arcade/blob/master/Documentation/ArcadeSdk.md This repo follows _most_ of the conventions described there.
+
 ```
 artifacts/
   installers/
     $(Configuration)/
-        *.msi            = Windows installers
-        *.deb, *.rpm     = Linux installers
-        *.zip, *.tar.gz  = archives versions of installers
+      *.msi            = Windows installers
+      *.deb, *.rpm     = Linux installers
+      *.zip, *.tar.gz  = archives versions of installers
+  log/
+    runningProcesses*.txt = Process list from just before build completed
+    runningProcesses*.bak = Process list from two minutes before runningProcesses*.txt files were written
+    *.binlog           = Binary logs for a few build phases e.g. site extension build
+    **/
+      *.log            = Log files for test runs and individual tests
+    $(Configuration)/
+      *.binlog         = Binary logs for most build phases
   packages/
     $(Configuration)/
-        Shipping/        = Packages which are intended for use by customers. These, along with installers, represent the 'product'.
-            *.nupkg      = NuGet packages which ship to nuget.org
-            *.jar        = Java packages which ship to Maven Central and others
-            *.tgz        = NPM packages which ship to npmjs.org
-        NonShipping/
-            *.nupkg      = NuGet packages for internal use only. Used to hand off bits to Microsoft partner teams. Not intended for use by customers.
+      Shipping/        = Packages which are intended for use by customers. These, along with installers, represent the 'product'.
+        *.nupkg        = NuGet packages which ship to nuget.org
+        *.jar          = Java packages which ship to Maven Central and others
+        *.tgz          = NPM packages which ship to npmjs.org
+      NonShipping/
+        *.nupkg        = NuGet packages for internal use only. Used to hand off bits to Microsoft partner teams. Not intended for use by customers.
+  symbols/
+    $(Configuration)/
+      $(TargetFramework)/
+        *.pdb          = Loose symbol files for symbol server publication. Special cases where *.symbols.nupkg packaging is cumbersome.
   VSSetup/
     $(Configuration)/
-        *.vsix           = Visual Studio extensions
+      *.vsix           = Visual Studio extensions. None currently exist.
 ```

eng/AfterSolutionBuild.targets

@@ -17,15 +17,4 @@
       SharedFrameworkTargetFramework="netcoreapp$(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion)" />
   </Target>
 
-  <Target Name="GenerateBuildAssetManifest" AfterTargets="Pack" Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(DotNetBuildFromSource)' != 'true'">
-    <!-- Generate build manifests. These manifests are used by Maestro and the Build Asset Registry to flow dependencies to other repos. -->
-    <MSBuild Projects="$(MSBuildThisFileDirectory)tools\Maestro\Maestro.csproj"
-          Targets="Restore"
-          Properties="__DummyTarget=Restore" />
-
-    <MSBuild Projects="$(MSBuildThisFileDirectory)tools\Maestro\Maestro.csproj"
-             Targets="GenerateBuildAssetManifest"
-             Properties="__DummyTarget=GenerateBuildAssetManifest" />
-  </Target>
-
 </Project>

eng/Publishing.props

@@ -0,0 +1,56 @@
+<Project>
+  <PropertyGroup Condition=" HasTrailingSlash('$(ArtifactsDir)') ">
+    <!-- The one use of ArtifactsDir in Publish.proj adds an additional slash, confusing itself. -->
+    <ArtifactsDir>$(ArtifactsDir.Substring(0, $([MSBuild]::Subtract($(ArtifactsDir.Length), 1))))</ArtifactsDir>
+
+    <PublishDependsOnTargets>$(PublishDependsOnTargets);_PublishInstallers</PublishDependsOnTargets>
+
+    <_UploadPathRoot>aspnetcore</_UploadPathRoot>
+  </PropertyGroup>
+
+  <!-- $(InstallersOutputPath) and $(SymbolsOutputPath) are not defined. Root Directory.Build.props is not imported. -->
+  <ItemGroup>
+    <!-- Include our "loose" PDBs when publishing symbols. -->
+    <FilesToPublishToSymbolServer Include="$(ArtifactsDir)\symbols\**\*.pdb" />
+
+    <!-- Prepare for _PublishInstallers target. -->
+    <_InstallersToPublish Remove="@(_InstallersToPublish)" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\packages\**\*.jar" UploadPathSegment="jar" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\packages\**\*.pom" UploadPathSegment="jar" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\packages\**\*.tgz" UploadPathSegment="npm" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.deb" UploadPathSegment="Runtime" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.exe" UploadPathSegment="Runtime" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.msi" UploadPathSegment="Runtime" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.rpm" UploadPathSegment="Runtime" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.tar.gz" UploadPathSegment="Runtime" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.version" UploadPathSegment="Runtime"
+        Condition=" '$(PublishInstallerBaseVersion)' == 'true' " />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.wixlib" UploadPathSegment="Runtime" />
+    <_InstallersToPublish Include="$(ArtifactsDir)\installers\**\*.zip" UploadPathSegment="Runtime" />
+  </ItemGroup>
+
+  <Target Name="_PublishInstallers">
+    <!-- This target is defined in eng/targets/Packaging.targets and included in every C# and F# project. -->
+    <MSBuild Projects="$(RepoRoot)src\Mvc\Mvc\src\Microsoft.AspNetCore.Mvc.csproj"
+        Targets="_GetPackageVersionInfo"
+        SkipNonexistentProjects="false">
+      <Output TaskParameter="TargetOutputs" ItemName="_ResolvedPackageVersionInfo" />
+    </MSBuild>
+
+    <PropertyGroup>
+      <_PackageVersion>@(_ResolvedPackageVersionInfo->'%(PackageVersion)')</_PackageVersion>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <!-- Do not push .nupkg files from Linux and macOS builds. They'll be packed up separately and signed on Windows. -->
+      <ItemsToPushToBlobFeed Remove="@(ItemsToPushToBlobFeed)" Condition="'$(OS)' != 'Windows_NT'" />
+
+      <ItemsToPushToBlobFeed Include="@(_InstallersToPublish)">
+        <IsShipping>false</IsShipping>
+        <ManifestArtifactData>NonShipping=true;ShipInstaller=dotnetcli</ManifestArtifactData>
+        <PublishFlatContainer>true</PublishFlatContainer>
+        <RelativeBlobPath>$(_UploadPathRoot)/%(_InstallersToPublish.UploadPathSegment)/$(_PackageVersion)/%(Filename)%(Extension)</RelativeBlobPath>
+      </ItemsToPushToBlobFeed>
+    </ItemGroup>
+  </Target>
+</Project>

eng/SignCheckExclusionsFile.txt

@@ -0,0 +1,4 @@
+apphost.exe;; Exclude the apphost because this is expected to be code-signed by customers after the SDK modifies it.
+.js;; We do not sign JavaScript files.
+.binlog;; MSBuild binary logs are not signed though they are sometimes placed where validation thinks they should be.
+WixUIWixca|WixDepCA;; We do not sign WiX content in our installers.

eng/Signing.props

@@ -21,6 +21,7 @@
     -->
     <FileExtensionSignInfo Include=".jar" CertificateName="MicrosoftJARSHA2" />
     <FileExtensionSignInfo Include=".ps1;.psd1;.psm1;.psc1" CertificateName="Microsoft400" />
+    <FileExtensionSignInfo Include=".dll;.exe" CertificateName="Microsoft400" />
     <FileExtensionSignInfo Include=".nupkg" CertificateName="NuGet" />
     <FileExtensionSignInfo Include=".vsix" CertificateName="VsixSHA2" />
     <FileExtensionSignInfo Include=".zip" CertificateName="None" />

eng/Version.Details.xml

@@ -412,15 +412,15 @@
       <Uri>https://github.com/aspnet/Extensions</Uri>
       <Sha>73c259904d5db01bbeede15df2a72d1b94cb3306</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.GenAPI" Version="1.0.0-beta.19456.10">
+    <Dependency Name="Microsoft.DotNet.GenAPI" Version="1.0.0-beta.19458.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>2d393243ba4a0c95c2c18aa266df6e0f43ffe22d</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19456.10">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19458.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>2d393243ba4a0c95c2c18aa266df6e0f43ffe22d</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19456.10">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19458.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>2d393243ba4a0c95c2c18aa266df6e0f43ffe22d</Sha>
     </Dependency>

eng/Versions.props

@@ -10,8 +10,10 @@
     <AspNetCoreMinorVersion>0</AspNetCoreMinorVersion>
     <AspNetCorePatchVersion>0</AspNetCorePatchVersion>
     <PreReleasePreviewNumber>2</PreReleasePreviewNumber>
-    <PreReleaseVersionLabel>rc$(PreReleasePreviewNumber)</PreReleaseVersionLabel>
-    <PreReleaseBrandingLabel>Release Candidate $(PreReleasePreviewNumber)</PreReleaseBrandingLabel>
+    <IncludePreReleaseLabelInPackageVersion>true</IncludePreReleaseLabelInPackageVersion>
+    <IncludePreReleaseLabelInPackageVersion Condition=" '$(IsStableBuild)' == 'true' ">false</IncludePreReleaseLabelInPackageVersion>
+    <PreReleaseVersionLabel Condition="$(IncludePreReleaseLabelInPackageVersion)">rc$(PreReleasePreviewNumber)</PreReleaseVersionLabel>
+    <PreReleaseBrandingLabel Condition="$(IncludePreReleaseLabelInPackageVersion)">Release Candidate $(PreReleasePreviewNumber)</PreReleaseBrandingLabel>
     <!-- Blazor Client packages will not RTM with 3.0 -->
     <BlazorClientPreReleasePreviewNumber>9</BlazorClientPreReleasePreviewNumber>
     <BlazorClientPreReleaseVersionLabel>preview$(BlazorClientPreReleasePreviewNumber)</BlazorClientPreReleaseVersionLabel>
@@ -53,7 +55,7 @@
   -->
   <PropertyGroup Label="Automated">
     <!-- Packages from dotnet/arcade -->
-    <MicrosoftDotNetGenAPIPackageVersion>1.0.0-beta.19456.10</MicrosoftDotNetGenAPIPackageVersion>
+    <MicrosoftDotNetGenAPIPackageVersion>1.0.0-beta.19458.2</MicrosoftDotNetGenAPIPackageVersion>
     <!-- Packages from dotnet/roslyn -->
     <MicrosoftNetCompilersToolsetPackageVersion>3.3.1-beta3-19454-05</MicrosoftNetCompilersToolsetPackageVersion>
     <!-- Packages from dotnet/core-setup -->

eng/docker/alpine.Dockerfile

@@ -2,8 +2,9 @@ FROM microsoft/dotnet:2.1.0-preview1-runtime-deps-alpine
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 RUN mkdir -p "/home/$USER" && chown "${USER_ID}:${GROUP_ID}" "/home/$USER"
 ENV HOME "/home/$USER"
 

eng/docker/bionic.Dockerfile

@@ -3,8 +3,9 @@ FROM microsoft/dotnet:2.1-runtime-deps-bionic
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 RUN mkdir -p "/home/$USER" && chown "${USER_ID}:${GROUP_ID}" "/home/$USER"
 ENV HOME "/home/$USER"
 

eng/docker/rhel.Dockerfile

@@ -5,8 +5,9 @@ FROM mcr.microsoft.com/dotnet-buildtools/prereqs:rhel-7-rpmpkg-e1b4a89-201753110
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 
 RUN useradd -m ${USER} --uid ${USER_ID} -g root
 RUN echo '${USER} ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

eng/docker/ubuntu-alpine37.Dockerfile

@@ -2,8 +2,9 @@ FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine
 ARG USER
 ARG USER_ID
 ARG GROUP_ID
+ARG WORKDIR
 
-WORKDIR /code/build
+WORKDIR ${WORKDIR}
 RUN mkdir -p "/home/$USER" && chown "${USER_ID}:${GROUP_ID}" "/home/$USER"
 ENV HOME "/home/$USER"
 

eng/empty.proj

@@ -0,0 +1,5 @@
+<!-- eng\common\build.ps1 fails without a project to build, so we give it this empty project to satisfy its desires -->
+<Project>
+  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+  <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+</Project>