[automated] Merge branch 'release/5.0-preview3' => 'master'

.azure/pipelines/ci.yml

@@ -633,6 +633,7 @@ stages:
       jobDisplayName: "Test: Ubuntu 16.04 x64"
       agentOs: Linux
       isTestingJob: true
+      useHostedUbuntu: false
       buildArgs: --all --test "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-linux.sh"

.azure/pipelines/jobs/default-build.yml

@@ -59,6 +59,7 @@ parameters:
   installNodeJs: true
   installJdk: true
   timeoutInMinutes: 180
+  useHostedUbuntu: true
 
   # We need longer than the default amount of 5 minutes to upload our logs/artifacts. (We currently take around 5 mins in the best case).
   # This makes sure we have time to upload everything in the case of a build timeout - really important for investigating a build
@@ -91,7 +92,15 @@ jobs:
       ${{ if eq(parameters.agentOs, 'macOS') }}:
         vmImage: macOS-10.14
       ${{ if eq(parameters.agentOs, 'Linux') }}:
-        vmImage: ubuntu-16.04
+        ${{ if eq(parameters.useHostedUbuntu, true) }}:
+          vmImage: ubuntu-16.04
+        ${{ if eq(parameters.useHostedUbuntu, false) }}:
+          ${{ if eq(variables['System.TeamProject'], 'public') }}:
+            name: NetCorePublic-Pool
+            queue: BuildPool.Ubuntu.1604.Amd64.Open
+          ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+            name: NetCoreInternal-Pool
+            queue: BuildPool.Ubuntu.1604.Amd64
       ${{ if eq(parameters.agentOs, 'Windows') }}:
         ${{ if eq(variables['System.TeamProject'], 'public') }}:
           name: NetCorePublic-Pool
@@ -125,6 +134,9 @@ jobs:
       - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
         - _SignType: test
     steps:
+    - ${{ if ne(parameters.agentOs, 'Windows') }}:
+      - script: df -h
+        displayName: Disk size
     - checkout: self
       clean: true
     - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isTestingJob, true)) }}:
@@ -193,6 +205,13 @@ jobs:
 
     - ${{ parameters.afterBuild }}
 
+    - ${{ if eq(parameters.agentOs, 'Linux') }}:
+      - script: df -h && du -h --threshold=50MB ..
+        displayName: Disk utilization
+    - ${{ if eq(parameters.agentOs, 'macOS') }}:
+      - script: df -h && du -h -d 3 ..
+        displayName: Disk utilization
+
     - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isTestingJob, true)) }}:
       - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/
         displayName: Finish background dump collection

eng/helix/content/RunTests/Program.cs

@@ -23,6 +23,10 @@ static async Task Main(string[] args)
                 {
                     keepGoing = await runner.InstallAspNetAppIfNeededAsync();
                 }
+                if (keepGoing)
+                {
+                    keepGoing = runner.InstallAspNetRefIfNeeded();
+                }
 
                 runner.DisplayContents();
 

eng/helix/content/RunTests/RunTestsOptions.cs

@@ -50,6 +50,16 @@ public static RunTestsOptions Parse(string[] args)
                     aliases: new string[] { "--ef" },
                     description: "The version of the EF tool to use")
                 { Argument = new Argument<string>(), Required = true },
+
+                new Option(
+                    aliases: new string[] { "--aspnetruntime" },
+                    description: "The path to the aspnet runtime nupkg to install")
+                { Argument = new Argument<string>(), Required = true },
+
+                new Option(
+                    aliases: new string[] { "--aspnetref" },
+                    description: "The path to the aspnet ref nupkg to install")
+                { Argument = new Argument<string>(), Required = true },
             };
 
             var parseResult = command.Parse(args);
@@ -61,6 +71,8 @@ public static RunTestsOptions Parse(string[] args)
             options.Architecture = parseResult.ValueForOption<string>("--arch");
             options.Quarantined = parseResult.ValueForOption<bool>("--quarantined");
             options.EfVersion = parseResult.ValueForOption<string>("--ef");
+            options.AspNetRuntime = parseResult.ValueForOption<string>("--aspnetruntime");
+            options.AspNetRef = parseResult.ValueForOption<string>("--aspnetref");
             options.HELIX_WORKITEM_ROOT = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT");
             options.Path = Environment.GetEnvironmentVariable("PATH");
             options.DotnetRoot = Environment.GetEnvironmentVariable("DOTNET_ROOT");
@@ -70,6 +82,8 @@ public static RunTestsOptions Parse(string[] args)
         public string Target { get; set;}
         public string SdkVersion { get; set;}
         public string RuntimeVersion { get; set;}
+        public string AspNetRuntime { get; set;}
+        public string AspNetRef { get; set;}
         public string HelixQueue { get; set;}
         public string Architecture { get; set;}
         public bool Quarantined { get; set;}

eng/helix/content/RunTests/TestRunner.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.CommandLine;
 using System.IO;
+using System.IO.Compression;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 
@@ -91,18 +92,26 @@ public async Task<bool> InstallAspNetAppIfNeededAsync()
         {
             try 
             {
-                Console.WriteLine("Checking for Microsoft.AspNetCore.App/");
-                if (Directory.Exists("Microsoft.AspNetCore.App"))
+                if (File.Exists(Options.AspNetRuntime))
                 {
                     var appRuntimePath = $"{Options.DotnetRoot}/shared/Microsoft.AspNetCore.App/{Options.RuntimeVersion}";
                     Console.WriteLine($"Creating directory: {appRuntimePath}");
                     Directory.CreateDirectory(appRuntimePath);
-                    Console.WriteLine($"Found Microsoft.AspNetCore.App/, copying to {appRuntimePath}");
                     Console.WriteLine($"Set ASPNET_RUNTIME_PATH: {appRuntimePath}");
                     EnvironmentVariables.Add("ASPNET_RUNTIME_PATH", appRuntimePath);
-                    foreach (var file in Directory.EnumerateFiles("Microsoft.AspNetCore.App", "*.*", SearchOption.AllDirectories))
+                    Console.WriteLine($"Found AspNetRuntime: {Options.AspNetRuntime}, extracting *.txt,json,dll to {appRuntimePath}");
+                    using (var archive = ZipFile.OpenRead(Options.AspNetRuntime))
                     {
-                        File.Copy(file, Path.Combine(appRuntimePath, Path.GetFileName(file)), overwrite: true);
+                        foreach (var entry in archive.Entries)
+                        {
+                            // These are the only extensions that end up in the shared fx directory
+                            if (entry.Name.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) ||
+                                entry.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
+                                entry.Name.EndsWith(".dll", StringComparison.OrdinalIgnoreCase))
+                            {
+                                entry.ExtractToFile(Path.Combine(appRuntimePath, entry.Name));
+                            }
+                        }
                     }
 
                     DisplayContents(appRuntimePath);
@@ -135,7 +144,7 @@ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",
                 }
                 else 
                 {
-                    Console.WriteLine($"No app runtime found, skipping...");
+                    Console.WriteLine($"No AspNetRuntime found: {Options.AspNetRuntime}, skipping...");
                 }
                 return true;
             }
@@ -146,6 +155,31 @@ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",
             }
         }
 
+        public bool InstallAspNetRefIfNeeded() 
+        {
+            try 
+            {
+                if (File.Exists(Options.AspNetRef))
+                {
+                    var refPath = $"Microsoft.AspNetCore.App.Ref";
+                    Console.WriteLine($"Found AspNetRef: {Options.AspNetRef}, extracting to {refPath}");
+                    ZipFile.ExtractToDirectory(Options.AspNetRef, "Microsoft.AspNetCore.App.Ref");
+
+                    DisplayContents(refPath);
+                }
+                else 
+                {
+                    Console.WriteLine($"No AspNetRef found: {Options.AspNetRef}, skipping...");
+                }
+                return true;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine($"Exception in InstallAspNetRefIfNeeded: {e.ToString()}");
+                return false;
+            }
+        }
+
         public async Task<bool> CheckTestDiscoveryAsync()
         {
             try

eng/helix/content/runtests.cmd

@@ -21,10 +21,10 @@ echo "Installing Runtime"
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Runtime dotnet -Version %$runtimeVersion% -InstallDir %DOTNET_ROOT%"
 
 set exit_code=0
-echo "Restore for RunTests..."
+echo "Restore: dotnet restore RunTests\RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources..."
 dotnet restore RunTests\RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources
-echo "Running tests..."
-dotnet run --project RunTests\RunTests.csproj -- --target %1 --sdk %2 --runtime %3 --queue %4 --arch %5 --quarantined %6 --ef %7
+echo "Running tests: dotnet run --project RunTests\RunTests.csproj -- --target %1 --sdk %2 --runtime %3 --queue %4 --arch %5 --quarantined %6 --ef %7 --aspnetruntime %8 --aspnetref %9..."
+dotnet run --project RunTests\RunTests.csproj -- --target %1 --sdk %2 --runtime %3 --queue %4 --arch %5 --quarantined %6 --ef %7 --aspnetruntime %8 --aspnetref %9
 if errorlevel 1 (
     set exit_code=1
 )

eng/helix/content/runtests.sh

@@ -86,10 +86,10 @@ fi
 sync
 
 exit_code=0
-echo "Restore for RunTests..."
+echo "Restore: $DOTNET_ROOT/dotnet restore RunTests/RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources..."
 $DOTNET_ROOT/dotnet restore RunTests/RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources
-echo "Running tests..."
-$DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7
+echo "Running tests: $DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7 --aspnetruntime $8 --aspnetref $9..."
+$DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7 --aspnetruntime $8 --aspnetref $9
 exit_code=$?
 echo "Finished tests...exit_code=$exit_code"
 exit $exit_code

eng/scripts/install-nginx-linux.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 curl -sSL http://nginx.org/download/nginx-1.14.2.tar.gz | tar zxfv - -C /tmp && cd /tmp/nginx-1.14.2/
-./configure --prefix=$HOME/nginxinstall --with-http_ssl_module
+./configure --prefix=$HOME/nginxinstall --with-http_ssl_module --without-http_rewrite_module
 make
 make install
-echo "##vso[task.prependpath]$HOME/nginxinstall/sbin"
+echo "##vso[task.prependpath]$HOME/nginxinstall/sbin"