Allow BlazorWasm benchmark driver to perform stress runs

Adds an option that allows running the driver in a loop until timeout expires

Author: pranavkm

src/Components/benchmarkapps/Wasm.Performance/Driver/BenchmarkResultsStartup.cs

@@ -29,7 +29,7 @@ public void Configure(IApplicationBuilder app)
                     PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
                 });
                 await context.Response.WriteAsync("OK");
-                Program.SetBenchmarkResult(result);
+                Program.BenchmarkResultTask.TrySetResult(result);
             });
         }
     }

src/Components/benchmarkapps/Wasm.Performance/Driver/Program.cs

@@ -2,9 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.ExceptionServices;
@@ -23,52 +21,70 @@ namespace Wasm.Performance.Driver
 {
     public class Program
     {
-        static readonly TimeSpan Timeout = TimeSpan.FromMinutes(3);
-        static TaskCompletionSource<BenchmarkResult> benchmarkResult = new TaskCompletionSource<BenchmarkResult>();
+        internal static TaskCompletionSource<BenchmarkResult> BenchmarkResultTask;
 
         public static async Task<int> Main(string[] args)
         {
-            var seleniumPort = 4444;
+            // This cancellation token manages the timeout for the stress run.
+            // By default the driver executes and reports a single Benchmark run. For stress runs,
+            // we'll pass in the duration to execute the runs in milliseconds. This will cause this driver
+            // to repeat executions for the duration specified.
+            var stressRunCancellation = new CancellationToken(canceled: true);
             if (args.Length > 0)
             {
-                if (!int.TryParse(args[0], out seleniumPort))
+                if (!int.TryParse(args[0], out var stressRunSeconds))
                 {
-                    Console.Error.WriteLine("Usage Driver <selenium-port>");
+                    Console.Error.WriteLine("Usage Driver <stress-run-duration>");
                     return 1;
                 }
+
+                if (stressRunSeconds > 0)
+                {
+                    var stressRunDuration = TimeSpan.FromSeconds(stressRunSeconds);
+                    Console.WriteLine($"Stress run duration: {stressRunDuration}.");
+                    stressRunCancellation = new CancellationTokenSource(stressRunDuration).Token;
+                }
             }
 
             // This write is required for the benchmarking infrastructure.
             Console.WriteLine("Application started.");
 
-            var cancellationToken = new CancellationTokenSource(Timeout);
-            cancellationToken.Token.Register(() => benchmarkResult.TrySetException(new TimeoutException($"Timed out after {Timeout}")));
-
-            using var browser = await Selenium.CreateBrowser(seleniumPort, cancellationToken.Token);
+            using var browser = await Selenium.CreateBrowser(default);
             using var testApp = StartTestApp();
             using var benchmarkReceiver = StartBenchmarkResultReceiver();
-
             var testAppUrl = GetListeningUrl(testApp);
             var receiverUrl = GetListeningUrl(benchmarkReceiver);
-
             Console.WriteLine($"Test app listening at {testAppUrl}.");
 
-            var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated";
-            browser.Url = launchUrl;
-            browser.Navigate();
+            var first = true;
+            do
+            {
+                BenchmarkResultTask = new TaskCompletionSource<BenchmarkResult>();
+                var timeForEachRun = TimeSpan.FromMinutes(3);
+                using var runCancellationToken = new CancellationTokenSource(timeForEachRun);
+                runCancellationToken.Token.Register(() => BenchmarkResultTask.TrySetException(new TimeoutException($"Timed out after {timeForEachRun}")));
+
+                if (first)
+                {
+                    first = false;
+                    var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated";
+                    browser.Url = launchUrl;
+                    browser.Navigate();
+                }
+                else
+                {
+                    browser.FindElementById("runAll").Click();
+                }
 
-            FormatAsBenchmarksOutput(benchmarkResult.Task.Result);
+                var results = await BenchmarkResultTask.Task;
+                FormatAsBenchmarksOutput(results, includeStressRunDelimiter: !stressRunCancellation.IsCancellationRequested);
+            } while (!stressRunCancellation.IsCancellationRequested);
 
             Console.WriteLine("Done executing benchmark");
             return 0;
         }
 
-        internal static void SetBenchmarkResult(BenchmarkResult result)
-        {
-            benchmarkResult.TrySetResult(result);
-        }
-
-        private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult)
+        private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult, bool includeStressRunDelimiter)
         {
             // Sample of the the format: https://github.com/aspnet/Benchmarks/blob/e55f9e0312a7dd019d1268c1a547d1863f0c7237/src/Benchmarks/Program.cs#L51-L67
             var output = new BenchmarkOutput();
@@ -126,6 +142,15 @@ private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult)
                 });
             }
 
+            if (includeStressRunDelimiter)
+            {
+                output.Measurements.Add(new BenchmarkMeasurement
+                {
+                    Timestamp = DateTime.UtcNow,
+                    Name = "$$Delimiter$$",
+                });
+            }
+
             Console.WriteLine("#StartJobStatistics");
             Console.WriteLine(JsonSerializer.Serialize(output));
             Console.WriteLine("#EndJobStatistics");
@@ -184,9 +209,9 @@ static void RunInBackgroundThread(Action action)
                 isDone.Set();
             });
 
-            if (!isDone.WaitOne(Timeout))
+            if (!isDone.WaitOne(TimeSpan.FromSeconds(30)))
             {
-                throw new TimeoutException("Timed out waiting for: " + action);
+                throw new TimeoutException("Timed out waiting to start the host");
             }
 
             if (edi != null)

src/Components/benchmarkapps/Wasm.Performance/Driver/Selenium.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -15,6 +15,7 @@ namespace Wasm.Performance.Driver
 {
     class Selenium
     {
+        const int SeleniumPort = 4444;
         static bool RunHeadlessBrowser = true;
         static bool PoolForBrowserLogs = true;
 
@@ -55,9 +56,9 @@ private static async ValueTask<Uri> WaitForServerAsync(int port, CancellationTok
             throw new Exception($"Unable to connect to selenium-server at {uri}");
         }
 
-        public static async Task<RemoteWebDriver> CreateBrowser(int port, CancellationToken cancellationToken)
+        public static async Task<RemoteWebDriver> CreateBrowser(CancellationToken cancellationToken)
         {
-            var uri = await WaitForServerAsync(port, cancellationToken);
+            var uri = await WaitForServerAsync(SeleniumPort, cancellationToken);
 
             var options = new ChromeOptions();
 

src/Components/benchmarkapps/Wasm.Performance/dockerfile

@@ -1,5 +1,6 @@
 FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build
 
+ENV StressRunDuration=0
 ARG DEBIAN_FRONTEND=noninteractive
 
 # Setup for nodejs

src/Components/benchmarkapps/Wasm.Performance/exec.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
 /opt/bin/start-selenium-standalone.sh&
-./Wasm.Performance.Driver
+./Wasm.Performance.Driver $StressRunDuration
 

src/Components/benchmarkapps/Wasm.Performance/local.dockerfile

@@ -1,5 +1,7 @@
 FROM selenium/standalone-chrome:3.141.59-mercury as final
 
+ENV StressRunDuration=0
+
 WORKDIR /app
 COPY ./Driver/bin/Release/netcoreapp3.1/linux-x64/publish ./
 COPY ./exec.sh ./