Add summary to runtime tab

Author: Kobzol

site/frontend/src/pages/compare/compile/common.ts

@@ -1,5 +1,5 @@
 import {BenchmarkFilter, CompareResponse, StatComparison} from "../types";
-import {TestCaseComparison} from "../data";
+import {calculateComparison, TestCaseComparison} from "../data";
 
 export type CompileBenchmarkFilter = {
   profile: {
@@ -67,7 +67,7 @@ export interface CompileTestCase {
 
 export function computeCompileComparisonsWithNonRelevant(
   filter: CompileBenchmarkFilter,
-  data: CompareResponse,
+  comparisons: [CompileBenchmarkComparison],
   benchmarkMap: CompileBenchmarkMap
 ): TestCaseComparison<CompileTestCase>[] {
   function profileFilter(profile: Profile): boolean {
@@ -118,54 +118,29 @@ export function computeCompileComparisonsWithNonRelevant(
     );
   }
 
-  let testCases = data.compile_comparisons
+  let filteredComparisons = comparisons
     .map(
       (c: CompileBenchmarkComparison): TestCaseComparison<CompileTestCase> => {
-        const datumA = c.comparison.statistics[0];
-        const datumB = c.comparison.statistics[1];
-
-        // In the vast majority of cases, we can do the proportional change calculation. However, some
-        // metrics can be zero. If the initial value is 0, we can't compute the new value as a
-        // percentage change of the old one. If both values are 0, we can say the change is also 0%.
-        // If the new value is not 0, the percentage is not really meaningful, but we can say it's 100%.
-        let percent;
-        if (datumA === 0) {
-          if (datumB === 0) {
-            percent = 0;
-          } else {
-            percent = 100;
-          }
-        } else {
-          percent = 100 * ((datumB - datumA) / datumA);
-        }
-
-        return {
-          testCase: {
-            benchmark: c.benchmark,
-            profile: c.profile,
-            scenario: c.scenario,
-            category: (benchmarkMap[c.benchmark] || {}).category || "secondary",
-          },
-          isRelevant: c.comparison.is_relevant,
-          significanceFactor: c.comparison.significance_factor,
-          significanceThreshold: c.comparison.significance_threshold * 100.0, // ensure the threshold is in %
-          datumA,
-          datumB,
-          percent,
+        let testCase = {
+          benchmark: c.benchmark,
+          profile: c.profile,
+          scenario: c.scenario,
+          category: (benchmarkMap[c.benchmark] || {}).category || "secondary",
         };
+        return calculateComparison(c.comparison, testCase);
       }
     )
     .filter((tc) => shouldShowTestCase(tc));
 
   // Sort by name first, so that there is a canonical ordering
   // of test cases. This ensures the overall order is stable, even if
   // individual benchmarks have the same largestChange value.
-  testCases.sort((a, b) =>
+  filteredComparisons.sort((a, b) =>
     a.testCase.benchmark.localeCompare(b.testCase.benchmark)
   );
-  testCases.sort((a, b) => Math.abs(b.percent) - Math.abs(a.percent));
+  filteredComparisons.sort((a, b) => Math.abs(b.percent) - Math.abs(a.percent));
 
-  return testCases;
+  return filteredComparisons;
 }
 
 export function createCompileBenchmarkMap(

site/frontend/src/pages/compare/compile/page.vue

@@ -153,7 +153,7 @@ const benchmarkMap = createCompileBenchmarkMap(props.data);
 const allTestCases = computed(() =>
   computeCompileComparisonsWithNonRelevant(
     filter.value,
-    props.data,
+    props.data.compile_comparisons,
     benchmarkMap
   )
 );

site/frontend/src/pages/compare/data.ts

@@ -1,4 +1,4 @@
-import {BenchmarkFilter} from "./types";
+import {BenchmarkFilter, StatComparison} from "./types";
 
 export interface Summary {
   count: number;
@@ -23,6 +23,39 @@ export interface TestCaseComparison<Case> {
   percent: number;
 }
 
+export function calculateComparison<Case>(
+  comparison: StatComparison,
+  testCase: Case
+): TestCaseComparison<Case> {
+  const datumA = comparison.statistics[0];
+  const datumB = comparison.statistics[1];
+
+  // In the vast majority of cases, we can do the proportional change calculation. However, some
+  // metrics can be zero. If the initial value is 0, we can't compute the new value as a
+  // percentage change of the old one. If both values are 0, we can say the change is also 0%.
+  // If the new value is not 0, the percentage is not really meaningful, but we can say it's 100%.
+  let percent;
+  if (datumA === 0) {
+    if (datumB === 0) {
+      percent = 0;
+    } else {
+      percent = 100;
+    }
+  } else {
+    percent = 100 * ((datumB - datumA) / datumA);
+  }
+
+  return {
+    testCase,
+    isRelevant: comparison.is_relevant,
+    significanceFactor: comparison.significance_factor,
+    significanceThreshold: comparison.significance_threshold * 100.0, // ensure the threshold is in %
+    datumA,
+    datumB,
+    percent,
+  };
+}
+
 export function filterNonRelevant<Case>(
   filter: BenchmarkFilter,
   cases: TestCaseComparison<Case>[]

site/frontend/src/pages/compare/page.vue

@@ -24,6 +24,10 @@ import {
   defaultCompileFilter as defaultCompileFilter,
 } from "./compile/common";
 import RuntimeBenchmarksPage from "./runtime/page.vue";
+import {
+  computeRuntimeComparisonsWithNonRelevant,
+  defaultRuntimeFilter,
+} from "./runtime/common";
 
 function loadSelectorFromUrl(urlParams: Dict<string>): CompareSelector {
   const start = urlParams["start"] ?? "";
@@ -70,11 +74,20 @@ async function loadCompareData(
       defaultCompileFilter,
       computeCompileComparisonsWithNonRelevant(
         defaultCompileFilter,
-        response,
+        response.compile_comparisons,
         createCompileBenchmarkMap(response)
       )
     )
   );
+  runtimeSummary.value = computeSummary(
+    filterNonRelevant(
+      defaultRuntimeFilter,
+      computeRuntimeComparisonsWithNonRelevant(
+        defaultRuntimeFilter,
+        response.runtime_comparisons
+      )
+    )
+  );
   runtimeDataAvailable.value = response.runtime_comparisons.length > 0;
 }
 
@@ -90,10 +103,11 @@ function updateSelection(params: SelectionParams) {
 
 const urlParams = getUrlParams();
 
-// Include all relevant changes in the compile-time tab summary.
-// We do not wrap it in computed, because it should be loaded only once, after
-// the data is downloaded.
+// Include all relevant changes in the compile-time and runtime tab summaries.
+// We do not wrap these summaries in `computed`, because they should be loaded
+// only once, after the compare data is downloaded.
 const compileSummary: Ref<SummaryGroup | null> = ref(null);
+const runtimeSummary: Ref<SummaryGroup | null> = ref(null);
 
 const loading = ref(false);
 
@@ -139,13 +153,14 @@ loadCompareData(selector, loading);
         :data="data"
         :initial-tab="initialTab"
         :compile-time-summary="compileSummary"
+        :runtime-summary="runtimeSummary"
       />
       <template v-if="activeTab === Tab.CompileTime">
         <CompileBenchmarksPage :data="data" :selector="selector" />
       </template>
       <BootstrapTable v-if="activeTab === Tab.Bootstrap" :data="data" />
       <template v-if="runtimeDataAvailable && activeTab === Tab.Runtime">
-        Runtime data
+        <RuntimeBenchmarksPage :data="data" />
       </template>
     </div>
   </div>

site/frontend/src/pages/compare/runtime/common.ts

@@ -1,4 +1,9 @@
-import {BenchmarkFilter, StatComparison} from "../types";
+import {BenchmarkFilter, CompareResponse, StatComparison} from "../types";
+import {calculateComparison, computeSummary, TestCaseComparison} from "../data";
+
+export interface RuntimeTestCase {
+  benchmark: string;
+}
 
 export type RuntimeBenchmarkFilter = BenchmarkFilter;
 
@@ -12,3 +17,35 @@ export interface RuntimeBenchmarkComparison {
   benchmark: string;
   comparison: StatComparison;
 }
+
+export function computeRuntimeComparisonsWithNonRelevant(
+  filter: RuntimeBenchmarkFilter,
+  comparisons: [RuntimeBenchmarkComparison]
+): TestCaseComparison<RuntimeTestCase>[] {
+  function shouldShowTestCase(comparison: TestCaseComparison<RuntimeTestCase>) {
+    const name = comparison.testCase.benchmark;
+    const nameFilter = filter.name && filter.name.trim();
+    return !nameFilter || name.includes(nameFilter);
+  }
+
+  let filteredComparisons = comparisons
+    .map(
+      (c: RuntimeBenchmarkComparison): TestCaseComparison<RuntimeTestCase> => {
+        let testCase = {
+          benchmark: c.benchmark,
+        };
+        return calculateComparison(c.comparison, testCase);
+      }
+    )
+    .filter((tc) => shouldShowTestCase(tc));
+
+  // Sort by name first, so that there is a canonical ordering
+  // of test cases. This ensures the overall order is stable, even if
+  // individual benchmarks have the same largestChange value.
+  filteredComparisons.sort((a, b) =>
+    a.testCase.benchmark.localeCompare(b.testCase.benchmark)
+  );
+  filteredComparisons.sort((a, b) => Math.abs(b.percent) - Math.abs(a.percent));
+
+  return filteredComparisons;
+}

site/frontend/src/pages/compare/runtime/page.vue

@@ -0,0 +1,7 @@
+<script setup lang="ts">
+import {CompareResponse} from "../types";
+
+const props = defineProps<{data: CompareResponse}>();
+</script>
+
+<template></template>

site/frontend/src/pages/compare/tabs.vue

@@ -1,5 +1,5 @@
-<script setup lang="ts">
-import {Ref, ref} from "vue";
+<script setup lang="tsx">
+import {h, ref, Ref} from "vue";
 import {CompareResponse, Tab} from "./types";
 import {diffClass, percentClass} from "./shared";
 import {SummaryGroup} from "./data";
@@ -10,6 +10,7 @@ const props = withDefaults(
   defineProps<{
     data: CompareResponse;
     compileTimeSummary: SummaryGroup;
+    runtimeSummary: SummaryGroup;
     initialTab?: Tab;
   }>(),
   {
@@ -32,10 +33,39 @@ function formatBootstrap(value: number): string {
   return "???";
 }
 
+function SummaryTable({summary}: {summary: SummaryGroup}) {
+  const valid = summary.all.count > 0;
+  if (valid) {
+    return (
+      <table>
+        <thead>
+          <tr>
+            <th>Range</th>
+            <th>Mean</th>
+          </tr>
+        </thead>
+        <thead>
+          <tr>
+            <td>
+              <SummaryRange range={summary.all.range} />
+            </td>
+            <td>
+              <SummaryPercentValue
+                className={percentClass(summary.all.average)}
+                value={summary.all.average}
+              />
+            </td>
+          </tr>
+        </thead>
+      </table>
+    );
+  }
+  return <div>No relevant results</div>;
+}
+
 const bootstrapA = props.data.a.bootstrap_total;
 const bootstrapB = props.data.b.bootstrap_total;
 const bootstrapValid = bootstrapA > 0.0 && bootstrapB > 0.0;
-const compileTimeValid = props.compileTimeSummary.all.count > 0;
 
 const activeTab: Ref<Tab> = ref(props.initialTab);
 </script>
@@ -49,31 +79,8 @@ const activeTab: Ref<Tab> = ref(props.initialTab);
       @click="changeTab(Tab.CompileTime)"
     >
       <div class="title">Compile-time</div>
-      <div class="summary compile-time">
-        <template v-if="compileTimeValid">
-          <table>
-            <thead>
-              <tr>
-                <th>Range</th>
-                <th>Mean</th>
-              </tr>
-            </thead>
-            <thead>
-              <tr>
-                <td>
-                  <SummaryRange :range="compileTimeSummary.all.range" />
-                </td>
-                <td>
-                  <SummaryPercentValue
-                    :class="percentClass(compileTimeSummary.all.average)"
-                    :value="compileTimeSummary.all.average"
-                  />
-                </td>
-              </tr>
-            </thead>
-          </table>
-        </template>
-        <template v-else>No relevant results</template>
+      <div class="summary table-wrapper">
+        <SummaryTable :summary="compileTimeSummary" />
       </div>
     </div>
     <div
@@ -83,7 +90,9 @@ const activeTab: Ref<Tab> = ref(props.initialTab);
       @click="changeTab(Tab.Runtime)"
     >
       <div class="title">Runtime</div>
-      <div class="summary runtime"></div>
+      <div class="summary table-wrapper">
+        <SummaryTable :summary="runtimeSummary" />
+      </div>
     </div>
     <div
       class="tab"
@@ -144,6 +153,7 @@ const activeTab: Ref<Tab> = ref(props.initialTab);
     &:not(:first-child) {
       margin-left: 30px;
     }
+
     &:not(:last-child) {
       :after {
         content: "";
@@ -168,7 +178,7 @@ const activeTab: Ref<Tab> = ref(props.initialTab);
   border-color: black;
 }
 
-.compile-time {
+.table-wrapper {
   table {
     width: 100%;
     table-layout: auto;