[benchmark] Legacy factor

This adds optional `legacyFactor` to the `BenchmarkInfo`, which allows for linear modification of constants that unnecesarily inflate the base workload of benchmarks, while maintaining the continuity of log-term benchmark tracking.

For example, if a benchmark uses `for _ in N*10_000` in its run function, we could lower this to `for _ in N*1_000` and adding a `legacyFactor: 10` to its `BenchmarkInfo`.

Note that this doesn’t affect the real measurements gathered from the `--verbose` output. The `BenchmarkDoctor` has been slightly adjusted to work with these real samples, therefore `Benchmark_Driver check` will not flag these benchmarks for slow run time reported in the summary, if their real runtimes fall into the recommended range.

Author: palimondo

benchmark/scripts/Benchmark_Driver

@@ -343,7 +343,7 @@ class BenchmarkDoctor(object):
         setup, ratio = BenchmarkDoctor._setup_overhead(measurements)
         setup = 0 if ratio < 0.05 else setup
         runtime = min(
-            [(result.min - correction) for i_series in
+            [(result.samples.min - correction) for i_series in
              [BenchmarkDoctor._select(measurements, num_iters=i)
               for correction in [(setup / i) for i in [1, 2]]
               ] for result in i_series])
@@ -367,7 +367,8 @@ class BenchmarkDoctor(object):
     def _setup_overhead(measurements):
         select = BenchmarkDoctor._select
         ti1, ti2 = [float(min(mins)) for mins in
-                    [[result.min for result in i_series] for i_series in
+                    [[result.samples.min for result in i_series]
+                     for i_series in
                      [select(measurements, num_iters=i) for i in [1, 2]]]]
         setup = int(round(2.0 * (ti1 - ti2)))
         ratio = (setup / ti1) if ti1 > 0 else 0
@@ -439,8 +440,9 @@ class BenchmarkDoctor(object):
         Returns a dictionary with benchmark name and `PerformanceTestResult`s.
         """
         self.log.debug('Calibrating num-samples for {0}:'.format(benchmark))
-        r = self.driver.run(benchmark, num_samples=3, num_iters=1)  # calibrate
-        num_samples = self._adjusted_1s_samples(r.min)
+        r = self.driver.run(benchmark, num_samples=3, num_iters=1,
+                            verbose=True)  # calibrate
+        num_samples = self._adjusted_1s_samples(r.samples.min)
 
         def capped(s):
             return min(s, 2048)
@@ -449,7 +451,7 @@ class BenchmarkDoctor(object):
         opts = opts if isinstance(opts, list) else [opts]
         self.log.debug(
             'Runtime {0} μs yields {1} adjusted samples per second.'.format(
-                r.min, num_samples))
+                r.samples.min, num_samples))
         self.log.debug(
             'Measuring {0}, 5 x i1 ({1} samples), 5 x i2 ({2} samples)'.format(
                 benchmark, run_args[0][0], run_args[1][0]))

benchmark/scripts/test_Benchmark_Driver.py

@@ -423,7 +423,7 @@ def test_no_prefix_for_base_logging(self):
 
 def _PTR(min=700, mem_pages=1000, setup=None):
     """Create PerformanceTestResult Stub."""
-    return Stub(min=min, mem_pages=mem_pages, setup=setup)
+    return Stub(samples=Stub(min=min), mem_pages=mem_pages, setup=setup)
 
 
 def _run(test, num_samples=None, num_iters=None, verbose=None,
@@ -483,7 +483,8 @@ def test_measure_10_independent_1s_benchmark_series(self):
         """
         driver = BenchmarkDriverMock(tests=['B1'], responses=([
             # calibration run, returns a stand-in for PerformanceTestResult
-            (_run('B1', num_samples=3, num_iters=1), _PTR(min=300))] +
+            (_run('B1', num_samples=3, num_iters=1,
+                  verbose=True), _PTR(min=300))] +
             # 5x i1 series, with 300 μs runtime its possible to take 4098
             # samples/s, but it should be capped at 2k
             ([(_run('B1', num_samples=2048, num_iters=1,

benchmark/utils/DriverUtils.swift

@@ -524,6 +524,10 @@ final class TestRunner {
     }
 
     test.tearDownFunction?()
+    if let lf = test.legacyFactor {
+      logVerbose("    Applying legacy factor: \(lf)")
+      samples = samples.map { $0 * lf }
+    }
 
     return BenchResults(samples, maxRSS: measureMemoryUsage())
   }

benchmark/utils/TestsUtils.swift

@@ -26,18 +26,18 @@ public enum BenchmarkCategory : String {
   case runtime, refcount, metadata
   // Other general areas of compiled code validation.
   case abstraction, safetychecks, exceptions, bridging, concurrency
-   
+
   // Algorithms are "micro" that test some well-known algorithm in isolation:
   // sorting, searching, hashing, fibonaci, crypto, etc.
   case algorithm
-   
+
   // Miniapplications are contrived to mimic some subset of application behavior
   // in a way that can be easily measured. They are larger than micro-benchmarks,
   // combining multiple APIs, data structures, or algorithms. This includes small
   // standardized benchmarks, pieces of real applications that have been extracted
   // into a benchmark, important functionality like JSON parsing, etc.
   case miniapplication
-   
+
   // Regression benchmarks is a catch-all for less important "micro"
   // benchmarks. This could be a random piece of code that was attached to a bug
   // report. We want to make sure the optimizer as a whole continues to handle
@@ -46,12 +46,12 @@ public enum BenchmarkCategory : String {
   // as highly as "validation" benchmarks and likely won't be the subject of
   // future investigation unless they significantly regress.
   case regression
-   
+
   // Most benchmarks are assumed to be "stable" and will be regularly tracked at
   // each commit. A handful may be marked unstable if continually tracking them is
   // counterproductive.
   case unstable
-   
+
   // CPU benchmarks represent instrinsic Swift performance. They are useful for
   // measuring a fully baked Swift implementation across different platforms and
   // hardware. The benchmark should also be reasonably applicable to real Swift
@@ -151,16 +151,20 @@ public struct BenchmarkInfo {
     return _tearDownFunction
   }
 
+  public var legacyFactor: Int?
+
   public init(name: String, runFunction: @escaping (Int) -> (), tags: [BenchmarkCategory],
               setUpFunction: (() -> ())? = nil,
               tearDownFunction: (() -> ())? = nil,
-              unsupportedPlatforms: BenchmarkPlatformSet = []) {
+              unsupportedPlatforms: BenchmarkPlatformSet = [],
+              legacyFactor: Int? = nil) {
     self.name = name
     self._runFunction = runFunction
     self.tags = Set(tags)
     self._setUpFunction = setUpFunction
     self._tearDownFunction = tearDownFunction
     self.unsupportedPlatforms = unsupportedPlatforms
+    self.legacyFactor = legacyFactor
   }
 
   /// Returns true if this benchmark should be run on the current platform.
@@ -266,4 +270,3 @@ public func getString(_ s: String) -> String { return s }
 // The same for Substring.
 @inline(never)
 public func getSubstring(_ s: Substring) -> Substring { return s }
-