[benchmark] Benchmark_Driver: faster iterations

With the faster benchmarks in the Swift Benchmark Suite, we now don’t need to spend a whole 1 second measuring each one of them. So I’m adjusting the Benchmark_Driver to sample each one for 50 ms and gather up to 102 actual runtime values (percentiles + MIN + MAX). In my tests, for most optmized benchmarks, the sample distribution was still roughly comparable with full second mesurements. It is more important to gather samples from multiple independent runs to cover the possible variations in distribution.

Author: palimondo

benchmark/scripts/Benchmark_Driver

@@ -184,7 +184,8 @@ class BenchmarkDriver(object):
             return a
 
         return reduce(merge_results,
-                      [self.run(test, num_iters=1, quantile=20,
+                      [self.run(test, num_iters=1, min_samples=10,
+                                sample_time=0.05, quantile=100,
                                 measure_memory=True, gather_metadata=True)
                        for _ in range(self.args.independent_samples)])
 

benchmark/scripts/test_Benchmark_Driver.py

@@ -337,12 +337,13 @@ def test_report_quantiles(self):
             ('/benchmarks/Benchmark_O', 'b', '--quantile=4', '--delta'))
 
     def test_run_benchmark_independent_samples(self):
-        """Extract up to 20 measurements from an independent run."""
+        """Extract up to 100 measurements from an independent run."""
         self.driver.args.independent_samples = 3
         r = self.driver.run_independent_samples('b1')
         self.assertEqual(self.subprocess_mock.calls.count(
-            ('/benchmarks/Benchmark_O', 'b1', '--num-iters=1', '--memory',
-             '--meta', '--quantile=20', '--delta')), 3)
+            ('/benchmarks/Benchmark_O', 'b1', '--min-samples=10',
+             '--num-iters=1', '--sample-time=0.05',
+             '--memory', '--meta', '--quantile=100', '--delta')), 3)
         self.assertEqual(r.num_samples, 3)  # results are merged
 
     def test_run_and_log(self):