[benchmark] Keep merged independent run samples

When merging `PerformanceTestResults`s keep the original `PerformanceTestSample`s from all independent runs. These will be used to choose the most stable (least variable) location estimate for the `ResultComparison` down the road.

Author: palimondo

benchmark/scripts/compare_perf_tests.py

@@ -113,7 +113,7 @@ def num_samples(self):
 
     @property
     def all_samples(self):
-        """List of all samples in ascending order."""
+        """List of all samples in original order."""
         return self._all_samples
 
     @property
@@ -236,6 +236,7 @@ def __init__(self, csv_row, quantiles=False, memory=False, delta=False,
             self.samples = PerformanceTestSamples(
                 self.name, [int(runtime) for runtime in runtimes])
             self.samples.exclude_outliers(top_only=True)
+            self.independent_runs = [self.samples]
             sams = self.samples
             self.min, self.max, self.median, self.mean, self.sd = \
                 sams.min, sams.max, sams.median, sams.mean, sams.sd
@@ -277,12 +278,18 @@ def merge(self, r):
         """
         # Statistics
         if self.samples and r.samples:
-            self.samples.samples = sorted(
-                self.samples.samples + r.samples.samples)
-            self.samples._recompute_stats()
+            if hasattr(self, 'independent_runs'):
+                self.independent_runs.append(r.samples)
+            else:
+                self.independent_runs = [self.samples, r.samples]
+            outliers = self.samples.outliers + r.samples.outliers
+            all_samples = self.samples.all_samples + r.samples.all_samples
+            self.samples = PerformanceTestSamples(
+                self.name, sorted(self.samples.samples + r.samples.samples))
             sams = self.samples
             self.num_samples += r.num_samples
-            sams.outliers += r.samples.outliers
+            sams.outliers = outliers
+            sams._all_samples = all_samples
             self.min, self.max, self.median, self.mean, self.sd = \
                 sams.min, sams.max, sams.median, sams.mean, sams.sd
         else:

benchmark/scripts/test_compare_perf_tests.py

@@ -423,18 +423,29 @@ def as_tuple(r):
         r = results[0]
         self.assertEqual(as_tuple(r),
                          (200, 21, 18, 967, 996, 1008, None, None, None))
+        self.assertEqual(len(r.independent_runs), 1)
+        self.assertEqual(r.independent_runs[0], r.samples)
+
         # merging optional --meta data
         results[1].mem_pages = 9
         results[1].involuntary_cs = 1
         results[1].yield_count = 4
         r.merge(results[1])  # 18 + 17 = 35, after merge using only ventiles
         self.assertEqual(as_tuple(r), (400, 42, 35, 967, 983, 1010, 9, 1, 4))
+
         results[2].mem_pages = 7
         results[2].involuntary_cs = 2
         results[2].yield_count = 6
         r.merge(results[2])  # 35 + 18 = 53; sum yields and context switches
         self.assertEqual(as_tuple(r), (600, 63, 53, 967, 989, 1029, 7, 3, 10))
 
+        self.assertEqual(len(r.samples.all_samples), 63)
+        self.assertEqual(r.samples.outliers, [1019, 1095, 2922, 1040, 1186,
+                                              1880, 6470, 1057, 1281, 4183])
+        self.assertEqual(len(r.independent_runs), 3)
+        self.assertEqual([i.count for i in r.independent_runs], [18, 17, 18])
+        self.assertEqual([i.min for i in r.independent_runs], [967, 972, 986])
+
 
 class TestResultComparison(unittest.TestCase):
     def setUp(self):
@@ -744,6 +755,9 @@ def test_results_from_merge_verbose(self):
         samples = result.samples
         self.assertTrue(isinstance(samples, PerformanceTestSamples))
         self.assertEqual(samples.count, 8)
+        self.assertEqual(
+            samples.all_samples,
+            [355883, 358817, 353552, 350815, 363094, 369169, 376131, 364245])
 
     def test_excludes_outliers_from_samples(self):
         verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples.