Overhaul Benchmarking pipeline to use complete sample data, not summaries

The Swift benchmarking harness now has two distinct output formats:

* Default: Formatted text that's intended for human consumption.
  Right now, this is just the minimum value, but we can augment that.

* `--json`: each output line is a JSON-encoded object that contains raw data
  This information is intended for use by python scripts that aggregate
  or compare multiple independent tests.

Previously, we tried to use the same output for both purposes.  This required
the python scripts to do more complex parsing of textual layouts, and also meant
that the python scripts had only summary data to work with instead of full raw
sample information.  This in turn made it almost impossible to derive meaningful
comparisons between runs or to aggregate multiple runs.

Typical output in the new JSON format looks like this:
```
{"number":89, "name":"PerfTest", "samples":[1.23, 2.35], "max_rss":16384}
{"number":91, "name":"OtherTest", "samples":[14.8, 19.7]}
```

This format is easy to parse in Python.  Just iterate over
lines and decode each one separately. Also note that the
optional fields (`"max_rss"` above) are trivial to handle:
```
import json
for l in lines:
   j = json.loads(l)
   # Default 0 if not present
   max_rss = j.get("max_rss", 0)
```
Note the `"samples"` array includes the runtime for each individual run.

Because optional fields are so much easier to handle in this form, I reworked
the Python logic to translate old formats into this JSON format for more
uniformity.  Hopefully, we can simplify the code in a year or so by stripping
out the old log formats entirely, along with some of the redundant statistical
calculations.  In particular, the python logic still makes an effort to preserve
mean, median, max, min, stdev, and other statistical data whenever the full set
of samples is not present.  Once we've gotten to a point where we're always
keeping full samples, we can compute any such information on the fly as needed,
eliminating the need to record it.

This is a pretty big rearchitecture of the core benchmarking logic. In order to
try to keep things a bit more manageable, I have not taken this opportunity to
replace any of the actual statistics used in the higher level code or to change
how the actual samples are measured. (But I expect this rearchitecture will make
such changes simpler.) In particular, this should not actually change any
benchmark results.

For the future, please keep this general principle in mind: Statistical
summaries (averages, medians, etc) should as a rule be computed for immediate
output and rarely if ever stored or used as input for other processing. Instead,
aim to store and transfer raw data from which statistics can be recomputed as
necessary.

Author: tbkka

benchmark/scripts/Benchmark_Driver

@@ -28,6 +28,7 @@ class `BenchmarkDoctor` analyzes performance tests, implements `check` COMMAND.
 import argparse
 import functools
 import glob
+import json
 import logging
 import math
 import os
@@ -88,9 +89,10 @@ class BenchmarkDriver(object):
     def test_harness(self):
         """Full path to test harness binary."""
         suffix = self.args.optimization if hasattr(self.args, "optimization") else "O"
+        suffix += "-"
         if hasattr(self.args, "architecture") and self.args.architecture:
-            suffix += "-" + self.args.architecture + "*"
-        pattern = os.path.join(self.args.tests, "Benchmark_" + suffix)
+            suffix += self.args.architecture
+        pattern = os.path.join(self.args.tests, "Benchmark_" + suffix + "*")
         executables = []
         if hasattr(self._subprocess, "test_mode") and self._subprocess.test_mode:
             executables = [pattern]
@@ -134,48 +136,39 @@ class BenchmarkDriver(object):
 
     @property
     def _cmd_list_benchmarks(self):
-        # Use tab delimiter for easier parsing to override the default comma.
-        # (The third 'column' is always comma-separated list of tags in square
-        # brackets -- currently unused here.)
-        return [self.test_harness, "--list", "--delim=\t"] + (
+        return [self.test_harness, "--list", "--json"] + (
             ["--skip-tags="] if (self.args.benchmarks or self.args.filters) else []
         )
 
     def _get_tests(self):
         """Return a list of performance tests to run."""
-        number_name_pairs = [
-            line.split("\t")[:2]
-            for line in self._invoke(self._cmd_list_benchmarks).split("\n")[1:-1]
-        ]
-        # unzip list of pairs into 2 lists
-        test_numbers, self.all_tests = map(list, zip(*number_name_pairs))
-        self.test_number = dict(zip(self.all_tests, test_numbers))
+        lines = self._invoke(self._cmd_list_benchmarks).split("\n")
+        json_tests = []
+        for l in lines:
+            if l.strip() != "":
+                json_tests.append(json.loads(l))
+        self.all_tests = [json["name"] for json in json_tests]
+        test_numbers = [json["number"] for json in json_tests]
+        self.test_number = dict([(json["name"], json["number"]) for json in json_tests])
         if self.args.filters:
             return self._tests_matching_patterns()
         if self.args.benchmarks:
             return self._tests_by_name_or_number(test_numbers)
         return self.all_tests
 
     def _tests_matching_patterns(self):
-        regexes = [re.compile(pattern) for pattern in self.args.filters]
-        return sorted(
-            list(
-                set(
-                    [
-                        name
-                        for pattern in regexes
-                        for name in self.all_tests
-                        if pattern.match(name)
-                    ]
-                )
-            )
-        )
+        regexes = map(re.compile, self.args.filters)
+        matches = set()
+        for pattern in regexes:
+            new_matches = filter(pattern.match, self.all_tests)
+            matches.union(new_matches)
+        return sorted(list(matches))
 
     def _tests_by_name_or_number(self, test_numbers):
         benchmarks = set(self.args.benchmarks)
         number_to_name = dict(zip(test_numbers, self.all_tests))
         tests_by_number = [
-            number_to_name[i] for i in benchmarks.intersection(set(test_numbers))
+            number_to_name[i] for i in benchmarks.intersection(test_numbers)
         ]
         return sorted(
             list(benchmarks.intersection(set(self.all_tests)).union(tests_by_number))
@@ -188,16 +181,15 @@ class BenchmarkDriver(object):
         num_iters=None,
         sample_time=None,
         verbose=None,
-        measure_memory=False,
-        quantile=None,
+        measure_memory=False
     ):
         """Execute benchmark and gather results."""
         num_samples = num_samples or 0
         num_iters = num_iters or 0  # automatically determine N to run for 1s
         sample_time = sample_time or 0  # default is 1s
 
         cmd = self._cmd_run(
-            test, num_samples, num_iters, sample_time, verbose, measure_memory, quantile
+            test, num_samples, num_iters, sample_time, verbose, measure_memory
         )
         output = self._invoke(cmd)
         results = self.parser.results_from_string(output)
@@ -210,8 +202,7 @@ class BenchmarkDriver(object):
         num_iters,
         sample_time,
         verbose,
-        measure_memory,
-        quantile,
+        measure_memory
     ):
         cmd = [self.test_harness]
         if test:
@@ -228,9 +219,7 @@ class BenchmarkDriver(object):
             cmd.append("--verbose")
         if measure_memory:
             cmd.append("--memory")
-        if quantile:
-            cmd.append("--quantile={0}".format(quantile))
-            cmd.append("--delta")
+        cmd.append("--json")
         return cmd
 
     def run_independent_samples(self, test):
@@ -246,12 +235,12 @@ class BenchmarkDriver(object):
         return functools.reduce(
             merge_results,
             [
-                self.run(test, measure_memory=True, num_iters=1, quantile=20)
+                self.run(test, measure_memory=True, num_iters=1)
                 for _ in range(self.args.independent_samples)
             ],
         )
 
-    def log_results(self, output, log_file=None):
+    def log_results(self, results, log_file=None):
         """Log output to `log_file`.
 
         Creates `args.output_dir` if it doesn't exist yet.
@@ -262,7 +251,8 @@ class BenchmarkDriver(object):
             os.makedirs(dir)
         print("Logging results to: %s" % log_file)
         with open(log_file, "w") as f:
-            f.write(output)
+            for r in results:
+                print(r, file=f)
 
     RESULT = "{:>3} {:<40} {:>7} {:>7} {:>6} {:>10} {:>6} {:>7} {:>10}"
 
@@ -284,25 +274,25 @@ class BenchmarkDriver(object):
         def console_log(values):
             print(format(values))
 
-        def result_values(r):
+        def summary(r):
             return list(
                 map(
                     str,
                     [
                         r.test_num,
                         r.name,
                         r.num_samples,
-                        r.min,
-                        r.samples.q1,
+                        r.min_value,
+                        r.q1,
                         r.median,
-                        r.samples.q3,
-                        r.max,
+                        r.q3,
+                        r.max_value,
                         r.max_rss,
                     ],
                 )
             )
 
-        header = [
+        summary_header = [
             "#",
             "TEST",
             "SAMPLES",
@@ -313,25 +303,23 @@ class BenchmarkDriver(object):
             "MAX(μs)",
             "MAX_RSS(B)",
         ]
-        console_log(header)
-        results = [header]
+        console_log(summary_header)
+        results = []
         for test in self.tests:
-            result = result_values(self.run_independent_samples(test))
-            console_log(result)
+            result = self.run_independent_samples(test)
+            console_log(summary(result))
             results.append(result)
 
         print("\nTotal performance tests executed: {0}".format(len(self.tests)))
-        return (
-            None if csv_console else ("\n".join([",".join(r) for r in results]) + "\n")
-        )  # csv_log
+        return results
 
     @staticmethod
     def run_benchmarks(args):
         """Run benchmarks and log results."""
         driver = BenchmarkDriver(args)
-        csv_log = driver.run_and_log(csv_console=(args.output_dir is None))
-        if csv_log:
-            driver.log_results(csv_log)
+        results = driver.run_and_log(csv_console=(args.output_dir is None))
+        if args.output_dir:
+            driver.log_results([r.json for r in results])
         return 0
 
 
@@ -444,7 +432,6 @@ class BenchmarkDoctor(object):
         Optional `driver` parameter for injecting dependency; used for testing.
         """
         super(BenchmarkDoctor, self).__init__()
-        self.driver = driver or BenchmarkDriver(args)
         self.results = {}
 
         if hasattr(args, "markdown") and args.markdown:
@@ -457,6 +444,7 @@ class BenchmarkDoctor(object):
             self.console_handler.setLevel(
                 logging.DEBUG if args.verbose else logging.INFO
             )
+        self.driver = driver or BenchmarkDriver(args)
         self.log.addHandler(self.console_handler)
         self.log.debug("Checking tests: %s", ", ".join(self.driver.tests))
         self.requirements = [
@@ -531,7 +519,7 @@ class BenchmarkDoctor(object):
             correction = setup / i
             i_series = BenchmarkDoctor._select(measurements, num_iters=i)
             for result in i_series:
-                runtimes.append(result.samples.min - correction)
+                runtimes.append(result.min_value - correction)
         runtime = min(runtimes)
 
         threshold = 1000
@@ -583,7 +571,7 @@ class BenchmarkDoctor(object):
         ti1, ti2 = [
             float(min(mins))
             for mins in [
-                [result.samples.min for result in i_series]
+                [result.min_value for result in i_series]
                 for i_series in [select(measurements, num_iters=i) for i in [1, 2]]
             ]
         ]
@@ -678,7 +666,7 @@ class BenchmarkDoctor(object):
         r = self.driver.run(
             benchmark, num_samples=3, num_iters=1, verbose=True
         )  # calibrate
-        num_samples = self._adjusted_1s_samples(r.samples.min)
+        num_samples = self._adjusted_1s_samples(r.min_value)
 
         def capped(s):
             return min(s, 200)
@@ -688,7 +676,7 @@ class BenchmarkDoctor(object):
         opts = opts if isinstance(opts, list) else [opts]
         self.log.debug(
             "Runtime {0} μs yields {1} adjusted samples per second.".format(
-                r.samples.min, num_samples
+                r.min_value, num_samples
             )
         )
         self.log.debug(