Fix some test failures

The new code stores test numbers as numbers (not strings), which
requires a few adjustments. I also apparently missed a few test updates.

Author: tbkka

benchmark/scripts/Benchmark_Driver

@@ -144,9 +144,9 @@ class BenchmarkDriver(object):
         """Return a list of performance tests to run."""
         lines = self._invoke(self._cmd_list_benchmarks).split("\n")
         json_tests = []
-        for l in lines:
-            if l.strip() != "":
-                json_tests.append(json.loads(l))
+        for line in lines:
+            if line.strip() != "":
+                json_tests.append(json.loads(line))
         self.all_tests = [json["name"] for json in json_tests]
         test_numbers = [json["number"] for json in json_tests]
         self.test_number = dict([(json["name"], json["number"]) for json in json_tests])
@@ -157,18 +157,19 @@ class BenchmarkDriver(object):
         return self.all_tests
 
     def _tests_matching_patterns(self):
-        regexes = map(re.compile, self.args.filters)
         matches = set()
-        for pattern in regexes:
+        for fil in self.args.filters:
+            pattern = re.compile(fil)
             new_matches = filter(pattern.match, self.all_tests)
-            matches.union(new_matches)
+            matches = matches.union(new_matches)
         return sorted(list(matches))
 
     def _tests_by_name_or_number(self, test_numbers):
         benchmarks = set(self.args.benchmarks)
-        number_to_name = dict(zip(test_numbers, self.all_tests))
+        numbers = list(map(str, test_numbers))
+        number_to_name = dict(zip(numbers, self.all_tests))
         tests_by_number = [
-            number_to_name[i] for i in benchmarks.intersection(test_numbers)
+            number_to_name[i] for i in benchmarks.intersection(numbers)
         ]
         return sorted(
             list(benchmarks.intersection(set(self.all_tests)).union(tests_by_number))
@@ -208,7 +209,7 @@ class BenchmarkDriver(object):
         if test:
             cmd.append(test)
         else:
-            cmd.extend([self.test_number.get(name, name) for name in self.tests])
+            cmd.extend([str(self.test_number.get(name, name)) for name in self.tests])
         if num_samples > 0:
             cmd.append("--num-samples={0}".format(num_samples))
         if num_iters > 0:

benchmark/scripts/compare_perf_tests.py

@@ -254,7 +254,6 @@ def setup(self):
         """
         return 0
 
-
     @property
     def max_rss(self):
         """Return max_rss if available
@@ -646,8 +645,8 @@ def values(result):
             # isinstance(result, ResultComparison)
             (
                 result.name,
-                str(result.old.min_value) if result.old.min_value else "-",
-                str(result.new.min_value) if result.new.min_value else "-",
+                str(result.old.min_value) if result.old.min_value is not None else "-",
+                str(result.new.min_value) if result.new.min_value is not None else "-",
                 "{0:+.1f}%".format(result.delta),
                 "{0:.2f}x{1}".format(result.ratio, " (?)" if result.is_dubious else ""),
             )

benchmark/scripts/test_Benchmark_Driver.py

@@ -208,7 +208,7 @@ def test_test_harness(self):
                 self.args,
                 tests=["ignored"],
                 _subprocess=self.subprocess_mock).test_harness,
-            "/benchmarks/Benchmark_O",
+            "/benchmarks/Benchmark_O-*",
         )
         self.args.tests = "/path"
         self.args.optimization = "Suffix"
@@ -217,28 +217,27 @@ def test_test_harness(self):
                 self.args,
                 tests=["ignored"],
                 _subprocess=self.subprocess_mock).test_harness,
-            "/path/Benchmark_Suffix",
+            "/path/Benchmark_Suffix-*",
         )
 
     def test_gets_list_of_precommit_benchmarks(self):
         self.subprocess_mock.expect(
-            "/benchmarks/Benchmark_O --list".split(" "),
-            "#\tTest\t[Tags]\n1\tBenchmark1\t[t1, t2]\n2\tBenchmark2\t[t3]\n",
+            "/benchmarks/Benchmark_O-* --list --json".split(" "),
+            """{"number":1,"name":"Benchmark1","tags":["t1","t2"]}\n"""
+            + """{"number":2,"name":"Benchmark2","tags":["t3"]}\n""",
         )
         driver = BenchmarkDriver(self.args, _subprocess=self.subprocess_mock)
         self.subprocess_mock.assert_called_all_expected()
         self.assertEqual(driver.tests, ["Benchmark1", "Benchmark2"])
         self.assertEqual(driver.all_tests, ["Benchmark1", "Benchmark2"])
-        self.assertEqual(driver.test_number["Benchmark1"], "1")
-        self.assertEqual(driver.test_number["Benchmark2"], "2")
+        self.assertEqual(driver.test_number["Benchmark1"], 1)
+        self.assertEqual(driver.test_number["Benchmark2"], 2)
 
     list_all_tests = (
-        "/benchmarks/Benchmark_O --list --skip-tags=".split(" "),
-        """#	Test	[Tags]
-1	Benchmark1	[t1, t2]
-2	Benchmark2	[t3]
-3	Benchmark3	[t3, t4]
-""",
+        "/benchmarks/Benchmark_O-* --list --json --skip-tags=".split(" "),
+        """{"number":1, "name": "Benchmark1", "tags":["t1","t2"]}\n"""
+        + """{"number":2, "name": "Benchmark2", "tags":["t3"]}\n"""
+        + """{"number":3, "name": "Benchmark3", "tags":["t3","t4"]}\n""",
     )
 
     def test_gets_list_of_all_benchmarks_when_benchmarks_args_exist(self):
@@ -251,7 +250,7 @@ def test_gets_list_of_all_benchmarks_when_benchmarks_args_exist(self):
         self.assertEqual(driver.all_tests, ["Benchmark1", "Benchmark2", "Benchmark3"])
 
     def test_filters_benchmarks_by_pattern(self):
-        self.args.filters = "-f .+3".split()
+        self.args.filters = [".+3"]
         self.subprocess_mock.expect(*self.list_all_tests)
         driver = BenchmarkDriver(self.args, _subprocess=self.subprocess_mock)
         self.subprocess_mock.assert_called_all_expected()
@@ -320,37 +319,37 @@ def setUp(self):
         self.parser_stub = LogParserStub()
         self.subprocess_mock = SubprocessMock()
         self.subprocess_mock.expect(
-            "/benchmarks/Benchmark_O --list".split(" "),
-            "#\tTest\t[Tags]\n1\tb1\t[tag]\n",
+            "/benchmarks/Benchmark_O-* --list --json".split(" "),
+            """{"number":1, "name":"b1", "tags":["tag"]}""",
         )
         self.driver = BenchmarkDriver(
             self.args, _subprocess=self.subprocess_mock, parser=self.parser_stub
         )
 
     def test_run_benchmark_with_multiple_samples(self):
         self.driver.run("b1")
-        self.subprocess_mock.assert_called_with(("/benchmarks/Benchmark_O", "b1"))
+        self.subprocess_mock.assert_called_with(("/benchmarks/Benchmark_O-*", "b1", "--json"))
         self.driver.run("b2", num_samples=5)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O", "b2", "--num-samples=5")
+            ("/benchmarks/Benchmark_O-*", "b2", "--num-samples=5", "--json")
         )
 
     def test_run_benchmark_with_specified_number_of_iterations(self):
         self.driver.run("b", num_iters=1)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O", "b", "--num-iters=1")
+            ("/benchmarks/Benchmark_O-*", "b", "--num-iters=1", "--json")
         )
 
     def test_run_benchmark_for_specified_time(self):
         self.driver.run("b", sample_time=0.5)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O", "b", "--sample-time=0.5")
+            ("/benchmarks/Benchmark_O-*", "b", "--sample-time=0.5", "--json")
         )
 
     def test_run_benchmark_in_verbose_mode(self):
         self.driver.run("b", verbose=True)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O", "b", "--verbose")
+            ("/benchmarks/Benchmark_O-*", "b", "--verbose", "--json")
         )
 
     def test_run_batch(self):
@@ -361,7 +360,7 @@ def test_run_batch(self):
         """
         self.driver.tests = ["b1", "bx"]
         self.driver.run()
-        self.subprocess_mock.assert_called_with(("/benchmarks/Benchmark_O", "1", "bx"))
+        self.subprocess_mock.assert_called_with(("/benchmarks/Benchmark_O-*", "1", "bx", "--json"))
 
     def test_parse_results_from_running_benchmarks(self):
         """Parse measurements results using LogParser.
@@ -379,7 +378,7 @@ def test_parse_results_from_running_benchmarks(self):
     def test_measure_memory(self):
         self.driver.run("b", measure_memory=True)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O", "b", "--memory")
+            ("/benchmarks/Benchmark_O-*", "b", "--memory", "--json")
         )
 
     def test_run_benchmark_independent_samples(self):
@@ -389,10 +388,11 @@ def test_run_benchmark_independent_samples(self):
         self.assertEqual(
             self.subprocess_mock.calls.count(
                 (
-                    "/benchmarks/Benchmark_O",
+                    "/benchmarks/Benchmark_O-*",
                     "b1",
                     "--num-iters=1",
                     "--memory",
+                    "--json",
                 )
             ),
             3,
@@ -501,7 +501,7 @@ def _run(
     def record_and_respond(self, test, num_samples, num_iters, verbose, measure_memory):
         args = (test, num_samples, num_iters, verbose, measure_memory)
         self.calls.append(args)
-        return self.respond.get(args, _PTR(min=700))
+        return self.respond.get(args, _PTR(min_value=700))
 
 
 class TestLoggingReportFormatter(unittest.TestCase):
@@ -604,9 +604,9 @@ def test_names_in_code_format(self):
         self.assert_contains(["| `QuotedName`"])
 
 
-def _PTR(min=700, mem_pages=1000, setup=None):
+def _PTR(min_value=700, mem_pages=1000, setup=None):
     """Create PerformanceTestResult Stub."""
-    return Stub(samples=Stub(min=min), mem_pages=mem_pages, setup=setup)
+    return Stub(min_value=min_value, mem_pages=mem_pages, setup=setup)
 
 
 def _run(test, num_samples=None, num_iters=None, verbose=None, measure_memory=False):
@@ -677,7 +677,7 @@ def test_measure_10_independent_1s_benchmark_series(self):
                     # calibration run, returns a stand-in for PerformanceTestResult
                     (
                         _run("B1", num_samples=3, num_iters=1, verbose=True),
-                        _PTR(min=300),
+                        _PTR(min_value=300),
                     )
                 ]
                 +
@@ -693,7 +693,7 @@ def test_measure_10_independent_1s_benchmark_series(self):
                                 verbose=True,
                                 measure_memory=True,
                             ),
-                            _PTR(min=300),
+                            _PTR(min_value=300),
                         )
                     ]
                     * 5
@@ -710,7 +710,7 @@ def test_measure_10_independent_1s_benchmark_series(self):
                                 verbose=True,
                                 measure_memory=True,
                             ),
-                            _PTR(min=300),
+                            _PTR(min_value=300),
                         )
                     ]
                     * 5
@@ -838,8 +838,8 @@ def test_benchmark_runtime_range(self):
         def measurements(name, runtime):
             return {
                 "name": name,
-                name + " O i1a": _PTR(min=runtime + 2),
-                name + " O i2a": _PTR(min=runtime),
+                name + " O i1a": _PTR(min_value=runtime + 2),
+                name + " O i2a": _PTR(min_value=runtime),
             }
 
         with captured_output() as (out, _):
@@ -852,8 +852,8 @@ def measurements(name, runtime):
             doctor.analyze(
                 {
                     "name": "OverheadTurtle",
-                    "OverheadTurtle O i1a": _PTR(min=800000),
-                    "OverheadTurtle O i2a": _PTR(min=700000),
+                    "OverheadTurtle O i1a": _PTR(min_value=800000),
+                    "OverheadTurtle O i2a": _PTR(min_value=700000),
                 }
             )
         output = out.getvalue()
@@ -909,30 +909,30 @@ def test_benchmark_has_no_significant_setup_overhead(self):
                 {
                     "name": "NoOverhead",  # not 'significant' enough
                     # Based on DropFirstArray a10/e10: overhead 3.7% (6 μs)
-                    "NoOverhead O i1a": _PTR(min=162),
-                    "NoOverhead O i2a": _PTR(min=159),
+                    "NoOverhead O i1a": _PTR(min_value=162),
+                    "NoOverhead O i2a": _PTR(min_value=159),
                 }
             )
             doctor.analyze(
                 {
                     "name": "SO",  # Setup Overhead
                     # Based on SuffixArrayLazy a10/e10: overhead 5.8% (4 μs)
-                    "SO O i1a": _PTR(min=69),
-                    "SO O i1b": _PTR(min=70),
-                    "SO O i2a": _PTR(min=67),
-                    "SO O i2b": _PTR(min=68),
+                    "SO O i1a": _PTR(min_value=69),
+                    "SO O i1b": _PTR(min_value=70),
+                    "SO O i2a": _PTR(min_value=67),
+                    "SO O i2b": _PTR(min_value=68),
                 }
             )
             doctor.analyze(
-                {"name": "Zero", "Zero O i1a": _PTR(min=0), "Zero O i2a": _PTR(min=0)}
+                {"name": "Zero", "Zero O i1a": _PTR(min_value=0), "Zero O i2a": _PTR(min_value=0)}
             )
             doctor.analyze(
                 {
                     "name": "LOA",  # Limit of Accuracy
                     # Impossible to detect overhead:
                     # Even 1μs change in 20μs runtime is 5%.
-                    "LOA O i1a": _PTR(min=21),
-                    "LOA O i2a": _PTR(min=20),
+                    "LOA O i1a": _PTR(min_value=21),
+                    "LOA O i2a": _PTR(min_value=20),
                 }
             )
         output = out.getvalue()