[benchmark] Restore running benchmarks by numbers

Reintroduced feature lost during `BenchmarkInfo` modernization: All registered benchmarks are ordered alphabetically and assigned an index. This number can be used as a shortcut to invoke the test instead of its full name. (Adding and removing tests from the suite will naturally reassign the indices, but they are stable for a given build.)

The `--list` parameter now prints the test *number*, *name* and *tags* separated by delimiter.

The `--list` output format is modified from:
````
Enabled Tests,Tags
AngryPhonebook,[String, api, validation]
...
````
to this:
````
\#,Test,[Tags]
2,AngryPhonebook,[String, api, validation]
…
````
(There isn’t a backslash before the #, git was eating the whole line without it.)
Note: Test number 1 is Ackermann, which is marked as “skip”, so it’s not listed with the default `skip-tags` value.

Fixes the issue where running tests via `Benchmark_Driver` always reported each test as number 1. Each test is run independently, therefore every invocation was “first”. Restoring test numbers resolves this issue back to original state: The number reported in the first column when executing the tests is its ordinal number in the Swift Benchmark Suite.

Author: palimondo

benchmark/scripts/Benchmark_Driver

@@ -127,19 +127,19 @@ def get_tests(driver_path, args):
     driver.append('--delim=\t')
     if args.benchmarks or args.filters:
         driver.append('--skip-tags=')  # list all tests, don't skip any tags
-    names = [
-        line.split('\t')[0] for line in
+    index_name_pairs = [
+        line.split('\t')[:2] for line in
         subprocess.check_output(driver).split('\n')[1:-1]
     ]
+    indices, names = zip(*index_name_pairs)  # unzip list of pairs into 2 lists
     if args.filters:
         regexes = [re.compile(pattern) for pattern in args.filters]
         return sorted(list(set([name for pattern in regexes
                                 for name in names if pattern.match(name)])))
     if not args.benchmarks:
         return names
     benchmarks = set(args.benchmarks)
-    indices = [str(i) for i in range(1, len(names) + 1)]
-    index_to_name = dict(zip(indices, names))
+    index_to_name = dict(index_name_pairs)
     indexed_names = [index_to_name[i]
                      for i in benchmarks.intersection(set(indices))]
     return sorted(list(

benchmark/utils/DriverUtils.swift

@@ -228,18 +228,21 @@ struct TestConfig {
 
   mutating func findTestsToRun() {
     registeredBenchmarks.sort()
-    let benchmarkNames = Set(filters)
+    let indices = Dictionary(uniqueKeysWithValues:
+      zip(registeredBenchmarks.map{$0.name}, 1...))
+    let benchmarkNamesOrIndices = Set(filters)
     // needed so we don't capture an ivar of a mutable inout self.
     let (_tags, _skipTags) = (tags, skipTags)
 
     tests = registeredBenchmarks.filter { benchmark in
-      if benchmarkNames.isEmpty {
+      if benchmarkNamesOrIndices.isEmpty {
         return benchmark.tags.isSuperset(of: _tags) &&
           benchmark.tags.isDisjoint(with: _skipTags)
       } else {
-        return benchmarkNames.contains(benchmark.name)
+        return benchmarkNamesOrIndices.contains(benchmark.name) ||
+          benchmarkNamesOrIndices.contains(String(indices[benchmark.name]!))
       }
-    }.enumerated().map { Test(benchInfo: $0.element, index: $0.offset + 1) }
+    }.map { Test(benchInfo: $0, index: indices[$0.name]!) }
   }
 }
 
@@ -478,9 +481,9 @@ public func main() {
       fatalError("\(msg)")
     case .listTests:
       config.findTestsToRun()
-      print("Enabled Tests\(config.delim)Tags")
+      print("#\(config.delim)Test\(config.delim)[Tags]")
       for t in config.tests {
-        print("\(t.name)\(config.delim)\(t.tags)")
+        print("\(t.index)\(config.delim)\(t.name)\(config.delim)\(t.tags)")
       }
     case .run:
       config.findTestsToRun()

test/benchmark/Benchmark_O.test-sh

@@ -41,9 +41,9 @@
 // ORSKIPTAGS-NOT: RomanNumbers
 
 // RUN: %Benchmark_O  --list | %FileCheck %s --check-prefix LISTPRECOMMIT
-// LISTPRECOMMIT: Enabled Tests,Tags
+// LISTPRECOMMIT: #,Test,[Tags]
 // LISTPRECOMMIT-NOT: Ackermann
-// LISTPRECOMMIT: AngryPhonebook
+// LISTPRECOMMIT: {{[0-9]+}},AngryPhonebook
 
 // RUN: %Benchmark_O  --list --skip-tags= | %FileCheck %s --check-prefix LISTALL
 // LISTALL: Ackermann