[BOLT][Utils] Add nfc-stat-parser.py (llvm#71979)

Add a utility to parse output from llvm-bolt-wrapper script and
detect individual and aggregate time/memory swings. The wrapper reports
wall time and peak RSS for each BOLT invocation.

Exit code:
The utility exits with non-zero exit code if any individual test has
time or memory swing larger than `threshold_single` (default 10%), or
the aggregate (geometric mean) swing larger than `threshold_agg`
(default 5%). Short tests where BOLT wall time is less than
`check_longer_than` seconds (default 0.5s) are excluded from threshold
calculation.

Output:
The script prints test results exceeding the individual threshold, and
geomean values if it exceeds aggregate results. In `--verbose` mode all
individual results are printed (short time results are marked with '?').

Example usage:
```
$ cd ~/llvm-build # build folder where NFC testing was invoked
$ python3 ~/llvm-project/bolt/utils/nfc-stat-parser.py \
  --check_longer_than 0.1 `find -name timing.log`
./tools/bolt/test/runtime/X86/exceptions-pic.test/ -88.46% -0.13%
Geomean -19.78% +0.37%
$ echo $?
1
```

Author: aaupov

bolt/utils/nfc-stat-parser.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import re
+import sys
+import os
+from statistics import geometric_mean
+
+TIMING_LOG_RE = re.compile(r"(.*)/(.*).tmp(.*)")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="BOLT NFC stat parser",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "input", nargs="+", help="timing.log files produced by llvm-bolt-wrapper"
+    )
+    parser.add_argument(
+        "--check_longer_than",
+        default=0.5,
+        type=float,
+        help="Only warn on tests longer than X seconds for at least one side",
+    )
+    parser.add_argument(
+        "--threshold_single",
+        default=10,
+        type=float,
+        help="Threshold for a single test result swing, abs percent",
+    ),
+    parser.add_argument(
+        "--threshold_agg",
+        default=5,
+        type=float,
+        help="Threshold for geomean test results swing, abs percent",
+    ),
+    parser.add_argument("--verbose", "-v", action="store_true")
+    args = parser.parse_args()
+
+    def fmt_delta(value, exc_threshold, above_bound=True):
+        formatted_value = format(value, "+.2%")
+        if not above_bound:
+            formatted_value += "?"
+        elif exc_threshold and sys.stdout.isatty():  # terminal supports colors
+            return f"\033[1m{formatted_value}\033[0m"
+        return formatted_value
+
+    # Ratios for geomean computation
+    time_ratios = []
+    mem_ratios = []
+    # Whether any test exceeds the single test threshold (mem or time)
+    threshold_single = False
+    # Whether geomean exceeds aggregate test threshold (mem or time)
+    threshold_agg = False
+
+    if args.verbose:
+        print(f"# Individual test threshold: +-{args.threshold_single}%")
+        print(f"# Aggregate (geomean) test threshold: +-{args.threshold_agg}%")
+        print(
+            f"# Checking time swings for tests with runtime >"
+            f"{args.check_longer_than}s - otherwise marked as ?"
+        )
+        print("Test/binary BOLT_wall_time BOLT_max_rss")
+
+    for input_file in args.input:
+        input_dir = os.path.dirname(input_file)
+        with open(input_file) as timing_file:
+            timing_reader = csv.reader(timing_file, delimiter=";")
+            for row in timing_reader:
+                test_name = row[0]
+                m = TIMING_LOG_RE.match(row[0])
+                if m:
+                    test_name = f"{input_dir}/{m.groups()[1]}/{m.groups()[2]}"
+                else:
+                    # Prepend input dir to unparsed test name
+                    test_name = input_dir + "#" + test_name
+                time_a, time_b = float(row[1]), float(row[3])
+                mem_a, mem_b = int(row[2]), int(row[4])
+                # Check if time is above bound for at least one side
+                time_above_bound = any(
+                    [x > args.check_longer_than for x in [time_a, time_b]]
+                )
+                # Compute B/A ratios (for % delta and geomean)
+                time_ratio = time_b / time_a if time_a else float('nan')
+                mem_ratio = mem_b / mem_a if mem_a else float('nan')
+                # Keep ratios for geomean
+                if time_above_bound and time_ratio > 0:  # must be >0 for gmean
+                    time_ratios += [time_ratio]
+                mem_ratios += [mem_ratio]
+                # Deltas: (B/A)-1 = (B-A)/A
+                time_delta = time_ratio - 1
+                mem_delta = mem_ratio - 1
+                # Check individual test results vs single test threshold
+                time_exc = (
+                    100.0 * abs(time_delta) > args.threshold_single and time_above_bound
+                )
+                mem_exc = 100.0 * abs(mem_delta) > args.threshold_single
+                if time_exc or mem_exc:
+                    threshold_single = True
+                # Print deltas with formatting in verbose mode
+                if args.verbose or time_exc or mem_exc:
+                    print(
+                        test_name,
+                        fmt_delta(time_delta, time_exc, time_above_bound),
+                        fmt_delta(mem_delta, mem_exc),
+                    )
+
+    time_gmean_delta = geometric_mean(time_ratios) - 1
+    mem_gmean_delta = geometric_mean(mem_ratios) - 1
+    time_agg_threshold = 100.0 * abs(time_gmean_delta) > args.threshold_agg
+    mem_agg_threshold = 100.0 * abs(mem_gmean_delta) > args.threshold_agg
+    if time_agg_threshold or mem_agg_threshold:
+        threshold_agg = True
+    if time_agg_threshold or mem_agg_threshold or args.verbose:
+        print(
+            "Geomean",
+            fmt_delta(time_gmean_delta, time_agg_threshold),
+            fmt_delta(mem_gmean_delta, mem_agg_threshold),
+        )
+    exit(threshold_single or threshold_agg)
+
+
+if __name__ == "__main__":
+    main()