Validate that benchmark commits contain freshly generated test harness

Author: palimondo

benchmark/scripts/generate_harness/generate_harness.py

@@ -12,51 +12,70 @@
 #
 # ===---------------------------------------------------------------------===//
 
-# Generate CMakeLists.txt and utils/main.swift from templates.
+# Generate boilerplate, CMakeLists.txt and utils/main.swift from templates.
 
 from __future__ import print_function
 
-import glob
+import argparse
+import jinja2
 import os
 import re
 import subprocess
 
-import jinja2
-
 script_dir = os.path.dirname(os.path.realpath(__file__))
 perf_dir = os.path.realpath(os.path.join(script_dir, '../..'))
+gyb_script = os.path.realpath(os.path.join(perf_dir, '../utils/gyb'))
 single_source_dir = os.path.join(perf_dir, 'single-source')
 multi_source_dir = os.path.join(perf_dir, 'multi-source')
+parser = argparse.ArgumentParser()
+parser.add_argument("--output-dir",
+                    help="Output directory (for validation test)",
+                    default=perf_dir)
+args = parser.parse_args()
+output_dir = args.output_dir
 
 template_map = {
-    'CMakeLists.txt_template': os.path.join(perf_dir, 'CMakeLists.txt'),
-    'main.swift_template': os.path.join(perf_dir, 'utils/main.swift')
+    'CMakeLists.txt_template': os.path.join(output_dir, 'CMakeLists.txt'),
+    'main.swift_template': os.path.join(output_dir, 'utils/main.swift')
 }
 ignored_run_funcs = ["Ackermann", "Fibonacci"]
 
 template_loader = jinja2.FileSystemLoader(searchpath="/")
 template_env = jinja2.Environment(loader=template_loader, trim_blocks=True,
                                   lstrip_blocks=True)
 
+def all_files(directory, extension): # matching: [directory]/**/*[extension]
+    return [
+        os.path.join(root, f)
+        for root, _, files in os.walk(directory)
+        for f in files if f.endswith(extension)
+    ]
+
+def will_write(filename): # ensure path to file exists before writing
+    print(filename)
+    output_path = os.path.split(filename)[0]
+    if not os.path.exists(output_path): os.makedirs(output_path)
+
 if __name__ == '__main__':
-    # Generate Your Boilerplate single-source
-    gyb_files = glob.glob(os.path.join(single_source_dir, '*.swift.gyb'))
-    gyb_script = os.path.realpath(os.path.join(perf_dir, '../utils/gyb'))
+    # Generate Your Boilerplate
+    gyb_files = all_files(perf_dir, '.swift.gyb')
     for f in gyb_files:
-        print(f[:-4])
-        subprocess.call([gyb_script, '--line-directive', '', '-o', f[:-4], f])
+        relative_path = os.path.relpath(f[:-4], perf_dir)
+        out_file = os.path.join(output_dir, relative_path)
+        will_write(out_file)
+        subprocess.call([gyb_script, '--line-directive', '', '-o', out_file, f])
 
     # CMakeList single-source
-    test_files = glob.glob(os.path.join(single_source_dir, '*.swift'))
+    test_files = all_files(single_source_dir,'.swift')
     tests = sorted(os.path.basename(x).split('.')[0] for x in test_files)
 
     # CMakeList multi-source
     class MultiSourceBench(object):
-
         def __init__(self, path):
             self.name = os.path.basename(path)
             self.files = [x for x in os.listdir(path)
                           if x.endswith('.swift')]
+
     if os.path.isdir(multi_source_dir):
         multisource_benches = [
             MultiSourceBench(os.path.join(multi_source_dir, x))
@@ -75,26 +94,19 @@ def get_run_funcs(filepath):
         matches = re.findall(r'func run_(.*?)\(', content)
         return filter(lambda x: x not in ignored_run_funcs, matches)
 
-    def find_run_funcs(dirs):
-        ret_run_funcs = []
-        for d in dirs:
-            for root, _, files in os.walk(d):
-                for name in filter(lambda x: x.endswith('.swift'), files):
-                    run_funcs = get_run_funcs(os.path.join(root, name))
-                    ret_run_funcs.extend(run_funcs)
-        return ret_run_funcs
-    run_funcs = sorted(
-        [(x, x)
-         for x in find_run_funcs([single_source_dir, multi_source_dir])],
-        key=lambda x: x[0]
-    )
+    def find_run_funcs():
+        swift_files = all_files(perf_dir, '.swift')
+        return [func for f in swift_files for func in get_run_funcs(f)]
+
+    run_funcs = [(f, f) for f in sorted(find_run_funcs())]
 
     # Replace originals with files generated from templates
     for template_file in template_map:
         template_path = os.path.join(script_dir, template_file)
         template = template_env.get_template(template_path)
-        print(template_map[template_file])
-        open(template_map[template_file], 'w').write(
+        out_file = template_map[template_file]
+        will_write(out_file)
+        open(out_file, 'w').write(
             template.render(tests=tests,
                             multisource_benches=multisource_benches,
                             imports=imports,

benchmark/scripts/generate_harness/test_generate_harness.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+SWIFT_SRC_DIR="$1"
+BENCHMARK_DIR="${SWIFT_SRC_DIR}/benchmark"
+SCRIPT_DIR="${BENCHMARK_DIR}/scripts"
+TEMP_DIR="$2"
+
+"${SCRIPT_DIR}/generate_harness/generate_harness.py" "--output-dir=${TEMP_DIR}"
+for f in $(cd "${TEMP_DIR}" && find ./ -type f); do
+    diff "${TEMP_DIR}/${f}" "${BENCHMARK_DIR}/${f}"
+    if [[ $? -ne 0 ]]; then
+       exit 1
+    fi
+done

validation-test/benchmarks/generate-harness.test-sh

@@ -1,4 +1,5 @@
-This will just be a shell script along the lines of bug-reducer.test-sh that 
-will generate the harness/gyb files in a temp directory and make sure that the
-diff to what is checked into tree is empty. Then at least we will know if
-someone forgets to regenerate the harness or gyb files.
+// RUN: rm -rfv %t
+// RUN: %swift_src_root/benchmarks/scripts/test_generate_harness.sh %swift_src_root %t
+// REQUIRES: OS=macosx
+// REQUIRES: asserts
+// REQUIRES: CMAKE_GENERATOR=Ninja