Merge pull request #4994 from gottesmm/allow_for_buildscript_to_specify_benchmark_iterations

Allow for buildscript to specify benchmark iterations

Author: gottesmm

benchmark/CMakeLists.txt

@@ -139,7 +139,8 @@ runcmd(COMMAND "xcrun" "-toolchain" "${SWIFT_DARWIN_XCRUN_TOOLCHAIN}" "-f" "clan
 # You have to delete CMakeCache.txt in the swift build to force a
 # reconfiguration.
 set(SWIFT_EXTRA_BENCH_CONFIGS CACHE STRING
-    "A semicolon separated list of benchmark configurations. Available configurations: <Optlevel>_SINGLEFILE, <Optlevel>_MULTITHREADED")
+    "A semicolon separated list of benchmark configurations. \
+Available configurations: <Optlevel>_SINGLEFILE, <Optlevel>_MULTITHREADED")
 
 # Syntax for an optset:  <optimization-level>_<configuration>
 #    where "_<configuration>" is optional.
@@ -148,6 +149,11 @@ if(NOT SWIFT_OPTIMIZATION_LEVELS)
                                 ${SWIFT_EXTRA_BENCH_CONFIGS})
 endif()
 
+set(SWIFT_BENCHMARK_NUM_O_ITERATIONS "" CACHE STRING
+    "Number of iterations to perform when running -O benchmarks via cmake")
+set(SWIFT_BENCHMARK_NUM_ONONE_ITERATIONS "" CACHE STRING
+    "Number of iterations to perform when running -Onone benchmarks via cmake")
+
 # Options for the default (= empty) configuration
 set(BENCHOPTS "-whole-module-optimization")
 

benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake

@@ -382,11 +382,11 @@ function(swift_benchmark_compile)
           COMMAND "${swift-bin-dir}/Benchmark_Driver" "run"
                   "-o" "O" "--output-dir" "${CMAKE_CURRENT_BINARY_DIR}/logs"
                   "--swift-repo" "${SWIFT_SOURCE_DIR}"
-                  "--iterations" "3"
+                  "--iterations" "${SWIFT_BENCHMARK_NUM_O_ITERATIONS}"
           COMMAND "${swift-bin-dir}/Benchmark_Driver" "run"
                   "-o" "Onone" "--output-dir" "${CMAKE_CURRENT_BINARY_DIR}/logs"
                   "--swift-repo" "${SWIFT_SOURCE_DIR}"
-                  "--iterations" "3"
+                  "--iterations" "${SWIFT_BENCHMARK_NUM_ONONE_ITERATIONS}"
           COMMAND "${swift-bin-dir}/Benchmark_Driver" "compare"
                   "--log-dir" "${CMAKE_CURRENT_BINARY_DIR}/logs"
                   "--swift-repo" "${SWIFT_SOURCE_DIR}"

utils/build-script

@@ -1560,6 +1560,15 @@ details of the setups of other systems or automated environments.""")
         "-B", "--benchmark",
         help="run the Swift Benchmark Suite after building",
         action="store_true")
+    run_tests_group.add_argument(
+        "--benchmark-num-o-iterations",
+        help="if the Swift Benchmark Suite is run after building, run N \
+iterations with -O",
+        metavar='N', type=int, default=3)
+    run_tests_group.add_argument(
+        "--benchmark-num-onone-iterations",
+        help="if the Swift Benchmark Suite is run after building, run N \
+        iterations with -Onone", metavar='N', type=int, default=3)
     run_tests_group.add_argument(
         "--skip-test-osx",
         help="skip testing Swift stdlibs for Mac OS X",

utils/swift_build_support/swift_build_support/products/swift.py

@@ -27,6 +27,9 @@ def __init__(self, args, toolchain, source_dir, build_dir):
         # Add any swift version related cmake flags.
         self.cmake_options.extend(self._version_flags)
 
+        # Add benchmark specific flags.
+        self.cmake_options.extend(self._benchmark_flags)
+
     @property
     def _runtime_sanitizer_flags(self):
         sanitizer_list = []
@@ -77,3 +80,15 @@ def _version_flags(self):
                 "-DCLANG_COMPILER_VERSION={}".format(clang_compiler_version)
             )
         return r
+
+    @property
+    def _benchmark_flags(self):
+        if not self.args.benchmark:
+            return []
+
+        onone_iters = self.args.benchmark_num_onone_iterations
+        o_iters = self.args.benchmark_num_o_iterations
+        return [
+            "-DSWIFT_BENCHMARK_NUM_ONONE_ITERATIONS={}".format(onone_iters),
+            "-DSWIFT_BENCHMARK_NUM_O_ITERATIONS={}".format(o_iters)
+        ]

utils/swift_build_support/tests/products/test_swift.py

@@ -51,7 +51,10 @@ def setUp(self):
             swift_compiler_version=None,
             clang_compiler_version=None,
             swift_user_visible_version=None,
-            darwin_deployment_version_osx="10.9")
+            darwin_deployment_version_osx="10.9",
+            benchmark=False,
+            benchmark_num_onone_iterations=3,
+            benchmark_num_o_iterations=3)
 
         # Setup shell
         shell.dry_run = True
@@ -72,6 +75,14 @@ def tearDown(self):
         self.toolchain = None
         self.args = None
 
+    def test_by_default_no_cmake_options(self):
+        swift = Swift(
+            args=self.args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        self.assertEqual(swift.cmake_options, [])
+
     def test_swift_runtime_tsan(self):
         self.args.enable_tsan_runtime = True
         swift = Swift(
@@ -204,3 +215,53 @@ def test_version_flags(self):
         )
         self.args.swift_compiler_version = None
         self.args.clang_compiler_version = None
+
+    def test_benchmark_flags(self):
+        self.args.benchmark = True
+        swift = Swift(
+            args=self.args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        # By default, we should get an argument of 3 for both -Onone and -O
+        self.assertEqual(
+            ['-DSWIFT_BENCHMARK_NUM_ONONE_ITERATIONS=3',
+             '-DSWIFT_BENCHMARK_NUM_O_ITERATIONS=3'],
+            swift.cmake_options)
+
+        self.args.benchmark_num_onone_iterations = 20
+        swift = Swift(
+            args=self.args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        self.assertEqual(
+            ['-DSWIFT_BENCHMARK_NUM_ONONE_ITERATIONS=20',
+             '-DSWIFT_BENCHMARK_NUM_O_ITERATIONS=3'],
+            swift.cmake_options)
+        self.args.benchmark_num_onone_iterations = 3
+
+        self.args.benchmark_num_o_iterations = 30
+        swift = Swift(
+            args=self.args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        self.assertEqual(
+            ['-DSWIFT_BENCHMARK_NUM_ONONE_ITERATIONS=3',
+             '-DSWIFT_BENCHMARK_NUM_O_ITERATIONS=30'],
+            swift.cmake_options)
+        self.args.benchmark_num_onone_iterations = 3
+
+        self.args.benchmark_num_onone_iterations = 10
+        self.args.benchmark_num_o_iterations = 25
+        swift = Swift(
+            args=self.args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        self.assertEqual(
+            ['-DSWIFT_BENCHMARK_NUM_ONONE_ITERATIONS=10',
+             '-DSWIFT_BENCHMARK_NUM_O_ITERATIONS=25'],
+            swift.cmake_options)
+