Merge pull request #12102 from gottesmm/external_benchmarks

Author: swift-ci

CMakeLists.txt

@@ -66,7 +66,11 @@ else()
 endif()
 
 option(SWIFT_BUILD_PERF_TESTSUITE
-    "Create targets for swift performance benchmarks."
+    "Create in-tree targets for building swift performance benchmarks."
+    FALSE)
+
+option(SWIFT_BUILD_EXTERNAL_PERF_TESTSUITE
+    "Create out-of-tree targets for building swift performance benchmarks."
     FALSE)
 
 option(SWIFT_INCLUDE_TESTS "Create targets for building/running tests." TRUE)
@@ -882,9 +886,16 @@ if(SWIFT_BUILD_DYNAMIC_STDLIB AND SWIFT_INCLUDE_TESTS)
   add_subdirectory(tools/swift-reflection-test)
 endif()
 
-if(SWIFT_BUILD_PERF_TESTSUITE AND "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
-  add_subdirectory(benchmark)
+if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
+  if(SWIFT_BUILD_PERF_TESTSUITE)
+    add_subdirectory(benchmark)
+  endif()
+  if(SWIFT_BUILD_EXTERNAL_PERF_TESTSUITE)
+    include(SwiftExternalBenchmarkBuild)
+    add_external_benchmark_suite()
+  endif()
 endif()
+
 if(SWIFT_INCLUDE_TESTS)
   add_subdirectory(test)
   add_subdirectory(unittests)

benchmark/CMakeLists.txt

@@ -231,6 +231,7 @@ endforeach()
 
 message("--")
 message("-- Swift Benchmark Suite:")
+message("--   SWIFT_BENCHMARK_BUILT_STANDALONE = ${SWIFT_BENCHMARK_BUILT_STANDALONE}")
 message("--   SWIFT_EXEC = ${SWIFT_EXEC}")
 message("--   SWIFT_LIBRARY_PATH = ${SWIFT_LIBRARY_PATH}")
 message("--   CLANG_EXEC = ${CLANG_EXEC}")

benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake

@@ -395,7 +395,16 @@ function(swift_benchmark_compile)
       list(APPEND platform_executables ${new_output_exec})
     endforeach()
 
-    set(executable_target "swift-benchmark-${SWIFT_BENCHMARK_COMPILE_PLATFORM}-${arch}")
+    # If we are building standalone, we add the -external suffix to all of our
+    # cmake target names. This enables the main swift build to simple create
+    # -external targets and forward them via AddExternalProject to the
+    # standalone benchmark project. The reason why this is necessary is that we
+    # want to be able to support in-tree and out-of-tree benchmark builds at the
+    # same time implying that we need some sort of way to distinguish the
+    # in-tree (which don't have the suffix) from the out of tree target (which
+    # do).
+    translate_flag(SWIFT_BENCHMARK_BUILT_STANDALONE "-external" external)
+    set(executable_target "swift-benchmark-${SWIFT_BENCHMARK_COMPILE_PLATFORM}-${arch}${external}")
 
     add_custom_target("${executable_target}"
         DEPENDS ${platform_executables})

benchmark/cmake/modules/SwiftBenchmarkUtils.cmake

@@ -27,3 +27,19 @@ function(precondition var)
     endif()
   endif()
 endfunction()
+
+# Translate a yes/no variable to the presence of a given string in a
+# variable.
+#
+# Usage:
+#   translate_flag(is_set flag_name var_name)
+#
+# If is_set is true, sets ${var_name} to ${flag_name}. Otherwise,
+# unsets ${var_name}.
+function(translate_flag is_set flag_name var_name)
+  if(${is_set})
+    set("${var_name}" "${flag_name}" PARENT_SCOPE)
+  else()
+    set("${var_name}" "" PARENT_SCOPE)
+  endif()
+endfunction()

cmake/modules/SwiftExternalBenchmarkBuild.cmake

@@ -0,0 +1,89 @@
+
+include(CMakeParseArguments)
+include(LLVMExternalProjectUtils)
+include(SwiftUtils)
+
+# This is the name of the target on the parent cmake side that is associated
+# with an external project target.
+#
+# If LLVMExternalProjectUtils refactors its external target code, so we can
+# easily add individual forwarded targets with different dependencies, this can
+# be removed.
+function(compute_external_target_name target_name_out target)
+  string(REPLACE ":" ";" target_list ${target})
+  list(GET target_list 0 target)
+  list(LENGTH target_list target_list_len)
+  if(${target_list_len} GREATER 1)
+    list(GET target_list 1 target_name)
+  else()
+    set(target_name "${target}")
+  endif()
+
+  set(${target_name_out} "${target_name}" PARENT_SCOPE)
+endfunction()
+
+function(compute_stdlib_dependencies stdlib_dependencies_out platform)
+  foreach(stdlib_dependency ${UNIVERSAL_LIBRARY_NAMES_${platform}})
+    string(FIND "${stdlib_dependency}" "Unittest" find_output)
+    if("${find_output}" STREQUAL "-1")
+      list(APPEND stdlib_dependencies "${stdlib_dependency}")
+    endif()
+  endforeach()
+  set(${stdlib_dependencies_out} "${stdlib_dependencies}" PARENT_SCOPE)
+endfunction()
+
+set(ONLY_PLATFORMS "macosx" "iphoneos" "appletvos" "watchos")
+
+function (get_platform_from_target target_platform_out target)
+  foreach (platform ${ONLY_PLATFORMS})
+    string(FIND "${target}" "${platform}" FOUND_TARGET_PLATFORM)
+    if (NOT FOUND_TARGET_PLATFORM)
+      continue()
+    endif()
+
+    set(${target_platform_out} "${platform}" PARENT_SCOPE)
+    break()
+  endforeach()
+endfunction()
+
+function (compute_target_stdlib_dependencies dependencies_out target)
+  get_platform_from_target(target_platform ${target})
+  precondition(target_platform
+    MESSAGE "Failed to find a platform for ${target_platform}")
+  compute_stdlib_dependencies(stdlib_dependencies ${target_platform})
+  set(${dependencies_out} ${stdlib_dependencies} PARENT_SCOPE)
+endfunction()
+
+function (add_external_benchmark_suite)
+  set(name swift-benchmark)
+  set(src_dir ${SWIFT_SOURCE_DIR}/benchmark)
+  set(bin_dir ${SWIFT_BINARY_DIR}/external-benchmark/binary)
+  set(stamp_dir ${SWIFT_BINARY_DIR}/external-benchmark/stamps)
+  set(prefix_dir ${SWIFT_BINARY_DIR}/external-benchmark/prefix)
+
+  set(bench_targets Benchmark_O Benchmark_Onone Benchmark_Ounchecked)
+  set(library_targets swift-benchmark-macosx-x86_64-external)
+
+  set(all_stdlib_dependencies)
+  foreach (target ${library_targets})
+    compute_target_stdlib_dependencies(stdlib_dependencies ${target})
+    precondition(stdlib_dependencies)
+    # Add dependencies from all of our stdlib dependencies to
+    # swift-bench-configure. This will ensure the stdlib is ready to be poked at
+    # in the configure script if we ever want to do so.
+    list(APPEND all_stdlib_dependencies ${stdlib_dependencies})
+  endforeach()
+
+  llvm_ExternalProject_add(swift-bench ${src_dir}
+    SOURCE_DIR ${src_dir}
+    EXCLUDE_FROM_ALL
+    DEPENDS swift ${all_stdlib_dependencies}
+    EXTRA_TARGETS ${bench_targets} ${library_targets}
+    CMAKE_ARGS
+      -DSWIFT_EXEC=${SWIFT_BINARY_DIR}/bin/swiftc
+      -DSWIFT_LIBRARY_PATH=${SWIFT_BINARY_DIR}/lib/swift
+      -DCMAKE_C_COMPILER=${PATH_TO_CLANG_BUILD}/bin/clang
+      -DCMAKE_CXX_COMPILER=${PATH_TO_CLANG_BUILD}/bin/clang++
+    PASSTHROUGH_PREFIXES SWIFT_BENCHMARK
+    )
+endfunction()

utils/build-script

@@ -127,7 +127,11 @@ class HostSpecificConfiguration(object):
             test = (
                 deployment_platform not in invocation.platforms_to_skip_test)
             test_host_only = None
-            build_benchmark = build and deployment_target.supports_benchmark
+            dt_supports_benchmark = deployment_target.supports_benchmark
+            build_benchmarks = build and dt_supports_benchmark
+            build_external_benchmarks = all([build, dt_supports_benchmark,
+                                             args.build_external_benchmarks])
+
             # FIXME: Note, `build-script-impl` computed a property here
             # w.r.t. testing, but it was actually unused.
 
@@ -161,13 +165,20 @@ class HostSpecificConfiguration(object):
                 else:
                     self.swift_stdlib_build_targets.append(
                         "swift-test-stdlib-" + name)
-            if build_benchmark:
+            if build_benchmarks:
                 self.swift_benchmark_build_targets.append(
                     "swift-benchmark-" + name)
                 # FIXME: This probably should respect `args.benchmark`, but
                 # a typo in build-script-impl meant we always would do this.
                 self.swift_benchmark_run_targets.append(
                     "check-swift-benchmark-" + name)
+
+            if build_external_benchmarks:
+                # Add support for the external benchmarks.
+                self.swift_benchmark_build_targets.append(
+                    "swift-benchmark-{}-external".format(name))
+                self.swift_benchmark_run_targets.append(
+                    "check-swift-benchmark-{}-external".format(name))
             if test:
                 if test_host_only:
                     suffix = "-non-executable"
@@ -506,6 +517,9 @@ class BuildScriptInvocation(object):
                           "--skip-build-swift"]
         if not args.build_benchmarks:
             impl_args += ["--skip-build-benchmarks"]
+        # Currently we do not build external benchmarks by default.
+        if args.build_external_benchmarks:
+            impl_args += ["--skip-build-external-benchmarks=0"]
         if not args.build_foundation:
             impl_args += ["--skip-build-foundation"]
         if not args.build_xctest:

utils/build-script-impl

@@ -126,6 +126,7 @@ KNOWN_SETTINGS=(
     skip-build-libdispatch      ""               "set to skip building libdispatch"
     skip-build-libicu           ""               "set to skip building libicu"
     skip-build-benchmarks       ""               "set to skip building Swift Benchmark Suite"
+    skip-build-external-benchmarks "1"            "set to skip building the external Swift Benchmark Suite. (skipped by default)"
     skip-build-playgroundlogger ""               "set to skip building PlaygroundLogger"
     skip-build-playgroundsupport ""              "set to skip building PlaygroundSupport"
     skip-test-cmark             ""               "set to skip testing CommonMark"
@@ -1324,6 +1325,7 @@ function calculate_targets_for_host() {
         local test_this_target=1
         local test_host_only=
         local build_benchmark_this_target=
+        local build_external_benchmark_this_target=
         local test_benchmark_this_target=
 
         case ${stdlib_deployment_target} in
@@ -1352,6 +1354,7 @@ function calculate_targets_for_host() {
                 build_for_this_target=$(not ${SKIP_BUILD_OSX})
                 test_this_target=$(not ${SKIP_TEST_OSX})
                 build_benchmark_this_target=$(not ${SKIP_BUILD_OSX})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_OSX})
                 test_benchmark_this_target=$(not ${SKIP_BUILD_OSX})
                 ;;
             iphoneos-*)
@@ -1363,10 +1366,12 @@ function calculate_targets_for_host() {
                     test_this_target=
                 fi
                 build_benchmark_this_target=$(not ${SKIP_BUILD_IOS_DEVICE})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_IOS_DEVICE})
 
                 # Never build iOS armv7s benchmarks.
                 if [[ "${stdlib_deployment_target}" == "iphoneos-armv7s" ]]; then
                     build_benchmark_this_target=
+                    build_external_benchmark_this_target=
                 fi
                 ;;
             iphonesimulator-x86_64)
@@ -1391,6 +1396,7 @@ function calculate_targets_for_host() {
                     test_this_target=
                 fi
                 build_benchmark_this_target=$(not ${SKIP_BUILD_TVOS_DEVICE})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_TVOS_DEVICE})
                 ;;
             appletvsimulator-*)
                 swift_sdk="TVOS_SIMULATOR"
@@ -1406,6 +1412,7 @@ function calculate_targets_for_host() {
                     test_this_target=
                 fi
                 build_benchmark_this_target=$(not ${SKIP_BUILD_WATCHOS_DEVICE})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_WATCHOS_DEVICE})
                 ;;
             watchsimulator-*)
                 swift_sdk="WATCHOS_SIMULATOR"
@@ -1443,6 +1450,16 @@ function calculate_targets_for_host() {
               SWIFT_RUN_BENCHMARK_TARGETS+=("check-swift-benchmark-${stdlib_deployment_target}")
             fi
         fi
+
+        if [[ "$(true_false ${SKIP_BUILD_EXTERNAL_BENCHMARKS})" == "FALSE"  ]] &&
+           [[ "${build_external_benchmark_this_target}" ]] &&
+           [[ "${is_in_build_list}" ]] ; then
+            SWIFT_BENCHMARK_TARGETS+=("swift-benchmark-${stdlib_deployment_target}-external")
+            if [[ $(not ${SKIP_TEST_BENCHMARK}) ]] ; then
+                SWIFT_RUN_BENCHMARK_TARGETS+=("check-swift-benchmark-${stdlib_deployment_target}-external")
+            fi
+        fi
+
         if [[ "${test_this_target}" ]] && [[ "${is_in_build_list}" ]]; then
             test_target_suffix=""
             if [[ -n "${test_host_only}" ]] ; then
@@ -2098,13 +2115,16 @@ for host in "${ALL_HOSTS[@]}"; do
 
                     # Don't build benchmarks and tests when building cross compiler.
                     build_perf_testsuite_this_time=false
+                    build_external_perf_testsuite_this_time=false
                     build_tests_this_time=false
 
                     native_llvm_tools_path="$(build_directory "${LOCAL_HOST}" llvm)/bin"
                     native_clang_tools_path="$(build_directory "${LOCAL_HOST}" llvm)/bin"
                     native_swift_tools_path="$(build_directory "${LOCAL_HOST}" swift)/bin"
                 else
+                    # FIXME: Why is the next line not using false_true?
                     build_perf_testsuite_this_time=$(true_false "$(not ${SKIP_BUILD_BENCHMARKS})")
+                    build_external_perf_testsuite_this_time=$(false_true "${SKIP_BUILD_EXTERNAL_BENCHMARKS}")
                     build_tests_this_time=${SWIFT_INCLUDE_TESTS}
                 fi
 
@@ -2162,6 +2182,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY:BOOL=$(true_false "${BUILD_SWIFT_DYNAMIC_SDK_OVERLAY}")
                     -DSWIFT_BUILD_STATIC_SDK_OVERLAY:BOOL=$(true_false "${BUILD_SWIFT_STATIC_SDK_OVERLAY}")
                     -DSWIFT_BUILD_PERF_TESTSUITE:BOOL=$(true_false "${build_perf_testsuite_this_time}")
+                    -DSWIFT_BUILD_EXTERNAL_PERF_TESTSUITE:BOOL=$(true_false "${build_external_perf_testsuite_this_time}")
                     -DSWIFT_BUILD_EXAMPLES:BOOL=$(true_false "${BUILD_SWIFT_EXAMPLES}")
                     -DSWIFT_INCLUDE_TESTS:BOOL=$(true_false "${build_tests_this_time}")
                     -DSWIFT_INSTALL_COMPONENTS:STRING="${SWIFT_INSTALL_COMPONENTS}"

utils/build_swift/driver_arguments.py

@@ -153,6 +153,7 @@ def _apply_default_arguments(args):
         args.build_watchos = False
         args.build_android = False
         args.build_benchmarks = False
+        args.build_external_benchmarks = False
         args.build_lldb = False
         args.build_llbuild = False
         args.build_swiftpm = False
@@ -771,6 +772,12 @@ def create_argument_parser():
         action=arguments.action.optional_false,
         help="skip building Swift Benchmark Suite")
 
+    run_build_group.add_argument(
+        "--build-external-benchmarks",
+        dest='build_external_benchmarks',
+        action=arguments.action.optional_true,
+        help="skip building Swift Benchmark Suite")
+
     skip_test_group = parser.add_argument_group(
         title="Skip testing specified targets")
     skip_test_group.add_argument(