[benchmark] Add support for building out of tree via build-script against the just built swift.

I recently broke the out of tree build by mistake [its fixed now ; )]. This
inspired me to make it easy to test this behavior by adding support to
build-script/cmake/etc for running an external benchmark build via
AddExternalProjects.

Now I can just call build-script with --build-external-benchmarks and thats it!
It should just work! It already helped me to avoid breaking the external build
twice!

I hope that eventually we get this on a bot to make sure it keeps working [or
even added to the smoke tests ; )].

*NOTE* This is disabled by default so it will not affect normal builds.

*NOTE* This just builds the external benchmarks. There is an rpath issue that
prevents you from running them (the benchmarks have the rpath set as if they are
next to the stdlib, but they are not. This can be fixed in a few different ways,
but I do not have time to finish implementing it = (. But this commit is a good
first step and at least detects build errors.

Author: gottesmm

CMakeLists.txt

@@ -66,7 +66,11 @@ else()
 endif()
 
 option(SWIFT_BUILD_PERF_TESTSUITE
-    "Create targets for swift performance benchmarks."
+    "Create in-tree targets for building swift performance benchmarks."
+    FALSE)
+
+option(SWIFT_BUILD_EXTERNAL_PERF_TESTSUITE
+    "Create out-of-tree targets for building swift performance benchmarks."
     FALSE)
 
 option(SWIFT_INCLUDE_TESTS "Create targets for building/running tests." TRUE)
@@ -882,9 +886,16 @@ if(SWIFT_BUILD_DYNAMIC_STDLIB AND SWIFT_INCLUDE_TESTS)
   add_subdirectory(tools/swift-reflection-test)
 endif()
 
-if(SWIFT_BUILD_PERF_TESTSUITE AND "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
-  add_subdirectory(benchmark)
+if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
+  if(SWIFT_BUILD_PERF_TESTSUITE)
+    add_subdirectory(benchmark)
+  endif()
+  if(SWIFT_BUILD_EXTERNAL_PERF_TESTSUITE)
+    include(SwiftExternalBenchmarkBuild)
+    add_external_benchmark_suite()
+  endif()
 endif()
+
 if(SWIFT_INCLUDE_TESTS)
   add_subdirectory(test)
   add_subdirectory(unittests)

benchmark/CMakeLists.txt

@@ -231,6 +231,7 @@ endforeach()
 
 message("--")
 message("-- Swift Benchmark Suite:")
+message("--   SWIFT_BENCHMARK_BUILT_STANDALONE = ${SWIFT_BENCHMARK_BUILT_STANDALONE}")
 message("--   SWIFT_EXEC = ${SWIFT_EXEC}")
 message("--   SWIFT_LIBRARY_PATH = ${SWIFT_LIBRARY_PATH}")
 message("--   CLANG_EXEC = ${CLANG_EXEC}")

benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake

@@ -395,7 +395,16 @@ function(swift_benchmark_compile)
       list(APPEND platform_executables ${new_output_exec})
     endforeach()
 
-    set(executable_target "swift-benchmark-${SWIFT_BENCHMARK_COMPILE_PLATFORM}-${arch}")
+    # If we are building standalone, we add the -external suffix to all of our
+    # cmake target names. This enables the main swift build to simple create
+    # -external targets and forward them via AddExternalProject to the
+    # standalone benchmark project. The reason why this is necessary is that we
+    # want to be able to support in-tree and out-of-tree benchmark builds at the
+    # same time implying that we need some sort of way to distinguish the
+    # in-tree (which don't have the suffix) from the out of tree target (which
+    # do).
+    translate_flag(SWIFT_BENCHMARK_BUILT_STANDALONE "-external" external)
+    set(executable_target "swift-benchmark-${SWIFT_BENCHMARK_COMPILE_PLATFORM}-${arch}${external}")
 
     add_custom_target("${executable_target}"
         DEPENDS ${platform_executables})

benchmark/cmake/modules/SwiftBenchmarkUtils.cmake

@@ -27,3 +27,19 @@ function(precondition var)
     endif()
   endif()
 endfunction()
+
+# Translate a yes/no variable to the presence of a given string in a
+# variable.
+#
+# Usage:
+#   translate_flag(is_set flag_name var_name)
+#
+# If is_set is true, sets ${var_name} to ${flag_name}. Otherwise,
+# unsets ${var_name}.
+function(translate_flag is_set flag_name var_name)
+  if(${is_set})
+    set("${var_name}" "${flag_name}" PARENT_SCOPE)
+  else()
+    set("${var_name}" "" PARENT_SCOPE)
+  endif()
+endfunction()

cmake/modules/SwiftExternalBenchmarkBuild.cmake

@@ -0,0 +1,89 @@
+
+include(CMakeParseArguments)
+include(LLVMExternalProjectUtils)
+include(SwiftUtils)
+
+# This is the name of the target on the parent cmake side that is associated
+# with an external project target.
+#
+# If LLVMExternalProjectUtils refactors its external target code, so we can
+# easily add individual forwarded targets with different dependencies, this can
+# be removed.
+function(compute_external_target_name target_name_out target)
+  string(REPLACE ":" ";" target_list ${target})
+  list(GET target_list 0 target)
+  list(LENGTH target_list target_list_len)
+  if(${target_list_len} GREATER 1)
+    list(GET target_list 1 target_name)
+  else()
+    set(target_name "${target}")
+  endif()
+
+  set(${target_name_out} "${target_name}" PARENT_SCOPE)
+endfunction()
+
+function(compute_stdlib_dependencies stdlib_dependencies_out platform)
+  foreach(stdlib_dependency ${UNIVERSAL_LIBRARY_NAMES_${platform}})
+    string(FIND "${stdlib_dependency}" "Unittest" find_output)
+    if("${find_output}" STREQUAL "-1")
+      list(APPEND stdlib_dependencies "${stdlib_dependency}")
+    endif()
+  endforeach()
+  set(${stdlib_dependencies_out} "${stdlib_dependencies}" PARENT_SCOPE)
+endfunction()
+
+set(ONLY_PLATFORMS "macosx" "iphoneos" "appletvos" "watchos")
+
+function (get_platform_from_target target_platform_out target)
+  foreach (platform ${ONLY_PLATFORMS})
+    string(FIND "${target}" "${platform}" FOUND_TARGET_PLATFORM)
+    if (NOT FOUND_TARGET_PLATFORM)
+      continue()
+    endif()
+
+    set(${target_platform_out} "${platform}" PARENT_SCOPE)
+    break()
+  endforeach()
+endfunction()
+
+function (compute_target_stdlib_dependencies dependencies_out target)
+  get_platform_from_target(target_platform ${target})
+  precondition(target_platform
+    MESSAGE "Failed to find a platform for ${target_platform}")
+  compute_stdlib_dependencies(stdlib_dependencies ${target_platform})
+  set(${dependencies_out} ${stdlib_dependencies} PARENT_SCOPE)
+endfunction()
+
+function (add_external_benchmark_suite)
+  set(name swift-benchmark)
+  set(src_dir ${SWIFT_SOURCE_DIR}/benchmark)
+  set(bin_dir ${SWIFT_BINARY_DIR}/external-benchmark/binary)
+  set(stamp_dir ${SWIFT_BINARY_DIR}/external-benchmark/stamps)
+  set(prefix_dir ${SWIFT_BINARY_DIR}/external-benchmark/prefix)
+
+  set(bench_targets Benchmark_O Benchmark_Onone Benchmark_Ounchecked)
+  set(library_targets swift-benchmark-macosx-x86_64-external)
+
+  set(all_stdlib_dependencies)
+  foreach (target ${library_targets})
+    compute_target_stdlib_dependencies(stdlib_dependencies ${target})
+    precondition(stdlib_dependencies)
+    # Add dependencies from all of our stdlib dependencies to
+    # swift-bench-configure. This will ensure the stdlib is ready to be poked at
+    # in the configure script if we ever want to do so.
+    list(APPEND all_stdlib_dependencies ${stdlib_dependencies})
+  endforeach()
+
+  llvm_ExternalProject_add(swift-bench ${src_dir}
+    SOURCE_DIR ${src_dir}
+    EXCLUDE_FROM_ALL
+    DEPENDS swift ${all_stdlib_dependencies}
+    EXTRA_TARGETS ${bench_targets} ${library_targets}
+    CMAKE_ARGS
+      -DSWIFT_EXEC=${SWIFT_BINARY_DIR}/bin/swiftc
+      -DSWIFT_LIBRARY_PATH=${SWIFT_BINARY_DIR}/lib/swift
+      -DCMAKE_C_COMPILER=${PATH_TO_CLANG_BUILD}/bin/clang
+      -DCMAKE_CXX_COMPILER=${PATH_TO_CLANG_BUILD}/bin/clang++
+    PASSTHROUGH_PREFIXES SWIFT_BENCHMARK
+    )
+endfunction()

utils/build-script

@@ -127,7 +127,11 @@ class HostSpecificConfiguration(object):
             test = (
                 deployment_platform not in invocation.platforms_to_skip_test)
             test_host_only = None
-            build_benchmark = build and deployment_target.supports_benchmark
+            dt_supports_benchmark = deployment_target.supports_benchmark
+            build_benchmarks = build and dt_supports_benchmark
+            build_external_benchmarks = all([build, dt_supports_benchmark,
+                                             args.build_external_benchmarks])
+
             # FIXME: Note, `build-script-impl` computed a property here
             # w.r.t. testing, but it was actually unused.
 
@@ -161,13 +165,20 @@ class HostSpecificConfiguration(object):
                 else:
                     self.swift_stdlib_build_targets.append(
                         "swift-test-stdlib-" + name)
-            if build_benchmark:
+            if build_benchmarks:
                 self.swift_benchmark_build_targets.append(
                     "swift-benchmark-" + name)
                 # FIXME: This probably should respect `args.benchmark`, but
                 # a typo in build-script-impl meant we always would do this.
                 self.swift_benchmark_run_targets.append(
                     "check-swift-benchmark-" + name)
+
+            if build_external_benchmarks:
+                # Add support for the external benchmarks.
+                self.swift_benchmark_build_targets.append(
+                    "swift-benchmark-{}-external".format(name))
+                self.swift_benchmark_run_targets.append(
+                    "check-swift-benchmark-{}-external".format(name))
             if test:
                 if test_host_only:
                     suffix = "-non-executable"
@@ -506,6 +517,9 @@ class BuildScriptInvocation(object):
                           "--skip-build-swift"]
         if not args.build_benchmarks:
             impl_args += ["--skip-build-benchmarks"]
+        # Currently we do not build external benchmarks by default.
+        if args.build_external_benchmarks:
+            impl_args += ["--skip-build-external-benchmarks=0"]
         if not args.build_foundation:
             impl_args += ["--skip-build-foundation"]
         if not args.build_xctest:

utils/build-script-impl

@@ -126,6 +126,7 @@ KNOWN_SETTINGS=(
     skip-build-libdispatch      ""               "set to skip building libdispatch"
     skip-build-libicu           ""               "set to skip building libicu"
     skip-build-benchmarks       ""               "set to skip building Swift Benchmark Suite"
+    skip-build-external-benchmarks "1"            "set to skip building the external Swift Benchmark Suite. (skipped by default)"
     skip-build-playgroundlogger ""               "set to skip building PlaygroundLogger"
     skip-build-playgroundsupport ""              "set to skip building PlaygroundSupport"
     skip-test-cmark             ""               "set to skip testing CommonMark"
@@ -1324,6 +1325,7 @@ function calculate_targets_for_host() {
         local test_this_target=1
         local test_host_only=
         local build_benchmark_this_target=
+        local build_external_benchmark_this_target=
         local test_benchmark_this_target=
 
         case ${stdlib_deployment_target} in
@@ -1352,6 +1354,7 @@ function calculate_targets_for_host() {
                 build_for_this_target=$(not ${SKIP_BUILD_OSX})
                 test_this_target=$(not ${SKIP_TEST_OSX})
                 build_benchmark_this_target=$(not ${SKIP_BUILD_OSX})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_OSX})
                 test_benchmark_this_target=$(not ${SKIP_BUILD_OSX})
                 ;;
             iphoneos-*)
@@ -1363,10 +1366,12 @@ function calculate_targets_for_host() {
                     test_this_target=
                 fi
                 build_benchmark_this_target=$(not ${SKIP_BUILD_IOS_DEVICE})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_IOS_DEVICE})
 
                 # Never build iOS armv7s benchmarks.
                 if [[ "${stdlib_deployment_target}" == "iphoneos-armv7s" ]]; then
                     build_benchmark_this_target=
+                    build_external_benchmark_this_target=
                 fi
                 ;;
             iphonesimulator-x86_64)
@@ -1391,6 +1396,7 @@ function calculate_targets_for_host() {
                     test_this_target=
                 fi
                 build_benchmark_this_target=$(not ${SKIP_BUILD_TVOS_DEVICE})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_TVOS_DEVICE})
                 ;;
             appletvsimulator-*)
                 swift_sdk="TVOS_SIMULATOR"
@@ -1406,6 +1412,7 @@ function calculate_targets_for_host() {
                     test_this_target=
                 fi
                 build_benchmark_this_target=$(not ${SKIP_BUILD_WATCHOS_DEVICE})
+                build_external_benchmark_this_target=$(not ${SKIP_BUILD_WATCHOS_DEVICE})
                 ;;
             watchsimulator-*)
                 swift_sdk="WATCHOS_SIMULATOR"
@@ -1443,6 +1450,16 @@ function calculate_targets_for_host() {
               SWIFT_RUN_BENCHMARK_TARGETS+=("check-swift-benchmark-${stdlib_deployment_target}")
             fi
         fi
+
+        if [[ "$(true_false ${SKIP_BUILD_EXTERNAL_BENCHMARKS})" == "FALSE"  ]] &&
+           [[ "${build_external_benchmark_this_target}" ]] &&
+           [[ "${is_in_build_list}" ]] ; then
+            SWIFT_BENCHMARK_TARGETS+=("swift-benchmark-${stdlib_deployment_target}-external")
+            if [[ $(not ${SKIP_TEST_BENCHMARK}) ]] ; then
+                SWIFT_RUN_BENCHMARK_TARGETS+=("check-swift-benchmark-${stdlib_deployment_target}-external")
+            fi
+        fi
+
         if [[ "${test_this_target}" ]] && [[ "${is_in_build_list}" ]]; then
             test_target_suffix=""
             if [[ -n "${test_host_only}" ]] ; then
@@ -2098,13 +2115,16 @@ for host in "${ALL_HOSTS[@]}"; do
 
                     # Don't build benchmarks and tests when building cross compiler.
                     build_perf_testsuite_this_time=false
+                    build_external_perf_testsuite_this_time=false
                     build_tests_this_time=false
 
                     native_llvm_tools_path="$(build_directory "${LOCAL_HOST}" llvm)/bin"
                     native_clang_tools_path="$(build_directory "${LOCAL_HOST}" llvm)/bin"
                     native_swift_tools_path="$(build_directory "${LOCAL_HOST}" swift)/bin"
                 else
+                    # FIXME: Why is the next line not using false_true?
                     build_perf_testsuite_this_time=$(true_false "$(not ${SKIP_BUILD_BENCHMARKS})")
+                    build_external_perf_testsuite_this_time=$(false_true "${SKIP_BUILD_EXTERNAL_BENCHMARKS}")
                     build_tests_this_time=${SWIFT_INCLUDE_TESTS}
                 fi
 
@@ -2162,6 +2182,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY:BOOL=$(true_false "${BUILD_SWIFT_DYNAMIC_SDK_OVERLAY}")
                     -DSWIFT_BUILD_STATIC_SDK_OVERLAY:BOOL=$(true_false "${BUILD_SWIFT_STATIC_SDK_OVERLAY}")
                     -DSWIFT_BUILD_PERF_TESTSUITE:BOOL=$(true_false "${build_perf_testsuite_this_time}")
+                    -DSWIFT_BUILD_EXTERNAL_PERF_TESTSUITE:BOOL=$(true_false "${build_external_perf_testsuite_this_time}")
                     -DSWIFT_BUILD_EXAMPLES:BOOL=$(true_false "${BUILD_SWIFT_EXAMPLES}")
                     -DSWIFT_INCLUDE_TESTS:BOOL=$(true_false "${build_tests_this_time}")
                     -DSWIFT_INSTALL_COMPONENTS:STRING="${SWIFT_INSTALL_COMPONENTS}"

utils/build_swift/driver_arguments.py

@@ -153,6 +153,7 @@ def _apply_default_arguments(args):
         args.build_watchos = False
         args.build_android = False
         args.build_benchmarks = False
+        args.build_external_benchmarks = False
         args.build_lldb = False
         args.build_llbuild = False
         args.build_swiftpm = False
@@ -771,6 +772,12 @@ def create_argument_parser():
         action=arguments.action.optional_false,
         help="skip building Swift Benchmark Suite")
 
+    run_build_group.add_argument(
+        "--build-external-benchmarks",
+        dest='build_external_benchmarks',
+        action=arguments.action.optional_true,
+        help="skip building Swift Benchmark Suite")
+
     skip_test_group = parser.add_argument_group(
         title="Skip testing specified targets")
     skip_test_group.add_argument(