Merge pull request swiftlang#23814 from atrick/benchmark-cmake

Benchmark cmake: Allow building from a Swift root.

Author: atrick

benchmark/CMakeLists.txt

@@ -253,6 +253,10 @@ set(BENCHOPTS_MULTITHREADED
     "-whole-module-optimization" "-num-threads" "4")
 set(BENCHOPTS_SINGLEFILE "")
 
+option(SWIFT_BENCHMARK_USE_OS_LIBRARIES
+  "Runtime link against the Swift libraries on the target (/usr/lib/swift)."
+  FALSE)
+
 configure_build()
 
 #===-----------------------------------------------------------------------===#
@@ -268,10 +272,18 @@ configure_sdks()
 message("--")
 message("-- Swift Benchmark Suite:")
 message("--   SWIFT_BENCHMARK_BUILT_STANDALONE = ${SWIFT_BENCHMARK_BUILT_STANDALONE}")
-message("--   SWIFT_EXEC = ${SWIFT_EXEC}")
-message("--   SWIFT_BENCHMARK_EXTRA_FLAGS = ${SWIFT_BENCHMARK_EXTRA_FLAGS}")
+message("--   SWIFT_BENCHMARK_USE_OS_LIBRARIES = ${SWIFT_BENCHMARK_USE_OS_LIBRARIES}")
+message("--   SWIFT_EXEC         = ${SWIFT_EXEC}")
 message("--   SWIFT_LIBRARY_PATH = ${SWIFT_LIBRARY_PATH}")
-message("--   CLANG_EXEC = ${CLANG_EXEC}")
+if (SWIFT_RPATH_BASE)
+message("--   SWIFT_RPATH_BASE   = ${SWIFT_RPATH_BASE}")
+endif()
+if (SWIFT_RPATH)
+message("--   SWIFT_RPATH        = ${SWIFT_RPATH}")
+message("---  ** WARNING ** Benchmarking against Swift-in-the-OS")
+endif()
+message("--   CLANG_EXEC         = ${CLANG_EXEC}")
+message("--   SWIFT_BENCHMARK_EXTRA_FLAGS = ${SWIFT_BENCHMARK_EXTRA_FLAGS}")
 message("--   SWIFT_OPTIMIZATION_LEVELS = ${SWIFT_OPTIMIZATION_LEVELS}")
 message("--   ONLY_PLATFORMS = ${ONLY_PLATFORMS}")
 message("--   PAGE_ALIGNMENT_OPTION = ${PAGE_ALIGNMENT_OPTION}")

benchmark/README.md

@@ -28,6 +28,11 @@ flags:
 OS X benchmark driver binaries are placed in `bin` alongside `swiftc`.
 Additional platform binaries are placed in the `benchmark/bin` build directory.
 
+The required Swift standard library dylibs are placed in `lib`. The
+drivers dynamically link Swift standard library dylibs from a path
+relative to their run-time location (../lib/swift) so the standard
+library should be distributed alongside them.
+
 Building Independently
 ----------------------
 
@@ -43,15 +48,19 @@ The following build options are available:
 * `-DSWIFT_LIBRARY_PATH`
     * An absolute path to the Swift standard library to use during compilation
       (default: `swiftc_directory`/../lib/swift)
+* `-DSWIFT_DARWIN_XCRUN_TOOLCHAIN`
+    * The Xcode toolchain to use when invoking `xcrun` to find `clang`.
+      (default: XcodeDefault)
 * `-DONLY_PLATFORMS`
     * A list of platforms to build the benchmarks for
       (default: "macosx;iphoneos;appletvos;watchos")
 * `-DSWIFT_OPTIMIZATION_LEVELS`
     * A list of Swift optimization levels to build against
       (default: "O;Onone;Osize")
-* `-DSWIFT_BENCHMARK_EMIT_SIB`
-    * A boolean value indicating whether .sib files should be generated
-      alongside .o files (default: FALSE)
+* `-DSWIFT_BENCHMARK_USE_OS_LIBRARIES`
+    * Enable this option to link the benchmark binaries against the target
+      machine's Swift standard library and runtime installed with the OS.
+    (default: OFF)
 
 The following build targets are available:
 
@@ -66,13 +75,40 @@ Build steps (with example options):
 1. `$ cd benchmark`
 2. `$ mkdir build`
 3. `$ cd build`
-4. `$ cmake ..`
-5. `$ make -j8 swift-benchmark-macosx-x86_64`
+4. `$ cmake ../benchmark -G Ninja -DSWIFT_EXEC=[path to built swiftc]`
+5. `$ ninja swift-benchmark-macosx-x86_64`
+
+Benchmark binaries are placed in `bin`.
+
+The binaries dynamically link Swift standard library dylibs from a
+path determined by the configuration. If `SWIFT_LIBRARY_PATH` is set,
+they link against the absolute path provided, regardless of where the
+binaries are installed. Otherwise, the runtime library path is
+relative to the benchmark binary at the time it was executed
+(`@executable_path/../lib/swift/<platform>`).
+
+For example, to benchmark against a locally built `swiftc`, including
+any standard library changes in that build, you might configure using:
+
+    cmake ../benchmark -G Ninja -DSWIFT_EXEC=<src>/swift/build/swift-macosx-x86_64/bin/swiftc
+    ninja swift-benchmark-iphoneos-arm64
+
+To build against the installed Xcode, simply omit SWIFT_EXEC:
+
+    cmake ../benchmark -G Ninja
+    ninja swift-benchmark-iphoneos-arm64
+
+In both examples above, to run the benchmarks on a device, the dynamic
+libraries must then be copied onto the device into the library path
+relative to `swiftc`. To benchmark against the target machine's
+installed libraries instead, enable
+`SWIFT_BENCHMARK_USE_OS_LIBRARIES`.
+
+    cmake ../benchmark -G Ninja -DSWIFT_BENCHMARK_USE_OS_LIBRARIES=ON
+    ninja swift-benchmark-iphoneos-arm64
 
-Benchmark driver binaries are placed in `build/bin` and the required Swift
-standard library dylibs are placed in `build/lib`. The drivers dynamically link
-Swift standard library dylibs from a path relative to their location
-(../lib/swift) so the standard library should be distributed alongside them.
+This will reflect the performance of the Swift standard library
+installed on the device, not the one included in the Swift root.
 
 Using the Benchmark Driver
 --------------------------

benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake

@@ -24,6 +24,9 @@ macro(configure_build)
     if(${c_compiler} STREQUAL "clang")
       set(CLANG_EXEC ${CMAKE_C_COMPILER})
     else()
+      if(NOT SWIFT_DARWIN_XCRUN_TOOLCHAIN)
+        set(SWIFT_DARWIN_XCRUN_TOOLCHAIN "XcodeDefault")
+      endif()
       runcmd(COMMAND "xcrun" "-toolchain" "${SWIFT_DARWIN_XCRUN_TOOLCHAIN}" "-f" "clang"
              VARIABLE CLANG_EXEC
              ERROR "Unable to find Clang driver")
@@ -57,12 +60,47 @@ macro(configure_build)
     endif()
   endif()
 
-  # We always infer the SWIFT_LIBRARY_PATH from SWIFT_EXEC unless
-  # SWIFT_LIBRARY_PATH is specified explicitly.
-  if(NOT SWIFT_LIBRARY_PATH)
-    get_filename_component(tmp_dir "${SWIFT_EXEC}" DIRECTORY)
-    get_filename_component(tmp_dir "${tmp_dir}" DIRECTORY)
-    set(SWIFT_LIBRARY_PATH "${tmp_dir}/lib/swift")
+  # Set LIBRARY_PATH and either RPATH or RPATH_BASE. To build and run
+  # for multiple platforms, RPATH_BASE must be set instead of RPATH. It
+  # is the platform-independent runtime library directory. The platform
+  # subdirectory name will be appended to form a different RPATH for
+  # on platform.
+
+  # If requested, use Swift-in-the-OS. This way, the benchmarks may be built
+  # standalone on the host, and the binaries can run directly from a temp dir
+  # on any target machine. Of course, this factors out performance changes in
+  # stdlib or overlays.
+  if(SWIFT_BENCHMARK_USE_OS_LIBRARIES)
+    set(SWIFT_RPATH "/usr/lib/swift")
+  endif()
+
+  # When SWIFT_LIBRARY_PATH is specified explicitly for a standalone
+  # build, use it as an absolute RPATH_BASE. This only works when
+  # running benchmarks on the host machine. Otherwise, RPATH is set
+  # assuming that libraries will be installed later (manually)
+  # relative to the benchmark binaries.
+  #
+  # When not building standalone, SWIFT_LIBRARY_PATH is set by LLVM
+  # cmake to the build directory for Swift dylibs. Otherwise, assume
+  # that the dylibs are built relative to SWIFT_EXEC.
+  if(SWIFT_LIBRARY_PATH AND SWIFT_BENCHMARK_BUILT_STANDALONE)
+    if (NOT SWIFT_RPATH)
+      set(SWIFT_RPATH_BASE ${SWIFT_LIBRARY_PATH})
+    endif()
+  else()
+    if (NOT SWIFT_LIBRARY_PATH)
+      get_filename_component(tmp_dir "${SWIFT_EXEC}" DIRECTORY)
+      get_filename_component(tmp_dir "${tmp_dir}" DIRECTORY)
+      set(SWIFT_LIBRARY_PATH "${tmp_dir}/lib/swift")
+    endif()
+    if (NOT SWIFT_RPATH)
+      # If the benchmarks are built against a local swift build, assume that
+      # either the benchmarks will be installed in the swift build dir,
+      # or the swift libraries will be installed in the benchmark location in
+      # a platform specific subdirectory.
+      # This way, performance always factors in changes to the libraries.
+      set(SWIFT_RPATH_BASE "@executable_path/../lib/swift")
+    endif()
   endif()
 endmacro()
 
@@ -571,6 +609,11 @@ function (swift_benchmark_compile_archopts)
   # both do exactly the same thing with both sets of arguments. It also lets us
   # avoid issues around code-signing.
   if (is_darwin)
+    if (SWIFT_RPATH)
+      set(SWIFT_LINK_RPATH "${SWIFT_RPATH}")
+    else()
+      set(SWIFT_LINK_RPATH "${SWIFT_RPATH_BASE}/${BENCH_COMPILE_ARCHOPTS_PLATFORM}")
+    endif()
     add_custom_command(
         OUTPUT "${OUTPUT_EXEC}"
         DEPENDS
@@ -593,7 +636,7 @@ function (swift_benchmark_compile_archopts)
           "-lobjc"
           "-L${SWIFT_LIBRARY_PATH}/${BENCH_COMPILE_ARCHOPTS_PLATFORM}"
           "-Xlinker" "-rpath"
-          "-Xlinker" "@executable_path/../lib/swift/${BENCH_COMPILE_ARCHOPTS_PLATFORM}"
+          "-Xlinker" "${SWIFT_LINK_RPATH}"
           ${bench_library_objects}
           ${bench_driver_objects}
           ${SWIFT_BENCH_OBJFILES}
@@ -612,7 +655,7 @@ function (swift_benchmark_compile_archopts)
           "${SWIFT_EXEC}"
           "-O"
           "-target" "${target}"
-          "-L${SWIFT_LIBRARY_PATH}/${BENCH_COMPILE_ARCHOPTS_PLATFORM}"
+          "-L${SWIFT_LIBRARY_PATH}"
           ${bench_library_objects}
           ${bench_driver_objects}
           ${SWIFT_BENCH_OBJFILES}