[libc++] Remove special handling of the native C++ library in benchmarks (#98529)

There were some ad-hoc settings that allowed running the benchmarks
against the native C++ Standard Library. While this ability is very
useful, it was done before the test suite was quite independent of
libc++ itself. Instead, it is better to streamline running the
benchmarks on the native standard library by using a custom Lit
configuration like we do with the test suite.

A follow-up patch will rework the integration of benchmarks with the Lit
configuration used for the test suite so that we can reuse the same
mechanism for both, making it easy to benchmark the native standard
library.

It will also make benchmarks way more user-friendly to run since we will
be able to run them like we run individual tests, which is a pain point
right now.

Author: ldionne

libcxx/CMakeLists.txt

@@ -159,20 +159,6 @@ set(LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT --benchmark_min_time=0.01)
 set(LIBCXX_BENCHMARK_TEST_ARGS "${LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT}" CACHE STRING
     "Arguments to pass when running the benchmarks using check-cxx-benchmarks")
 
-set(LIBCXX_BENCHMARK_NATIVE_STDLIB "" CACHE STRING
-        "Build the benchmarks against the specified native STL.
-         The value must be one of libc++/libstdc++")
-set(LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN "" CACHE STRING
-    "Use alternate GCC toolchain when building the native benchmarks")
-
-if (LIBCXX_BENCHMARK_NATIVE_STDLIB)
-  if (NOT (LIBCXX_BENCHMARK_NATIVE_STDLIB STREQUAL "libc++"
-        OR LIBCXX_BENCHMARK_NATIVE_STDLIB STREQUAL "libstdc++"))
-    message(FATAL_ERROR "Invalid value for LIBCXX_BENCHMARK_NATIVE_STDLIB: "
-            "'${LIBCXX_BENCHMARK_NATIVE_STDLIB}'")
-  endif()
-endif()
-
 option(LIBCXX_INCLUDE_DOCS "Build the libc++ documentation." ${LLVM_INCLUDE_DOCS})
 set(LIBCXX_LIBDIR_SUFFIX "${LLVM_LIBDIR_SUFFIX}" CACHE STRING
     "Define suffix of library directory name (32/64)")

libcxx/benchmarks/CMakeLists.txt

@@ -2,12 +2,12 @@ include(ExternalProject)
 include(CheckCXXCompilerFlag)
 
 #==============================================================================
-# Build Google Benchmark for libc++
+# Build Google Benchmark
 #==============================================================================
 
 set(CMAKE_FOLDER "${CMAKE_FOLDER}/Benchmarks")
 
-set(BENCHMARK_LIBCXX_COMPILE_FLAGS
+set(BENCHMARK_COMPILE_FLAGS
     -Wno-unused-command-line-argument
     -nostdinc++
     -isystem "${LIBCXX_GENERATED_INCLUDE_DIR}"
@@ -16,64 +16,37 @@ set(BENCHMARK_LIBCXX_COMPILE_FLAGS
     ${SANITIZER_FLAGS}
     )
 if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
-  list(APPEND BENCHMARK_LIBCXX_COMPILE_FLAGS
+  list(APPEND BENCHMARK_COMPILE_FLAGS
     -isystem "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}")
 endif()
 if (DEFINED LIBCXX_CXX_ABI_LIBRARY_PATH)
-  list(APPEND BENCHMARK_LIBCXX_COMPILE_FLAGS
+  list(APPEND BENCHMARK_COMPILE_FLAGS
           -L${LIBCXX_CXX_ABI_LIBRARY_PATH}
           -Wl,-rpath,${LIBCXX_CXX_ABI_LIBRARY_PATH})
 endif()
-split_list(BENCHMARK_LIBCXX_COMPILE_FLAGS)
+split_list(BENCHMARK_COMPILE_FLAGS)
 
-ExternalProject_Add(google-benchmark-libcxx
+ExternalProject_Add(google-benchmark
         EXCLUDE_FROM_ALL ON
         DEPENDS cxx cxx-headers
-        PREFIX benchmark-libcxx
+        PREFIX google-benchmark
         SOURCE_DIR ${LLVM_THIRD_PARTY_DIR}/benchmark
-        INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/benchmark-libcxx
+        INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/google-benchmark
         CMAKE_CACHE_ARGS
           -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
           -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
           -DCMAKE_BUILD_TYPE:STRING=RELEASE
           -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-          -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_LIBCXX_COMPILE_FLAGS}
+          -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_COMPILE_FLAGS}
           -DBENCHMARK_USE_LIBCXX:BOOL=ON
           -DBENCHMARK_ENABLE_TESTING:BOOL=OFF)
 
-#==============================================================================
-# Build Google Benchmark for the native stdlib
-#==============================================================================
-set(BENCHMARK_NATIVE_TARGET_FLAGS)
-if (LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN)
-  set(BENCHMARK_NATIVE_TARGET_FLAGS
-      --gcc-toolchain=${LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN})
-endif()
-split_list(BENCHMARK_NATIVE_TARGET_FLAGS)
-
-if (LIBCXX_BENCHMARK_NATIVE_STDLIB)
-  ExternalProject_Add(google-benchmark-native
-        EXCLUDE_FROM_ALL ON
-        PREFIX benchmark-native
-        SOURCE_DIR ${LLVM_THIRD_PARTY_DIR}/benchmark
-        INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/benchmark-native
-        CMAKE_CACHE_ARGS
-          -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
-          -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
-          -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_NATIVE_TARGET_FLAGS}
-          -DCMAKE_BUILD_TYPE:STRING=RELEASE
-          -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-          -DBENCHMARK_ENABLE_TESTING:BOOL=OFF)
-endif()
-
-
 #==============================================================================
 # Benchmark tests configuration
 #==============================================================================
 add_custom_target(cxx-benchmarks)
 set(BENCHMARK_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(BENCHMARK_LIBCXX_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/benchmark-libcxx)
-set(BENCHMARK_NATIVE_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/benchmark-native)
+set(BENCHMARK_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/google-benchmark)
 
 add_library(               cxx-benchmarks-flags INTERFACE)
 
@@ -97,41 +70,23 @@ else()
   target_compile_features( cxx-benchmarks-flags INTERFACE cxx_std_23)
 endif()
 
-target_compile_options(    cxx-benchmarks-flags INTERFACE -fsized-deallocation -nostdinc++)
+target_compile_options(cxx-benchmarks-flags INTERFACE -fsized-deallocation -nostdinc++
+                                                      ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
 target_include_directories(cxx-benchmarks-flags INTERFACE "${LIBCXX_GENERATED_INCLUDE_DIR}"
-                                                INTERFACE "${BENCHMARK_LIBCXX_INSTALL}/include"
+                                                INTERFACE "${BENCHMARK_INSTALL_DIR}/include"
                                                 INTERFACE "${LIBCXX_SOURCE_DIR}/test/support")
-
-add_library(           cxx-benchmarks-flags-native INTERFACE)
-target_link_libraries( cxx-benchmarks-flags-native INTERFACE cxx-benchmarks-flags)
-target_compile_options(cxx-benchmarks-flags-native INTERFACE ${BENCHMARK_NATIVE_TARGET_FLAGS})
-target_link_options(   cxx-benchmarks-flags-native INTERFACE ${BENCHMARK_NATIVE_TARGET_FLAGS} "-L${BENCHMARK_NATIVE_INSTALL}/lib")
-if (LIBCXX_BENCHMARK_NATIVE_STDLIB STREQUAL "libstdc++")
-  find_library(LIBSTDCXX_FILESYSTEM_TEST stdc++fs
-        PATHS ${LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN}
-        PATH_SUFFIXES lib lib64
-        DOC "The libstdc++ filesystem library used by the benchmarks"
-    )
-  if (LIBSTDCXX_FILESYSTEM_TEST)
-    target_link_libraries(cxx-benchmarks-flags-native INTERFACE -lstdc++fs)
-  endif()
-else()
-  target_link_libraries(cxx-benchmarks-flags-native INTERFACE -lc++fs -lc++experimental)
-endif()
-
-add_library(           cxx-benchmarks-flags-libcxx INTERFACE)
-target_link_libraries( cxx-benchmarks-flags-libcxx INTERFACE cxx-benchmarks-flags)
-target_compile_options(cxx-benchmarks-flags-libcxx INTERFACE ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
-target_link_options(   cxx-benchmarks-flags-libcxx INTERFACE -lm -nostdlib++ "-L${BENCHMARK_LIBCXX_INSTALL}/lib" "-L${BENCHMARK_LIBCXX_INSTALL}/lib64" ${SANITIZER_FLAGS})
+target_link_options(cxx-benchmarks-flags INTERFACE -lm -nostdlib++
+                                                   "-L${BENCHMARK_INSTALL_DIR}/lib" "-L${BENCHMARK_INSTALL_DIR}/lib64"
+                                                   ${SANITIZER_FLAGS})
 
 set(libcxx_benchmark_targets)
 
 function(add_benchmark_test name source_file)
   set(libcxx_target ${name}_libcxx)
   list(APPEND libcxx_benchmark_targets ${libcxx_target})
   add_executable(${libcxx_target} EXCLUDE_FROM_ALL ${source_file})
-  target_link_libraries(${libcxx_target} PRIVATE cxx-benchmarks-flags-libcxx)
-  add_dependencies(${libcxx_target} cxx google-benchmark-libcxx)
+  target_link_libraries(${libcxx_target} PRIVATE cxx-benchmarks-flags)
+  add_dependencies(${libcxx_target} cxx google-benchmark)
   add_dependencies(cxx-benchmarks ${libcxx_target})
   if (LIBCXX_ENABLE_SHARED)
     target_link_libraries(${libcxx_target} PRIVATE cxx_shared)
@@ -144,27 +99,10 @@ function(add_benchmark_test name source_file)
   endif()
   set_target_properties(${libcxx_target}
     PROPERTIES
-          OUTPUT_NAME "${name}.libcxx.out"
+          OUTPUT_NAME "${name}.bench.out"
           RUNTIME_OUTPUT_DIRECTORY "${BENCHMARK_OUTPUT_DIR}"
           CXX_EXTENSIONS NO)
   cxx_link_system_libraries(${libcxx_target})
-  if (LIBCXX_BENCHMARK_NATIVE_STDLIB)
-    set(native_target ${name}_native)
-    add_executable(${native_target} EXCLUDE_FROM_ALL ${source_file})
-    target_link_libraries(${native_target} PRIVATE cxx-benchmarks-flags-native)
-    add_dependencies(${native_target} google-benchmark-native
-                                      google-benchmark-libcxx)
-    target_link_libraries(${native_target} PRIVATE -lbenchmark)
-    if (LIBCXX_HAS_PTHREAD_LIB)
-      target_link_libraries(${native_target} PRIVATE -pthread)
-    endif()
-    add_dependencies(cxx-benchmarks ${native_target})
-    set_target_properties(${native_target}
-      PROPERTIES
-          OUTPUT_NAME "${name}.native.out"
-          RUNTIME_OUTPUT_DIRECTORY "${BENCHMARK_OUTPUT_DIR}"
-          CXX_EXTENSIONS NO)
-  endif()
 endfunction()
 
 

libcxx/benchmarks/lit.cfg.py

@@ -19,5 +19,5 @@
 config.test_source_root = config.test_exec_root
 
 config.test_format = GoogleBenchmark(
-    test_sub_dirs=".", test_suffix=".libcxx.out", benchmark_args=config.benchmark_args
+    test_sub_dirs=".", test_suffix=".bench.out", benchmark_args=config.benchmark_args
 )

libcxx/docs/BuildingLibcxx.rst

@@ -399,22 +399,6 @@ libc++ Feature Options
   since the primary use of ``check-cxx-benchmarks`` is to get test and sanitizer coverage, not to
   get accurate measurements.
 
-.. option:: LIBCXX_BENCHMARK_NATIVE_STDLIB:STRING
-
-  **Default**:: ``""``
-
-  **Values**:: ``libc++``, ``libstdc++``
-
-  Build the libc++ benchmark tests and Google Benchmark library against the
-  specified standard library on the platform. On Linux this can be used to
-  compare libc++ to libstdc++ by building the benchmark tests against both
-  standard libraries.
-
-.. option:: LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN:STRING
-
-  Use the specified GCC toolchain and standard library when building the native
-  stdlib benchmark tests.
-
 .. option:: LIBCXX_ASSERTION_HANDLER_FILE:PATH
 
   **Default**:: ``"${CMAKE_CURRENT_SOURCE_DIR}/vendor/llvm/default_assertion_handler.in"``

libcxx/docs/ReleaseNotes/19.rst

@@ -178,3 +178,7 @@ Build System Changes
   to automatically detect the presence of ``clang-tidy`` and the required ``Clang`` libraries.
 
 - The CMake options ``LIBCXX_INSTALL_MODULES`` now defaults to ``ON``.
+
+- The CMake options ``LIBCXX_BENCHMARK_NATIVE_STDLIB`` and ``LIBCXX_BENCHMARK_NATIVE_GCC_TOOLCHAIN`` have
+  been removed. To benchmark the native standard library, configure the test suite against the native
+  standard library directly instead.

libcxx/docs/TestingLibcxx.rst

@@ -351,7 +351,7 @@ Test Filenames`_ when determining the names for new test files.
      - Same as ``FOO.pass.cpp``, but for Objective-C++.
 
    * - ``FOO.compile.pass.cpp``
-     - Checks whether the C++ code in the file compiles successfully. In general, prefer ``compile`` tests over ``verify`` tests, 
+     - Checks whether the C++ code in the file compiles successfully. In general, prefer ``compile`` tests over ``verify`` tests,
        subject to the specific recommendations, below, for when to write ``verify`` tests.
    * - ``FOO.compile.pass.mm``
      - Same as ``FOO.compile.pass.cpp``, but for Objective-C++.
@@ -447,19 +447,12 @@ An example build would look like:
 
 .. code-block:: bash
 
-  $ cd build
-  $ ninja cxx-benchmarks
+  $ ninja -C build cxx-benchmarks
 
 This will build all of the benchmarks under ``<libcxx-src>/benchmarks`` to be
 built against the just-built libc++. The compiled tests are output into
 ``build/projects/libcxx/benchmarks``.
 
-The benchmarks can also be built against the platforms native standard library
-using the ``-DLIBCXX_BUILD_BENCHMARKS_NATIVE_STDLIB=ON`` CMake option. This
-is useful for comparing the performance of libc++ to other standard libraries.
-The compiled benchmarks are named ``<test>.libcxx.out`` if they test libc++ and
-``<test>.native.out`` otherwise.
-
 Also See:
 
   * :ref:`Building Libc++ <build instructions>`
@@ -476,8 +469,8 @@ For example:
 .. code-block:: bash
 
   $ cd build/projects/libcxx/benchmarks
-  $ ./algorithms.libcxx.out # Runs all the benchmarks
-  $ ./algorithms.libcxx.out --benchmark_filter=BM_Sort.* # Only runs the sort benchmarks
+  $ ./algorithms.bench.out # Runs all the benchmarks
+  $ ./algorithms.bench.out --benchmark_filter=BM_Sort.* # Only runs the sort benchmarks
 
 For more information about running benchmarks see `Google Benchmark`_.