[libc++] Allow testing Apple's system library as it is installed (#99086)

In order to test libc++ under the "Apple System Library" configuration,
we need to run the tests using DYLD_LIBRARY_PATH. This is required
because libc++ gets an install_name of /usr/lib when built as a system
library, which means that we must override the copy of libc++ used by
the whole process. This effectively reverts 2cf2f1b, which was the wrong
solution for the problem I was having.

Of course, this assumes that the just-built libc++ is sufficient to
replace the system library, which is not actually the case
out-of-the-box. Indeed, the system library contains a few symbols that
are not provided by the upstream library, leading to undefined symbols
when replacing the system library by the just-built one.

To solve this problem, we separately build shims that provide those
missing symbols and we manually link against them when we build
executables in the tests. While this is somewhat brittle, it provides a
localized and unintrusive way to allow testing the Apple system
configuration in an upstream environment, which has been a frequent
request.

Author: ldionne

libcxx/cmake/caches/Apple.cmake

@@ -15,3 +15,6 @@ set(LIBCXXABI_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "")
 set(LIBCXXABI_ENABLE_ASSERTIONS OFF CACHE BOOL "")
 set(LIBCXXABI_ENABLE_FORGIVING_DYNAMIC_CAST ON CACHE BOOL "")
 set(LIBCXXABI_USE_LLVM_UNWINDER OFF CACHE BOOL "") # libunwind is built separately
+
+set(LIBCXX_TEST_CONFIG "apple-libc++-shared.cfg.in" CACHE STRING "")
+set(LIBCXXABI_TEST_CONFIG "apple-libc++abi-shared.cfg.in" CACHE STRING "")

libcxx/test/configs/apple-libc++-shared.cfg.in

@@ -0,0 +1,51 @@
+# Testing configuration for Apple's system libc++.
+#
+# This configuration differs from a normal LLVM shared library configuration in
+# that we must use DYLD_LIBRARY_PATH to run the tests against the just-built library,
+# since Apple's libc++ has an absolute install_name.
+#
+# We also don't use a per-target include directory layout, so we have only one
+# include directory for the libc++ headers.
+#
+# Finally, we also link against an artificial shims library that provides
+# the functionality necessary for the upstream libc++ to be usable in place
+# of a system-provided libc++. Without that, attempting to replace the system
+# libc++ with DYLD_LIBRARY_PATH would result in missing symbols and other similar
+# issues since the upstream libc++ does not contain all the symbols provided by
+# the system library.
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.config, libcxx.test.dsl
+ADDITIONAL_PARAMETERS = [
+    libcxx.test.dsl.Parameter(name='apple_system_shims', type=str,
+        actions=lambda path: [libcxx.test.dsl.AddSubstitution('%{apple-system-shims}', path)],
+        help="""
+        Path to a pre-built object file or static archive that contains shims necessary to
+        allow replacing the system libc++ by the just-built one.
+        """),
+]
+
+config.substitutions.append(('%{flags}',
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
+))
+config.substitutions.append(('%{compile_flags}',
+    '-nostdinc++ -I %{include-dir} -I %{libcxx-dir}/test/support'
+))
+config.substitutions.append(('%{link_flags}',
+    '-nostdlib++ -L %{lib-dir} -lc++ %{apple-system-shims}'
+))
+config.substitutions.append(('%{exec}',
+    '%{executor} --execdir %T --env DYLD_LIBRARY_PATH=%{lib-dir} -- '
+))
+
+config.stdlib = 'apple-libc++'
+
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS + ADDITIONAL_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)

libcxx/utils/ci/apple-install-libcxx.sh

@@ -114,8 +114,13 @@ for arch in ${architectures}; do
     #       Then LLVM would guess the LLVM_DEFAULT_TARGET_TRIPLE properly and we wouldn't have to specify it.
     target=$(xcrun clang -arch ${arch} -xc - -### 2>&1 | grep --only-matching -E '"-triple" ".+?"' | grep --only-matching -E '"[^ ]+-apple-[^ ]+?"' | tr -d '"')
 
-    step "Building libc++.dylib and libc++abi.dylib for architecture ${arch}"
     mkdir -p "${build_dir}/${arch}"
+
+    step "Building shims to make libc++ compatible with the system libc++ on Apple platforms when running the tests"
+    shims_library="${build_dir}/${arch}/apple-system-shims.a"
+    xcrun clang++ -c -std=c++2b -target ${target} "${llvm_root}/libcxxabi/src/vendor/apple/shims.cpp" -static -o "${shims_library}"
+
+    step "Building libc++.dylib and libc++abi.dylib for architecture ${arch}"
     xcrun cmake -S "${llvm_root}/runtimes" \
                 -B "${build_dir}/${arch}" \
                 -GNinja \
@@ -127,9 +132,9 @@ for arch in ${architectures}; do
                 -DCMAKE_OSX_ARCHITECTURES="${arch}" \
                 -DLIBCXXABI_LIBRARY_VERSION="${version}" \
                 -DLIBCXX_LIBRARY_VERSION="${version}" \
-                -DLIBCXX_TEST_PARAMS="target_triple=${target};stdlib=apple-libc++" \
-                -DLIBCXXABI_TEST_PARAMS="target_triple=${target};stdlib=apple-libc++" \
-                -DLIBUNWIND_TEST_PARAMS="target_triple=${target};stdlib=apple-libc++"
+                -DLIBCXX_TEST_PARAMS="target_triple=${target};apple_system_shims=${shims_library}" \
+                -DLIBCXXABI_TEST_PARAMS="target_triple=${target};apple_system_shims=${shims_library}" \
+                -DLIBUNWIND_TEST_PARAMS="target_triple=${target};apple_system_shims=${shims_library}"
 
     if [ "$headers_only" = true ]; then
         xcrun cmake --build "${build_dir}/${arch}" --target install-cxx-headers install-cxxabi-headers -- -v

libcxxabi/src/vendor/apple/shims.cpp

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//
+// These shims implement symbols that are present in the system libc++ on Apple platforms
+// but are not implemented in upstream libc++. This allows testing libc++ under a system
+// library configuration, which requires the just-built libc++ to be ABI compatible with
+// the system library it is replacing.
+//
+
+#include <cstddef>
+#include <new>
+
+namespace std { // purposefully not versioned, like align_val_t
+enum class __type_descriptor_t : unsigned long long;
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::__type_descriptor_t) {
+  return ::operator new(__sz);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, const std::nothrow_t& __nt,
+                                                std::__type_descriptor_t) noexcept {
+  return ::operator new(__sz, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, std::__type_descriptor_t) {
+  return ::operator new[](__sz);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new[](std::size_t __sz, const std::nothrow_t& __nt,
+                                                  std::__type_descriptor_t) noexcept {
+  return ::operator new(__sz, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::__type_descriptor_t) noexcept {
+  return ::operator delete(__p);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t& __nt,
+                                                  std::__type_descriptor_t) noexcept {
+  return ::operator delete(__p, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::__type_descriptor_t) noexcept {
+  return ::operator delete[](__p);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t& __nt,
+                                                    std::__type_descriptor_t) noexcept {
+  return ::operator delete[](__p, __nt);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz, std::__type_descriptor_t) noexcept {
+  return ::operator delete(__p, __sz);
+}
+
+_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz, std::__type_descriptor_t) noexcept {
+  return ::operator delete[](__p, __sz);
+}

libcxxabi/test/configs/apple-libc++abi-shared.cfg.in

@@ -0,0 +1,49 @@
+# Testing configuration for Apple's system libc++abi.
+#
+# This configuration differs from a normal LLVM shared library configuration in
+# that we must use DYLD_LIBRARY_PATH to run the tests against the just-built library,
+# since Apple's libc++abi has an absolute install_name.
+#
+# Finally, we also link against an artificial shims library that provides
+# the functionality necessary for the upstream libc++abi to be usable in place
+# of a system-provided libc++abi. Without that, attempting to replace the system
+# libc++abi with DYLD_LIBRARY_PATH would result in missing symbols and other similar
+# issues since the upstream libc++abi does not contain all the symbols provided by
+# the system library.
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXXABI_LIBCXX_PATH@', 'utils'))
+import libcxx.test.params, libcxx.test.config, libcxx.test.dsl
+ADDITIONAL_PARAMETERS = [
+    libcxx.test.dsl.Parameter(name='apple_system_shims', type=str,
+        actions=lambda path: [libcxx.test.dsl.AddSubstitution('%{apple-system-shims}', path)],
+        help="""
+        Path to a pre-built object file or static archive that contains shims necessary to
+        allow replacing the system libc++abi by the just-built one.
+        """),
+]
+
+config.substitutions.append(('%{flags}',
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
+))
+config.substitutions.append(('%{compile_flags}',
+    '-nostdinc++ -I %{include} -I %{cxx-include} -I %{cxx-target-include} %{maybe-include-libunwind} -D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS ' +
+    '-I %{libcxx}/test/support -I %{libcxx}/src'
+))
+config.substitutions.append(('%{link_flags}',
+    '-nostdlib++ -L %{lib} -lc++ %{apple-system-shims}'
+))
+config.substitutions.append(('%{exec}',
+    '%{executor} --execdir %T --env DYLD_LIBRARY_PATH=%{lib} -- '
+))
+
+config.stdlib = 'apple-libc++'
+
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS + ADDITIONAL_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)