[libc++] Add tombstone traits to use in optional, variant

Author: philnik777

libcxx/include/CMakeLists.txt

@@ -560,6 +560,7 @@ set(files
   __memory/swap_allocator.h
   __memory/temp_value.h
   __memory/temporary_buffer.h
+  __memory/tombstone_traits.h
   __memory/uninitialized_algorithms.h
   __memory/unique_ptr.h
   __memory/unique_temporary_buffer.h

libcxx/include/__configuration/abi.h

@@ -124,6 +124,8 @@
 // This setting disables the addition of such artificial padding, leading to a more optimal
 // representation for several types.
 #  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
+// Use __tombstone_traits to optimize the memory layout of std::optional.
+#  define _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support
@@ -141,6 +143,9 @@
 #  if defined(__FreeBSD__) && __FreeBSD__ < 14
 #    define _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR
 #  endif
+
+// TODO: This shouldn't be in the final commit - this just to test the changes across all the different configurations
+#  define _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
 #endif
 
 // We had some bugs where we use [[no_unique_address]] together with construct_at,

libcxx/include/__functional/reference_wrapper.h

@@ -13,15 +13,18 @@
 #include <__compare/synth_three_way.h>
 #include <__concepts/boolean_testable.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
+#include <__memory/tombstone_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
+#include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -120,7 +123,13 @@ class _LIBCPP_TEMPLATE_VIS reference_wrapper : public __weak_result_type<_Tp> {
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<reference_wrapper<_Tp>> {
+  static constexpr uintptr_t __disengaged_value_  = 0;
+  static constexpr size_t __is_disengaged_offset_ = 0;
+};
+#endif // _LIBCPP_STD_VER >= 17
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {

libcxx/include/__locale

@@ -13,6 +13,7 @@
 #include <__config>
 #include <__locale_dir/locale_base_api.h>
 #include <__memory/shared_ptr.h> // __shared_count
+#include <__memory/tombstone_traits.h>
 #include <__mutex/once_flag.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/no_destroy.h>
@@ -114,6 +115,11 @@ private:
   friend const _Facet& use_facet(const locale&);
 };
 
+#if _LIBCPP_STD_VER >= 17
+template <>
+struct __tombstone_traits<locale> : __tombstone_traits_assume_aligned_pointer {};
+#endif
+
 class _LIBCPP_EXPORTED_FROM_ABI locale::facet : public __shared_count {
 protected:
   _LIBCPP_HIDE_FROM_ABI explicit facet(size_t __refs = 0) : __shared_count(static_cast<long>(__refs) - 1) {}

libcxx/include/__memory/shared_ptr.h

@@ -30,6 +30,7 @@
 #include <__memory/compressed_pair.h>
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
+#include <__memory/tombstone_traits.h>
 #include <__memory/uninitialized_algorithms.h>
 #include <__memory/unique_ptr.h>
 #include <__type_traits/add_lvalue_reference.h>
@@ -831,7 +832,14 @@ template <class _Tp>
 shared_ptr(weak_ptr<_Tp>) -> shared_ptr<_Tp>;
 template <class _Tp, class _Dp>
 shared_ptr(unique_ptr<_Tp, _Dp>) -> shared_ptr<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<shared_ptr<_Tp>> {
+  static constexpr auto __disengaged_value_ = __tombstone_traits_assume_aligned_pointer::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ =
+      sizeof(void*) + __tombstone_traits_assume_aligned_pointer::__is_disengaged_offset_;
+};
+#endif // _LIBCPP_STD_VER >= 17
 
 //
 // std::allocate_shared and std::make_shared
@@ -1381,7 +1389,14 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 weak_ptr(shared_ptr<_Tp>) -> weak_ptr<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<weak_ptr<_Tp>> {
+  static constexpr auto __disengaged_value_ = __tombstone_traits_assume_aligned_pointer::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ =
+      sizeof(void*) + __tombstone_traits_assume_aligned_pointer::__is_disengaged_offset_;
+};
+#endif // _LIBCPP_STD_VER >= 17
 
 template <class _Tp>
 inline _LIBCPP_CONSTEXPR weak_ptr<_Tp>::weak_ptr() _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}

libcxx/include/__memory/tombstone_traits.h

@@ -0,0 +1,277 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
+#define _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H
+
+#include <__assert>
+#include <__config>
+#include <__cstddef/size_t.h>
+#include <__memory/construct_at.h>
+#include <__type_traits/datasizeof.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_fundamental.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_trivial.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/remove_cv.h>
+#include <__type_traits/void_t.h>
+#include <__utility/forward.h>
+#include <__utility/move.h>
+#include <__utility/piecewise_construct.h>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+struct __tombstone_traits;
+
+// bools have always exactly one bit set. If there is more than one set it's disengaged.
+template <>
+struct __tombstone_traits<bool> {
+  static constexpr uint8_t __disengaged_value_    = 3;
+  static constexpr size_t __is_disengaged_offset_ = 0;
+};
+
+struct __tombstone_traits_assume_aligned_pointer {
+  static constexpr uint8_t __disengaged_value_ = 1;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+  static constexpr size_t __is_disengaged_offset_ = 0;
+#else
+  static constexpr size_t __is_disengaged_offset_ = sizeof(void*) - 1;
+#endif
+};
+
+// TODO: Look into
+// - filesystem::directory_iterator
+// - vector<T> with alignof(T) == 1
+// - string_view (basic_string_view<T> works with alignof(T) >= 2)
+
+// This is constrained on fundamental types because we might not always know the alignment of a user-defined type.
+// For example, in one TU there may only be a forward declaration and in another there is already the definition
+// available. If we made this optimization conditional on the completeness of the type this would result in a non-benign
+// ODR violation.
+template <class _Tp>
+struct __tombstone_traits<__enable_specialization_if<is_fundamental_v<_Tp> && alignof(_Tp) >= 2, _Tp*>>
+    : __tombstone_traits_assume_aligned_pointer {};
+
+template <class _Tp>
+struct __tombstone_traits<_Tp**> : __tombstone_traits_assume_aligned_pointer {
+  static_assert(alignof(_Tp*) >= 2, "alignment of a pointer isn't at least 2!?");
+};
+
+inline constexpr struct __init_engaged_t {
+} __init_engaged;
+inline constexpr struct __init_disengaged_t {
+} __init_disengaged;
+
+template <class _Tp, class _Payload, bool = __tombstone_traits<_Tp>::__is_disengaged_offset_ == 0>
+struct __tombstone_is_disengaged {
+  using _TombstoneLayout = __tombstone_traits<_Tp>;
+  using _IsDisengagedT   = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+  char __padding_[_TombstoneLayout::__is_disengaged_offset_];
+  _IsDisengagedT __is_disengaged_;
+};
+
+template <class _Tp, class _Payload>
+struct __tombstone_is_disengaged<_Tp, _Payload, true> {
+  using _TombstoneLayout = __tombstone_traits<_Tp>;
+  using _IsDisengagedT   = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+  _IsDisengagedT __is_disengaged_;
+};
+
+template <class _Tp, class _Payload, bool = __tombstone_traits<_Tp>::__is_disengaged_offset_ == 0>
+struct __tombstone_data {
+  using _TombstoneLayout = __tombstone_traits<_Tp>;
+  using _IsDisengagedT   = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+  static_assert(is_trivial<_IsDisengagedT>::value, "disengaged type has to be trivial!");
+  static_assert(_TombstoneLayout::__is_disengaged_offset_ >= __datasizeof_v<_Payload>);
+
+  _LIBCPP_NO_UNIQUE_ADDRESS _Payload __payload_;
+  char __padding_[_TombstoneLayout::__is_disengaged_offset_ - __datasizeof_v<_Payload>];
+  _IsDisengagedT __is_disengaged_;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_data(_Args&&... __args)
+      : __payload_(std::forward<_Args>(__args)...), __is_disengaged_(_TombstoneLayout::__disengaged_value_) {}
+};
+
+template <class _Tp, class _Payload>
+struct __tombstone_data<_Tp, _Payload, true> {
+  using _TombstoneLayout = __tombstone_traits<_Tp>;
+  using _IsDisengagedT   = remove_cv_t<decltype(_TombstoneLayout::__disengaged_value_)>;
+
+  _IsDisengagedT __is_disengaged_;
+  _LIBCPP_NO_UNIQUE_ADDRESS _Payload __payload_;
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstone_data(_Args&&... __args)
+      : __is_disengaged_(_TombstoneLayout::__disengaged_value_), __payload_(std::forward<_Args>(__args)...) {}
+};
+
+template <class _Tp, class _Payload>
+struct __tombstoned_value final {
+  using _TombstoneLayout = __tombstone_traits<_Tp>;
+  using _TombstoneData   = __tombstone_data<_Tp, _Payload>;
+
+  template <bool = is_trivially_destructible_v<_Tp> && is_trivially_destructible_v<_Payload>>
+  union _MaybeTombstone {
+    _Tp __value_;
+    _TombstoneData __tombstone_;
+
+    template <class... _Args>
+    constexpr _MaybeTombstone(__init_disengaged_t, _Args&&... __args) : __tombstone_(std::forward<_Args>(__args)...) {}
+
+    template <class... _Args>
+    constexpr _MaybeTombstone(__init_engaged_t, _Args&&... __args) : __value_(std::forward<_Args>(__args)...) {}
+
+    _MaybeTombstone(const _MaybeTombstone&)            = default;
+    _MaybeTombstone(_MaybeTombstone&&)                 = default;
+    _MaybeTombstone& operator=(const _MaybeTombstone&) = default;
+    _MaybeTombstone& operator=(_MaybeTombstone&&)      = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept {
+      if (__libcpp_is_constant_evaluated())
+        return !__builtin_constant_p(__tombstone_.__is_disengaged_ == _TombstoneLayout::__disengaged_value_);
+      __tombstone_is_disengaged<_Tp, _Payload> __is_disengaged;
+      static_assert(sizeof(__tombstone_is_disengaged<_Tp, _Payload>) <= sizeof(_MaybeTombstone));
+      __builtin_memcpy(&__is_disengaged, this, sizeof(__tombstone_is_disengaged<_Tp, _Payload>));
+
+      return __is_disengaged.__is_disengaged_ != _TombstoneLayout::__disengaged_value_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~_MaybeTombstone() {
+      if (__is_engaged()) {
+        std::destroy_at(&__value_);
+      } else {
+        std::destroy_at(&__tombstone_);
+      }
+    }
+  };
+
+  template <>
+  union _MaybeTombstone<true> {
+    _Tp __value_;
+    _TombstoneData __tombstone_;
+
+    template <class... _Args>
+    constexpr _MaybeTombstone(__init_disengaged_t, _Args&&... __args) : __tombstone_(std::forward<_Args>(__args)...) {}
+
+    template <class... _Args>
+    constexpr _MaybeTombstone(__init_engaged_t, _Args&&... __args) : __value_(std::forward<_Args>(__args)...) {}
+
+    _MaybeTombstone(const _MaybeTombstone&)            = default;
+    _MaybeTombstone(_MaybeTombstone&&)                 = default;
+    _MaybeTombstone& operator=(const _MaybeTombstone&) = default;
+    _MaybeTombstone& operator=(_MaybeTombstone&&)      = default;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept {
+      if (__libcpp_is_constant_evaluated())
+        return !__builtin_constant_p(__tombstone_.__is_disengaged_ == _TombstoneLayout::__disengaged_value_);
+      __tombstone_is_disengaged<_Tp, _Payload> __is_disengaged;
+      static_assert(sizeof(__tombstone_is_disengaged<_Tp, _Payload>) <= sizeof(_MaybeTombstone));
+      __builtin_memcpy(&__is_disengaged, this, sizeof(__tombstone_is_disengaged<_Tp, _Payload>));
+
+      return __is_disengaged.__is_disengaged_ != _TombstoneLayout::__disengaged_value_;
+    }
+
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 ~_MaybeTombstone() = default;
+  };
+
+  _MaybeTombstone<> __data_;
+
+  static_assert(sizeof(__tombstone_data<_Tp, _Payload>) <= sizeof(_Tp));
+  static_assert(is_integral_v<decltype(_TombstoneLayout::__disengaged_value_)>);
+  static_assert(__builtin_offsetof(_TombstoneData, __is_disengaged_) == _TombstoneLayout::__is_disengaged_offset_);
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstoned_value(__init_disengaged_t, _Args&&... __args)
+      : __data_(__init_disengaged, std::forward<_Args>(__args)...) {}
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr __tombstoned_value(__init_engaged_t, _Args&&... __args)
+      : __data_(__init_engaged, std::forward<_Args>(__args)...) {}
+
+  _LIBCPP_HIDE_FROM_ABI constexpr bool __is_engaged() const noexcept { return __data_.__is_engaged(); }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() & noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__self.__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+    return __data_.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() const & noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__self.__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+    return __data_.__value_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() && noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__self.__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+    return std::move(__data_.__value_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_value() const && noexcept {
+    _LIBCPP_ASSERT_INTERNAL(__self.__is_engaged(), "Trying to get the value of a disenagaged tombstoned value");
+    return std::move(__data_.__value_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() & noexcept {
+    _LIBCPP_ASSERT_INTERNAL(!__self.__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+    return __data_.__tombstone_.__payload_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() const & noexcept {
+    _LIBCPP_ASSERT_INTERNAL(!__self.__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+    return __data_.__tombstone_.__payload_;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() && noexcept {
+    _LIBCPP_ASSERT_INTERNAL(!__self.__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+    return std::move(__data_.__tombstone_.__payload_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto&& __get_payload() const && noexcept {
+    _LIBCPP_ASSERT_INTERNAL(!__self.__is_engaged(), "Trying to get the payload of an enagaged tombstoned value");
+    return std::move(__data_.__tombstone_.__payload_);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __engage(piecewise_construct_t, _Args&&... __args) {
+    _LIBCPP_ASSERT_INTERNAL(!__self.__is_engaged(), "Trying to enage a already engaged tombstoned value");
+    std::destroy_at(&__data_.__tombstone_);
+    std::__construct_at(&__data_.__value_, std::forward<_Args>(__args)...);
+  }
+
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI constexpr void __disengage(piecewise_construct_t, _Args&&... __args) {
+    _LIBCPP_ASSERT_INTERNAL(!__self.__is_engaged(), "Trying to disenage a disengaged tombstoned value");
+    std::destroy_at(&__data_.__value_);
+    std::__construct_at(&__data_.__tombstone_, std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Tp, class = void>
+inline constexpr bool __has_tombstone_v = false;
+
+template <class _Tp>
+inline constexpr bool __has_tombstone_v<_Tp, void_t<decltype(sizeof(__tombstone_traits<_Tp>))>> = true;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___TYPE_TRAITS_DISENGAGED_TRAITS_H