[libc++] Add tombstone traits and use them in optional

libcxx/include/CMakeLists.txt

@@ -563,6 +563,7 @@ set(files
   __memory/swap_allocator.h
   __memory/temp_value.h
   __memory/temporary_buffer.h
+  __memory/tombstone_traits.h
   __memory/uninitialized_algorithms.h
   __memory/unique_ptr.h
   __memory/unique_temporary_buffer.h

libcxx/include/__configuration/abi.h

@@ -124,6 +124,8 @@
 // This setting disables the addition of such artificial padding, leading to a more optimal
 // representation for several types.
 #  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
+// Use __tombstone_traits to optimize the memory layout of std::optional.
+#  define _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support
@@ -141,6 +143,9 @@
 #  if defined(__FreeBSD__) && __FreeBSD__ < 14
 #    define _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR
 #  endif
+
+// TODO: This shouldn't be in the final commit - this just to test the changes across all the different configurations
+#  define _LIBCPP_ABI_OPTIONAL_USE_TOMBSTONE_TRAITS
 #endif
 
 // We had some bugs where we use [[no_unique_address]] together with construct_at,

libcxx/include/__functional/reference_wrapper.h

@@ -13,15 +13,18 @@
 #include <__compare/synth_three_way.h>
 #include <__concepts/boolean_testable.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
+#include <__memory/tombstone_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
+#include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -120,7 +123,15 @@ class _LIBCPP_TEMPLATE_VIS reference_wrapper : public __weak_result_type<_Tp> {
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<reference_wrapper<_Tp>> {
+  // reference_wrapper<_Tp> is conceptually that same as _Tp& and can only be constructed from a _Tp&. Since references
+  // can never be null, reference_wrapper can also never be null, allowing us to use it as an invalid state.
+  static constexpr _Tp* __disengaged_value_       = nullptr;
+  static constexpr size_t __is_disengaged_offset_ = __builtin_offsetof(reference_wrapper<_Tp>, __f_);
+};
+#endif // _LIBCPP_STD_VER >= 17
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {

libcxx/include/__locale

@@ -13,6 +13,7 @@
 #include <__config>
 #include <__locale_dir/locale_base_api.h>
 #include <__memory/shared_count.h>
+#include <__memory/tombstone_traits.h>
 #include <__mutex/once_flag.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/no_destroy.h>
@@ -95,8 +96,9 @@ public:
   static locale global(const locale&);
   static const locale& classic();
 
-private:
   class __imp;
+
+private:
   __imp* __locale_;
 
   template <class>
@@ -112,8 +114,19 @@ private:
   friend bool has_facet(const locale&) _NOEXCEPT;
   template <class _Facet>
   friend const _Facet& use_facet(const locale&);
+
+  friend struct __tombstone_traits<locale>;
 };
 
+#if _LIBCPP_STD_VER >= 17
+// Derived from the void* specialization, since __imp is never defined.
+template <>
+struct __tombstone_traits<locale> : __tombstone_traits_assume_aligned_pointer<void*> {
+  static_assert(__builtin_offsetof(locale, __locale_) == 0);
+  static_assert(sizeof(locale) == sizeof(void*));
+};
+#endif
+
 class _LIBCPP_EXPORTED_FROM_ABI locale::facet : public __shared_count {
 protected:
   _LIBCPP_HIDE_FROM_ABI explicit facet(size_t __refs = 0) : __shared_count(static_cast<long>(__refs) - 1) {}

libcxx/include/__memory/shared_ptr.h

@@ -31,6 +31,7 @@
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/shared_count.h>
+#include <__memory/tombstone_traits.h>
 #include <__memory/uninitialized_algorithms.h>
 #include <__memory/unique_ptr.h>
 #include <__type_traits/add_lvalue_reference.h>
@@ -712,14 +713,25 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr {
   friend class _LIBCPP_TEMPLATE_VIS shared_ptr;
   template <class _Up>
   friend class _LIBCPP_TEMPLATE_VIS weak_ptr;
+
+  friend struct __tombstone_traits<shared_ptr<_Tp> >;
 };
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 shared_ptr(weak_ptr<_Tp>) -> shared_ptr<_Tp>;
 template <class _Tp, class _Dp>
 shared_ptr(unique_ptr<_Tp, _Dp>) -> shared_ptr<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<shared_ptr<_Tp>> {
+  static constexpr auto __disengaged_value_ =
+      __tombstone_traits_assume_aligned_pointer<__shared_weak_count*>::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ =
+      __builtin_offsetof(shared_ptr<_Tp>, __cntrl_) +
+      __tombstone_traits_assume_aligned_pointer<__shared_weak_count*>::__is_disengaged_offset_;
+};
+#endif // _LIBCPP_STD_VER >= 17
 
 //
 // std::allocate_shared and std::make_shared
@@ -1264,12 +1276,23 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
   friend class _LIBCPP_TEMPLATE_VIS weak_ptr;
   template <class _Up>
   friend class _LIBCPP_TEMPLATE_VIS shared_ptr;
+
+  friend struct __tombstone_traits<weak_ptr<_Tp> >;
 };
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 weak_ptr(shared_ptr<_Tp>) -> weak_ptr<_Tp>;
-#endif
+
+template <class _Tp>
+struct __tombstone_traits<weak_ptr<_Tp>> {
+  static constexpr auto __disengaged_value_ =
+      __tombstone_traits_assume_aligned_pointer<__shared_weak_count*>::__disengaged_value_;
+  static constexpr size_t __is_disengaged_offset_ =
+      __builtin_offsetof(weak_ptr<_Tp>, __cntrl_) +
+      __tombstone_traits_assume_aligned_pointer<__shared_weak_count*>::__is_disengaged_offset_;
+};
+#endif // _LIBCPP_STD_VER >= 17
 
 template <class _Tp>
 inline _LIBCPP_CONSTEXPR weak_ptr<_Tp>::weak_ptr() _NOEXCEPT : __ptr_(nullptr), __cntrl_(nullptr) {}