[libc++] Mark more types as trivially relocatable

Author: philnik777

libcxx/docs/ReleaseNotes/19.rst

@@ -59,6 +59,7 @@ Improvements and New Features
 -----------------------------
 
 - The performance of growing ``std::vector`` has been improved for trivially relocatable types.
+- A lot of types are considered trivially relocatable now, including ``vector`` and ``string``.
 - The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector<bool>::iterator``\s,
   resulting in a performance increase of up to 1400x.
 - The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance

libcxx/include/__exception/exception_ptr.h

@@ -66,6 +66,9 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   friend _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT;
 
 public:
+  // exception_ptr is basically a COW string.
+  using __trivially_relocatable = exception_ptr;
+
   _LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
   _LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
 

libcxx/include/__expected/expected.h

@@ -31,6 +31,7 @@
 #include <__type_traits/is_swappable.h>
 #include <__type_traits/is_trivially_constructible.h>
 #include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/lazy.h>
 #include <__type_traits/negation.h>
@@ -463,6 +464,11 @@ class expected : private __expected_base<_Tp, _Err> {
   using error_type      = _Err;
   using unexpected_type = unexpected<_Err>;
 
+  using __trivially_relocatable =
+      __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value && __libcpp_is_trivially_relocatable<_Err>::value,
+                      expected,
+                      void>;
+
   template <class _Up>
   using rebind = expected<_Up, error_type>;
 

libcxx/include/__locale

@@ -49,6 +49,9 @@ _LIBCPP_HIDE_FROM_ABI const _Facet& use_facet(const locale&);
 
 class _LIBCPP_EXPORTED_FROM_ABI locale {
 public:
+  // locale is essentially a shared_ptr that doesn't support weak_ptrs and never got a move constructor.
+  using __trivially_relocatable = locale;
+
   // types:
   class _LIBCPP_EXPORTED_FROM_ABI facet;
   class _LIBCPP_EXPORTED_FROM_ABI id;

libcxx/include/__memory/shared_ptr.h

@@ -422,6 +422,10 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr {
   typedef _Tp element_type;
 #endif
 
+  // A shared_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
+  // any bookkeeping, so it's always trivially relocatable.
+  using __trivially_relocatable = shared_ptr;
+
 private:
   element_type* __ptr_;
   __shared_weak_count* __cntrl_;
@@ -1304,6 +1308,10 @@ class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
   typedef _Tp element_type;
 #endif
 
+  // A weak_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
+  // any bookkeeping, so it's always trivially relocatable.
+  using __trivially_relocatable = weak_ptr;
+
 private:
   element_type* __ptr_;
   __shared_weak_count* __cntrl_;

libcxx/include/__split_buffer

@@ -24,12 +24,14 @@
 #include <__memory/pointer_traits.h>
 #include <__memory/swap_allocator.h>
 #include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/conditional.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_nothrow_assignable.h>
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__type_traits/is_swappable.h>
 #include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/remove_reference.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
@@ -64,6 +66,15 @@ public:
   using iterator        = pointer;
   using const_iterator  = const_pointer;
 
+  // A __split_buffer contains the following members which may be trivially relocatable:
+  // - pointer: may be trivially relocatable, so it's checked
+  // - allocator_type: may be trivially relocatable, so it's checked
+  // __split_buffer doesn't have any self-references, so it's trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      __split_buffer,
+      void>;
+
   pointer __first_;
   pointer __begin_;
   pointer __end_;

libcxx/include/__utility/pair.h

@@ -34,6 +34,7 @@
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/nat.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/unwrap_ref.h>
@@ -71,6 +72,11 @@ struct _LIBCPP_TEMPLATE_VIS pair
   _T1 first;
   _T2 second;
 
+  using __trivially_relocatable =
+      __conditional_t<__libcpp_is_trivially_relocatable<_T1>::value && __libcpp_is_trivially_relocatable<_T2>::value,
+                      pair,
+                      void>;
+
   _LIBCPP_HIDE_FROM_ABI pair(pair const&) = default;
   _LIBCPP_HIDE_FROM_ABI pair(pair&&)      = default;
 

libcxx/include/array

@@ -130,6 +130,7 @@ template <size_t I, class T, size_t N> const T&& get(const array<T, N>&&) noexce
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/remove_cv.h>
 #include <__utility/empty.h>
 #include <__utility/integer_sequence.h>
@@ -166,6 +167,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, size_t _Size>
 struct _LIBCPP_TEMPLATE_VIS array {
+  using __trivially_relocatable = __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, array, void>;
+
   // types:
   using __self          = array;
   using value_type      = _Tp;

libcxx/include/deque

@@ -477,6 +477,16 @@ public:
   using reverse_iterator       = std::reverse_iterator<iterator>;
   using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
+  // A deque contains the following members which may be trivially relocatable:
+  // - __map: is a `__split_buffer`, see `__split_buffer` for more information on when it is trivially relocatable
+  // - size_type: is always trivially relocatable, since it is required to be an integral type
+  // - allocator_type: may not be trivially relocatable, so it's checked
+  // None of these are referencing the `deque` itself, so if all of them are trivially relocatable, `deque` is too.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<__map>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      deque,
+      void>;
+
   static_assert(is_same<allocator_type, __rebind_alloc<__alloc_traits, value_type> >::value,
                 "[allocator.requirements] states that rebinding an allocator to the same type should result in the "
                 "original allocator");

libcxx/include/optional

@@ -209,6 +209,7 @@ namespace std {
 #include <__type_traits/is_trivially_assignable.h>
 #include <__type_traits/is_trivially_constructible.h>
 #include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/negation.h>
 #include <__type_traits/remove_const.h>
 #include <__type_traits/remove_cvref.h>
@@ -580,6 +581,8 @@ class _LIBCPP_DECLSPEC_EMPTY_BASES optional
 public:
   using value_type = _Tp;
 
+  using __trivially_relocatable = conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, optional, void>;
+
 private:
   // Disable the reference extension using this static assert.
   static_assert(!is_same_v<__remove_cvref_t<value_type>, in_place_t>,

libcxx/include/tuple

@@ -240,6 +240,7 @@ template <class... Types>
 #include <__type_traits/is_reference.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_swappable.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/lazy.h>
 #include <__type_traits/maybe_const.h>
 #include <__type_traits/nat.h>
@@ -537,6 +538,8 @@ class _LIBCPP_TEMPLATE_VIS tuple {
   get(const tuple<_Up...>&&) _NOEXCEPT;
 
 public:
+  using __trivially_relocatable = __conditional_t<_And<__libcpp_is_trivially_relocatable<_Tp>...>::value, tuple, void>;
+
   // [tuple.cnstr]
 
   // tuple() constructors (including allocator_arg_t variants)

libcxx/include/variant

@@ -241,6 +241,7 @@ namespace std {
 #include <__type_traits/is_trivially_assignable.h>
 #include <__type_traits/is_trivially_constructible.h>
 #include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/remove_const.h>
 #include <__type_traits/remove_cvref.h>
@@ -1180,6 +1181,9 @@ class _LIBCPP_TEMPLATE_VIS _LIBCPP_DECLSPEC_EMPTY_BASES variant
   using __first_type = variant_alternative_t<0, variant>;
 
 public:
+  using __trivially_relocatable =
+      conditional_t<_And<__libcpp_is_trivially_relocatable<_Types>...>::value, variant, void>;
+
   template <bool _Dummy                                                                               = true,
             enable_if_t<__dependent_type<is_default_constructible<__first_type>, _Dummy>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI constexpr variant() noexcept(is_nothrow_default_constructible_v<__first_type>)

libcxx/include/vector

@@ -406,6 +406,15 @@ public:
   typedef std::reverse_iterator<iterator> reverse_iterator;
   typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
+  // A vector containers the following members which may be trivially relocatable:
+  // - pointer: may be trivially relocatable, so it's checked
+  // - allocator_type: may be trivially relocatable, so it's checked
+  // vector doesn't contain any self-references, so it's trivially relocatable if its members are.
+  using __trivially_relocatable = __conditional_t<
+      __libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
+      vector,
+      void>;
+
   static_assert((is_same<typename allocator_type::value_type, value_type>::value),
                 "Allocator::value_type must be same type as value_type");
 

libcxx/test/libcxx/transitive_includes/cxx03.csv

@@ -906,6 +906,7 @@ thread type_traits
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple exception
 tuple iosfwd
 tuple new

libcxx/test/libcxx/transitive_includes/cxx11.csv

@@ -913,6 +913,7 @@ thread type_traits
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple exception
 tuple iosfwd
 tuple new

libcxx/test/libcxx/transitive_includes/cxx14.csv

@@ -916,6 +916,7 @@ thread type_traits
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple exception
 tuple iosfwd
 tuple new

libcxx/test/libcxx/transitive_includes/cxx17.csv

@@ -916,6 +916,7 @@ thread type_traits
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple exception
 tuple iosfwd
 tuple new

libcxx/test/libcxx/transitive_includes/cxx20.csv

@@ -926,6 +926,7 @@ thread type_traits
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple exception
 tuple iosfwd
 tuple new

libcxx/test/libcxx/transitive_includes/cxx23.csv

@@ -161,6 +161,7 @@ exception typeinfo
 exception version
 execution version
 expected cstddef
+expected cstdint
 expected initializer_list
 expected new
 expected version
@@ -494,6 +495,7 @@ regex typeinfo
 regex vector
 regex version
 scoped_allocator cstddef
+scoped_allocator cstdint
 scoped_allocator limits
 scoped_allocator new
 scoped_allocator tuple
@@ -650,6 +652,7 @@ thread typeinfo
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple version
 type_traits cstddef
 type_traits cstdint
@@ -684,6 +687,7 @@ unordered_set tuple
 unordered_set version
 utility compare
 utility cstddef
+utility cstdint
 utility initializer_list
 utility limits
 utility version

libcxx/test/libcxx/transitive_includes/cxx26.csv

@@ -161,6 +161,7 @@ exception typeinfo
 exception version
 execution version
 expected cstddef
+expected cstdint
 expected initializer_list
 expected new
 expected version
@@ -517,6 +518,7 @@ regex typeinfo
 regex vector
 regex version
 scoped_allocator cstddef
+scoped_allocator cstdint
 scoped_allocator limits
 scoped_allocator new
 scoped_allocator tuple
@@ -673,6 +675,7 @@ thread typeinfo
 thread version
 tuple compare
 tuple cstddef
+tuple cstdint
 tuple version
 type_traits cstddef
 type_traits cstdint
@@ -707,6 +710,7 @@ unordered_set tuple
 unordered_set version
 utility compare
 utility cstddef
+utility cstdint
 utility initializer_list
 utility limits
 utility version