[libc++][WIP] Add relocation facilities that P2687 could provide

Author: ldionne

libcxx/include/CMakeLists.txt

@@ -551,18 +551,21 @@ set(files
   __memory/construct_at.h
   __memory/destruct_n.h
   __memory/inout_ptr.h
+  __memory/is_trivially_allocator_relocatable.h
   __memory/noexcept_move_assign_container.h
   __memory/out_ptr.h
   __memory/pointer_traits.h
   __memory/ranges_construct_at.h
   __memory/ranges_uninitialized_algorithms.h
   __memory/raw_storage_iterator.h
+  __memory/relocate_at.h
   __memory/shared_count.h
   __memory/shared_ptr.h
   __memory/swap_allocator.h
   __memory/temp_value.h
   __memory/temporary_buffer.h
   __memory/uninitialized_algorithms.h
+  __memory/uninitialized_relocate.h
   __memory/unique_ptr.h
   __memory/unique_temporary_buffer.h
   __memory/uses_allocator.h

libcxx/include/__memory/is_trivially_allocator_relocatable.h

@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_IS_TRIVIALLY_ALLOCATOR_RELOCATABLE_H
+#define _LIBCPP___MEMORY_IS_TRIVIALLY_ALLOCATOR_RELOCATABLE_H
+
+#include <__config>
+#include <__memory/allocator_traits.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__type_traits/negation.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// A type is trivially allocator relocatable if the allocator's move construction and destruction
+// don't do anything beyond calling the type's move constructor and destructor, and if the type
+// itself is trivially relocatable.
+
+template <class _Alloc, class _Type>
+struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
+
+template <class _Type>
+struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
+
+template <class _Alloc, class _Tp>
+struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
+
+template <class _Tp, class _Up>
+struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
+
+template <class _Alloc, class _Tp>
+struct __is_trivially_allocator_relocatable
+    : integral_constant<bool,
+                        __allocator_has_trivial_move_construct<_Alloc, _Tp>::value &&
+                            __allocator_has_trivial_destroy<_Alloc, _Tp>::value &&
+                            __libcpp_is_trivially_relocatable<_Tp>::value> {};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_IS_TRIVIALLY_ALLOCATOR_RELOCATABLE_H

libcxx/include/__memory/relocate_at.h

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RELOCATE_AT_H
+#define _LIBCPP___MEMORY_RELOCATE_AT_H
+
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__utility/move.h>
+#include <__utility/scope_guard.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+struct __destroy_object {
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 void operator()() const { std::__destroy_at(__obj_); }
+  _Tp* __obj_;
+};
+
+template <class _Tp, __enable_if_t<!__libcpp_is_trivially_relocatable<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __relocate_at(_Tp* __source, _Tp* __dest) {
+  __scope_guard<__destroy_object<_Tp> > __guard(__destroy_object<_Tp>{__source});
+  return std::__construct_at(__dest, std::move(*__source));
+}
+
+template <class _Tp, __enable_if_t<__libcpp_is_trivially_relocatable<_Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __relocate_at(_Tp* __source, _Tp* __dest) {
+  if (__libcpp_is_constant_evaluated()) {
+    std::__construct_at(__dest, std::move(*__source));
+  } else {
+    // Casting to void* to suppress clang complaining that this is technically UB.
+    __builtin_memcpy(static_cast<void*>(__dest), __source, sizeof(_Tp));
+  }
+  return __dest;
+}
+
+template <class _Alloc, class _Tp, __enable_if_t<!__is_trivially_allocator_relocatable<_Alloc, _Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp*
+__allocator_relocate_at(_Alloc& __alloc, _Tp* __source, _Tp* __dest) {
+  allocator_traits<_Alloc>::construct(__alloc, __dest, std::move(*__source));
+  allocator_traits<_Alloc>::destroy(__alloc, __source);
+  return __dest;
+}
+
+template <class _Alloc, class _Tp, __enable_if_t<__is_trivially_allocator_relocatable<_Alloc, _Tp>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __allocator_relocate_at(_Alloc&, _Tp* __source, _Tp* __dest) {
+  return std::__relocate_at(__source, __dest);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_RELOCATE_AT_H

libcxx/include/__memory/uninitialized_algorithms.h

@@ -20,6 +20,7 @@
 #include <__memory/addressof.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/construct_at.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/extent.h>
@@ -591,19 +592,7 @@ __uninitialized_allocator_copy(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1,
   return std::__rewrap_iter(__first2, __result);
 }
 
-template <class _Alloc, class _Type>
-struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {};
-
-template <class _Type>
-struct __allocator_has_trivial_move_construct<allocator<_Type>, _Type> : true_type {};
-
-template <class _Alloc, class _Tp>
-struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {};
-
-template <class _Tp, class _Up>
-struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
-
-// __uninitialized_allocator_relocate relocates the objects in [__first, __last) into __result.
+// __uninitialized_allocator_relocate_for_vector relocates the objects in [__first, __last) into __result.
 // Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
 // except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
 //
@@ -616,15 +605,13 @@ struct __allocator_has_trivial_destroy<allocator<_Tp>, _Up> : true_type {};
 // - is_copy_constructible<_ValueType>
 // - __libcpp_is_trivially_relocatable<_ValueType>
 template <class _Alloc, class _ContiguousIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate(
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate_for_vector(
     _Alloc& __alloc, _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) {
   static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
   using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
   static_assert(__is_cpp17_move_insertable<_Alloc>::value,
                 "The specified type does not meet the requirements of Cpp17MoveInsertable");
-  if (__libcpp_is_constant_evaluated() || !__libcpp_is_trivially_relocatable<_ValueType>::value ||
-      !__allocator_has_trivial_move_construct<_Alloc, _ValueType>::value ||
-      !__allocator_has_trivial_destroy<_Alloc, _ValueType>::value) {
+  if (__libcpp_is_constant_evaluated() || !__is_trivially_allocator_relocatable<_Alloc, _ValueType>::value) {
     auto __destruct_first = __result;
     auto __guard          = std::__make_exception_guard(
         _AllocatorDestroyRangeReverse<_Alloc, _ContiguousIterator>(__alloc, __destruct_first, __result));

libcxx/include/__memory/uninitialized_relocate.h

@@ -0,0 +1,94 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
+#define _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
+#include <__memory/pointer_traits.h>
+#include <__memory/relocate_at.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__type_traits/is_trivially_relocatable.h>
+#include <__utility/move.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __uninitialized_relocate relocates the objects in [__first, __last) into __result.
+//
+// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy,
+// except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy.
+//
+// This algorithm works even if part of the resulting range overlaps with [__first, __last), as long as __result itself
+// is not in [__first, last).
+//
+// This algorithm doesn't throw any exceptions. However, it requires the types in the range to be nothrow
+// move-constructible and the iterator operations not to throw any exceptions.
+//
+// Preconditions:
+//  - __result doesn't contain any objects and [__first, __last) contains objects
+//  - __result is not in [__first, __last)
+// Postconditions: __result contains the objects from [__first, __last) and
+//                 [__first, __last) doesn't contain any objects
+template <class _ContiguousIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ContiguousIterator __uninitialized_relocate(
+    _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) _NOEXCEPT {
+  using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
+  static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
+  static_assert(is_nothrow_move_constructible<_ValueType>::value, "");
+  if (!__libcpp_is_constant_evaluated() && __libcpp_is_trivially_relocatable<_ValueType>::value) {
+    auto const __n = __last - __first;
+    // Casting to void* to suppress clang complaining that this is technically UB.
+    __builtin_memmove(
+        static_cast<void*>(std::__to_address(__result)), std::__to_address(__first), sizeof(_ValueType) * __n);
+    return __result + __n;
+  } else {
+    while (__first != __last) {
+      std::__relocate_at(std::__to_address(__first), std::__to_address(__result));
+      ++__first;
+      ++__result;
+    }
+    return __result;
+  }
+}
+
+// Equivalent to __uninitialized_relocate, but uses the provided allocator's construct() and destroy() methods.
+template <class _Alloc, class _ContiguousIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ContiguousIterator __uninitialized_allocator_relocate(
+    _Alloc& __alloc, _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) _NOEXCEPT {
+  using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type;
+  static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, "");
+  static_assert(is_nothrow_move_constructible<_ValueType>::value, "");
+  if (!__libcpp_is_constant_evaluated() && __is_trivially_allocator_relocatable<_Alloc, _ValueType>::value) {
+    (void)__alloc; // ignore the allocator
+    return std::__uninitialized_relocate(std::move(__first), std::move(__last), std::move(__result));
+  } else {
+    while (__first != __last) {
+      std::__allocator_relocate_at(__alloc, std::__to_address(__first), std::__to_address(__result));
+      ++__first;
+      ++__result;
+    }
+    return __result;
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H

libcxx/include/__vector/vector.h

@@ -806,7 +806,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void
 vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
   __annotate_delete();
   auto __new_begin = __v.__begin_ - (__end_ - __begin_);
-  std::__uninitialized_allocator_relocate(
+  std::__uninitialized_allocator_relocate_for_vector(
       this->__alloc_, std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
   __v.__begin_ = __new_begin;
   __end_       = __begin_; // All the objects have been destroyed by relocating them.
@@ -829,13 +829,13 @@ vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, a
 
   // Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
   // in case something in [__begin_, __p) throws.
-  std::__uninitialized_allocator_relocate(
+  std::__uninitialized_allocator_relocate_for_vector(
       this->__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
   __v.__end_ += (__end_ - __p);
   __end_           = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
   auto __new_begin = __v.__begin_ - (__p - __begin_);
 
-  std::__uninitialized_allocator_relocate(
+  std::__uninitialized_allocator_relocate_for_vector(
       this->__alloc_, std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
   __v.__begin_ = __new_begin;
   __end_       = __begin_; // All the objects have been destroyed by relocating them.

libcxx/include/module.modulemap

@@ -1535,6 +1535,7 @@ module std [system] {
     module destruct_n                         { header "__memory/destruct_n.h" }
     module fwd                                { header "__fwd/memory.h" }
     module inout_ptr                          { header "__memory/inout_ptr.h" }
+    module is_trivially_allocator_relocatable { header "__memory/is_trivially_allocator_relocatable.h" }
     module noexcept_move_assign_container     { header "__memory/noexcept_move_assign_container.h" }
     module out_ptr                            { header "__memory/out_ptr.h" }
     module pointer_traits                     { header "__memory/pointer_traits.h" }
@@ -1544,6 +1545,7 @@ module std [system] {
       export std.algorithm.in_out_result
     }
     module raw_storage_iterator               { header "__memory/raw_storage_iterator.h" }
+    module relocate_at                        { header "__memory/relocate_at.h" }
     module shared_count                       { header "__memory/shared_count.h" }
     module shared_ptr                         { header "__memory/shared_ptr.h" }
     module swap_allocator                     { header "__memory/swap_allocator.h" }
@@ -1556,6 +1558,9 @@ module std [system] {
       header "__memory/uninitialized_algorithms.h"
       export std.utility.pair
     }
+    module uninitialized_relocate {
+      header "__memory/uninitialized_relocate.h"
+    }
     module unique_ptr {
       header "__memory/unique_ptr.h"
     }