[libc++] Add __small_buffer

This is an implementation detail for `move_only_function` (and potentially other type-erasing classes).

Reviewed By: #libc, ldionne

Spies: Mordante, ldionne, EricWF, libcxx-commits

Differential Revision: https://reviews.llvm.org/D140259

Author: philnik777

libcxx/include/CMakeLists.txt

@@ -854,6 +854,7 @@ set(files
   __utility/piecewise_construct.h
   __utility/priority_tag.h
   __utility/rel_ops.h
+  __utility/small_buffer.h
   __utility/swap.h
   __utility/to_underlying.h
   __utility/unreachable.h

libcxx/include/__utility/small_buffer.h

@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_SMALL_BUFFER_H
+#define _LIBCPP___UTILITY_SMALL_BUFFER_H
+
+#include <__config>
+#include <__memory/construct_at.h>
+#include <__type_traits/decay.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/is_trivially_move_constructible.h>
+#include <__utility/exception_guard.h>
+#include <__utility/forward.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+// __small_buffer is a helper class to perform the well known SBO (small buffer optimization). It is mainly useful to
+// allow type-erasing classes like move_only_function to store small objects in a local buffer without requiring an
+// allocation.
+//
+// This small buffer class only allows storing  trivially relocatable objects inside the local storage to allow
+// __small_buffer to be trivially relocatable itself. Since the buffer doesn't know what's stored inside it, the user
+// has to manage the object's lifetime, in particular the destruction of the object.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _BufferSize, size_t _BufferAlignment>
+  requires(_BufferSize > 0 && _BufferAlignment > 0)
+class __small_buffer {
+public:
+  template <class _Tp, class _Decayed = decay_t<_Tp>>
+  static constexpr bool __fits_in_buffer =
+      is_trivially_move_constructible_v<_Decayed> && is_trivially_destructible_v<_Decayed> &&
+      sizeof(_Decayed) <= _BufferSize && alignof(_Decayed) <= _BufferAlignment;
+
+  _LIBCPP_HIDE_FROM_ABI __small_buffer()           = default;
+  __small_buffer(const __small_buffer&)            = delete;
+  __small_buffer& operator=(const __small_buffer&) = delete;
+  _LIBCPP_HIDE_FROM_ABI ~__small_buffer()          = default;
+
+  // Relocates the buffer - __delete() should never be called on a moved-from __small_buffer
+  _LIBCPP_HIDE_FROM_ABI __small_buffer(__small_buffer&&)            = default;
+  _LIBCPP_HIDE_FROM_ABI __small_buffer& operator=(__small_buffer&&) = default;
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI _Stored* __get() {
+    if constexpr (__fits_in_buffer<_Stored>)
+      return std::launder(reinterpret_cast<_Stored*>(__buffer_));
+    else
+      return *std::launder(reinterpret_cast<_Stored**>(__buffer_));
+  }
+
+  template <class _Stored>
+  _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE _LIBCPP_HIDE_FROM_ABI _Stored* __alloc() {
+    if constexpr (__fits_in_buffer<_Stored>) {
+      return std::launder(reinterpret_cast<_Stored*>(__buffer_));
+    } else {
+      byte* __allocation = static_cast<byte*>(::operator new[](sizeof(_Stored), align_val_t{alignof(_Stored)}));
+      std::construct_at(reinterpret_cast<byte**>(__buffer_), __allocation);
+      return std::launder(reinterpret_cast<_Stored*>(__allocation));
+    }
+  }
+
+  template <class _Stored>
+  _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE _LIBCPP_HIDE_FROM_ABI void __dealloc() noexcept {
+    if constexpr (!__fits_in_buffer<_Stored>)
+      ::operator delete[](*reinterpret_cast<void**>(__buffer_), sizeof(_Stored), align_val_t{alignof(_Stored)});
+  }
+
+  template <class _Stored, class... _Args>
+  _LIBCPP_AVAILABILITY_SIZED_NEW_DELETE _LIBCPP_HIDE_FROM_ABI void __construct(_Args&&... __args) {
+    _Stored* __buffer = __alloc<_Stored>();
+    auto __guard      = std::__make_exception_guard([&] { __dealloc<_Stored>(); });
+    std::construct_at(__buffer, std::forward<_Args>(__args)...);
+    __guard.__complete();
+  }
+
+private:
+  alignas(_BufferAlignment) byte __buffer_[_BufferSize];
+};
+
+#  undef _LIBCPP_SMALL_BUFFER_TRIVIAL_ABI
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___UTILITY_SMALL_BUFFER_H

libcxx/include/module.modulemap.in

@@ -2079,6 +2079,7 @@ module std_private_utility_pair_fwd               [system] { header "__fwd/pair.
 module std_private_utility_piecewise_construct    [system] { header "__utility/piecewise_construct.h" }
 module std_private_utility_priority_tag           [system] { header "__utility/priority_tag.h" }
 module std_private_utility_rel_ops                [system] { header "__utility/rel_ops.h" }
+module std_private_utility_small_buffer           [system] { header "__utility/small_buffer.h" }
 module std_private_utility_swap                   [system] {
   header "__utility/swap.h"
   export std_private_type_traits_is_swappable

libcxx/test/libcxx/utilities/utility/small_buffer.pass.cpp

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: availability-aligned_allocation-missing
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include "test_macros.h"
+
+TEST_DIAGNOSTIC_PUSH
+TEST_CLANG_DIAGNOSTIC_IGNORED("-Wprivate-header")
+#include <__utility/small_buffer.h>
+TEST_DIAGNOSTIC_POP
+
+#include <cassert>
+#include <memory>
+#include <utility>
+
+struct NotTriviallyRelocatable {
+  char c_;
+
+  NotTriviallyRelocatable(char c) : c_(c) {}
+  ~NotTriviallyRelocatable() {}
+};
+
+struct alignas(16) Overaligned {
+  int i;
+};
+
+int main(int, char**) {
+  using BufferT = std::__small_buffer<8, 8>;
+  static_assert(sizeof(BufferT) == 8);
+  static_assert(alignof(BufferT) == 8);
+  static_assert(BufferT::__fits_in_buffer<int>);
+  static_assert(!BufferT::__fits_in_buffer<Overaligned>);
+  static_assert(!BufferT::__fits_in_buffer<NotTriviallyRelocatable>);
+
+  BufferT buf;
+
+  { // construct/destroy in the same place
+    buf.__construct<int>(3);
+    assert(*buf.__get<int>() == 3);
+    std::destroy_at(buf.__get<int>());
+    buf.__dealloc<int>();
+
+    buf.__construct<NotTriviallyRelocatable>(3);
+    assert(buf.__get<NotTriviallyRelocatable>()->c_ == 3);
+    std::destroy_at(buf.__get<NotTriviallyRelocatable>());
+    buf.__dealloc<NotTriviallyRelocatable>();
+  }
+
+  { // Move the buffer around
+    buf.__construct<int>(3);
+    assert(*buf.__get<int>() == 3);
+    auto buf2 = std::move(buf);
+    assert(*buf2.__get<int>() == 3);
+    std::destroy_at(buf2.__get<int>());
+    buf2.__dealloc<int>();
+
+    buf.__construct<NotTriviallyRelocatable>(3);
+    assert(buf.__get<NotTriviallyRelocatable>()->c_ == 3);
+    auto buf3 = std::move(buf);
+    assert(buf3.__get<NotTriviallyRelocatable>()->c_ == 3);
+    std::destroy_at(buf3.__get<NotTriviallyRelocatable>());
+    buf3.__dealloc<NotTriviallyRelocatable>();
+  }
+
+  return 0;
+}