Merge pull request #42143 from compnerd/malign

runtime: allow over-aligned types in the runtime

Author: compnerd

include/swift/Runtime/Atomic.h

@@ -18,6 +18,8 @@
 #define SWIFT_RUNTIME_ATOMIC_H
 
 #include "swift/Runtime/Config.h"
+#include "swift/Runtime/Heap.h"
+
 #include <assert.h>
 #include <atomic>
 #if defined(_WIN64)
@@ -44,13 +46,72 @@
 namespace swift {
 namespace impl {
 
+// FIXME: why can we not use the definitions from Heap.h?  It seems that we
+// would fail to collapse the structure down in that case and end up with size
+// differences.
+template <std::size_t Alignment_>
+struct requires_aligned_alloc {
+#if defined(__cpp_aligned_new)
+  // If we have C++17 or newer we can use the alignment aware allocation
+  // implicitly.
+  static constexpr const bool value = false;
+#else
+#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
+  static constexpr const bool value =
+      Alignment_ > std::alignment_of<std::max_align_t>::value &&
+      Alignment_ > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
+#else
+  static constexpr const bool value =
+      Alignment_ > std::alignment_of<std::max_align_t>::value;
+#endif
+#endif
+};
+
+template <std::size_t Alignment_,
+          bool = requires_aligned_alloc<Alignment_>::value>
+struct aligned_alloc;
+
+template <std::size_t Alignment_>
+struct aligned_alloc<Alignment_, false> {};
+
+template <std::size_t Alignment_>
+struct aligned_alloc<Alignment_, true> {
+  [[nodiscard]] void *operator new(std::size_t size) noexcept {
+#if defined(_WIN32)
+    return _aligned_malloc(size, Alignment_);
+#else
+    static_assert(Alignment_ >= sizeof(void *),
+                  "posix_memalign requires minimal alignment of pointer");
+    void *ptr = nullptr;
+    (void)posix_memalign(&ptr, Alignment_, size);
+    return ptr;
+#endif
+  }
+
+  void operator delete(void *ptr) noexcept {
+#if defined(_WIN32)
+    _aligned_free(ptr);
+#else
+    free(ptr);
+#endif
+  }
+
+#if defined(_WIN32)
+  // FIXME: why is this even needed?  This is not permitted as per the C++
+  // standrd new.delete.placement (§17.6.3.4).
+  [[nodiscard]] void *operator new(std::size_t size, void *where) noexcept {
+    return ::operator new(size, where);
+  }
+#endif
+};
+
 /// The default implementation for swift::atomic<T>, which just wraps
 /// std::atomic with minor differences.
 ///
 /// TODO: should we make this use non-atomic operations when the runtime
 /// is single-threaded?
 template <class Value, size_t Size = sizeof(Value)>
-class alignas(Size) atomic_impl {
+class alignas(Size) atomic_impl : public aligned_alloc<Size> {
   std::atomic<Value> value;
 public:
   constexpr atomic_impl(Value value) : value(value) {}

include/swift/Runtime/AtomicWaitQueue.h

@@ -19,6 +19,7 @@
 #ifndef SWIFT_RUNTIME_ATOMICWAITQUEUE_H
 #define SWIFT_RUNTIME_ATOMICWAITQUEUE_H
 
+#include "swift/Runtime/Heap.h"
 #include "swift/Runtime/Mutex.h"
 #include <assert.h>
 
@@ -425,7 +426,8 @@ class AtomicWaitQueue {
     template <class... Args>
     static Impl *createNewQueue(Args &&...args) {
 #if !defined(__cpp_aligned_new)
-      static_assert(std::alignment_of<Impl>::value <= __STDCPP_DEFAULT_NEW_ALIGNMENT__,
+      static_assert(!swift::requires_aligned_alloc<std::alignment_of<Impl>::value>::value ||
+                     is_aligned_alloc_aware<Impl>::value,
                     "type is over-aligned for non-alignment aware operator new");
 #endif
       auto queue = new Impl(std::forward<Args>(args)...);

include/swift/Runtime/Heap.h

@@ -17,4 +17,86 @@
 #ifndef SWIFT_RUNTIME_HEAP_H
 #define SWIFT_RUNTIME_HEAP_H
 
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <type_traits>
+
+#if defined(_WIN32)
+#include <malloc.h>
+#endif
+
+namespace swift {
+namespace {
+// This is C++17 and newer, so we simply re-define it.  Since the codebase is
+// C++14, asume that DR1558 is accounted for and that unused parameters in alias
+// templates are guaranteed to ensure SFINAE and are not ignored.
+template <typename ...>
+using void_t = void;
+
+template <typename T, typename = void>
+struct is_aligned_alloc_aware : std::false_type {};
+
+template <typename T>
+struct is_aligned_alloc_aware<T, void_t<decltype(T::operator new(0))>>
+    : std::true_type {};
+}
+
+template <std::size_t Alignment_>
+struct requires_aligned_alloc {
+#if defined(__cpp_aligned_new)
+  // If we have C++17 or newer we can use the alignment aware allocation
+  // implicitly.
+  static constexpr const bool value = false;
+#else
+#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
+  static constexpr const bool value =
+      Alignment_ > std::alignment_of<std::max_align_t>::value &&
+      Alignment_ > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
+#else
+  static constexpr const bool value =
+      Alignment_ > std::alignment_of<std::max_align_t>::value;
+#endif
+#endif
+};
+
+template <std::size_t Alignment_,
+          bool = requires_aligned_alloc<Alignment_>::value>
+struct aligned_alloc;
+
+template <std::size_t Alignment_>
+struct aligned_alloc<Alignment_, false> {};
+
+template <std::size_t Alignment_>
+struct aligned_alloc<Alignment_, true> {
+  [[nodiscard]] void *operator new(std::size_t size) noexcept {
+#if defined(_WIN32)
+    return _aligned_malloc(size, Alignment_);
+#else
+    static_assert(Alignment_ >= sizeof(void *),
+                  "posix_memalign requires minimal alignment of pointer");
+    void *ptr = nullptr;
+    (void)posix_memalign(&ptr, Alignment_, size);
+    return ptr;
+#endif
+  }
+
+  void operator delete(void *ptr) noexcept {
+#if defined(_WIN32)
+    _aligned_free(ptr);
+#else
+    free(ptr);
+#endif
+  }
+
+#if defined(_WIN32)
+  // FIXME: why is this even needed?  This is not permitted as per the C++
+  // standrd new.delete.placement (§17.6.3.4).
+  [[nodiscard]] void *operator new(std::size_t size, void *where) noexcept {
+    return ::operator new(size, where);
+  }
+#endif
+};
+}
+
 #endif // SWIFT_RUNTIME_HEAP_H

stdlib/public/Concurrency/Actor.cpp

@@ -616,7 +616,8 @@ class JobRef {
 ///     achieved through careful arrangement of the storage for this in the
 ///     DefaultActorImpl. The additional alignment requirements are
 ///     enforced by static asserts below.
-class alignas(sizeof(void *) * 2) ActiveActorStatus {
+class alignas(2 * sizeof(void *)) ActiveActorStatus
+    : public swift::aligned_alloc<2 * sizeof(void *)> {
 #if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_4_BYTES
   uint32_t Flags;
   dispatch_lock_t DrainLock;

stdlib/public/Concurrency/AsyncLet.cpp

@@ -21,6 +21,7 @@
 #include "swift/ABI/Task.h"
 #include "swift/ABI/TaskOptions.h"
 #include "swift/Runtime/Mutex.h"
+#include "swift/Runtime/Heap.h"
 #include "swift/Runtime/HeapObject.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "TaskPrivate.h"
@@ -33,7 +34,9 @@
 using namespace swift;
 
 namespace {
-class alignas(Alignment_AsyncLet) AsyncLetImpl: public ChildTaskStatusRecord {
+class alignas(Alignment_AsyncLet) AsyncLetImpl
+    : public swift::aligned_alloc<Alignment_AsyncLet>,
+      public ChildTaskStatusRecord {
 public:
   // This is where we could define a Status or other types important for async-let
 

stdlib/public/Concurrency/TaskPrivate.h

@@ -26,6 +26,7 @@
 #include "swift/Runtime/DispatchShims.h"
 #include "swift/Runtime/Error.h"
 #include "swift/Runtime/Exclusivity.h"
+#include "swift/Runtime/Heap.h"
 #include "swift/Runtime/HeapObject.h"
 #include <atomic>
 
@@ -524,6 +525,12 @@ class alignas(2 * sizeof(void*)) ActiveTaskStatus {
 #endif
 static_assert(sizeof(ActiveTaskStatus) == ACTIVE_TASK_STATUS_SIZE,
   "ActiveTaskStatus is of incorrect size");
+#if !defined(_WIN64)
+static_assert(sizeof(swift::atomic<ActiveTaskStatus>) == sizeof(std::atomic<ActiveTaskStatus>),
+              "swift::atomic pads std::atomic, memory aliasing invariants violated");
+#endif
+static_assert(sizeof(swift::atomic<ActiveTaskStatus>) == sizeof(ActiveTaskStatus),
+              "swift::atomic pads ActiveTaskStatus, memory aliasing invariants violated");
 
 /// The size of an allocator slab. We want the full allocation to fit into a
 /// 1024-byte malloc quantum. We subtract off the slab header size, plus a
@@ -545,7 +552,7 @@ struct AsyncTask::PrivateStorage {
 
   /// Storage for the ActiveTaskStatus. See doc for ActiveTaskStatus for size
   /// and alignment requirements.
-  alignas(2 * sizeof(void *)) char StatusStorage[sizeof(ActiveTaskStatus)];
+  alignas(alignof(ActiveTaskStatus)) char StatusStorage[sizeof(ActiveTaskStatus)];
 
   /// The allocator for the task stack.
   /// Currently 2 words + 8 bytes.

stdlib/public/SwiftShims/RefCount.h

@@ -37,6 +37,7 @@ typedef InlineRefCountsPlaceholder InlineRefCounts;
 #include "swift/Runtime/Atomic.h"
 #include "swift/Runtime/Config.h"
 #include "swift/Runtime/Debug.h"
+#include "swift/Runtime/Heap.h"
 
 
 /*
@@ -625,8 +626,9 @@ class RefCountBitsT {
 
 typedef RefCountBitsT<RefCountIsInline> InlineRefCountBits;
 
-class alignas(sizeof(void*) * 2) SideTableRefCountBits : public RefCountBitsT<RefCountNotInline>
-{
+class alignas(2 * sizeof(void*)) SideTableRefCountBits
+    : public swift::aligned_alloc<2 * sizeof(void *)>,
+      public RefCountBitsT<RefCountNotInline> {
   uint32_t weakBits;
 
   public:

stdlib/public/runtime/KnownMetadata.cpp

@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/Runtime/Metadata.h"
+#include "swift/Runtime/Heap.h"
 #include "swift/Runtime/HeapObject.h"
 #include "swift/Runtime/Numeric.h"
 #include "MetadataImpl.h"
@@ -40,19 +41,19 @@ OpaqueValue *swift::swift_copyPOD(OpaqueValue *dest, OpaqueValue *src,
 namespace {
   // A type sized and aligned the way Swift wants Int128 (and Float80/Float128)
   // to be sized and aligned.
-  struct alignas(16) int128_like {
+  struct alignas(16) int128_like : swift::aligned_alloc<16> {
     char data[16];
   };
 
   static_assert(MaximumAlignment == 16, "max alignment was hardcoded");
-  struct alignas(16) int256_like {
+  struct alignas(16) int256_like : swift::aligned_alloc<16> {
     char data[32];
   };
-  struct alignas(16) int512_like {
+  struct alignas(16) int512_like : swift::aligned_alloc<16> {
     char data[64];
   };
 
-  struct alignas(16) float80_like {
+  struct alignas(16) float80_like : swift::aligned_alloc<16> {
     char data[10];
   };
 } // end anonymous namespace
@@ -89,7 +90,8 @@ namespace ctypes {
     // Types that are defined in the _Concurrency library
 
     // Default actor storage type.
-    struct alignas(2 * alignof(void*)) BD {
+    struct alignas(2 * alignof(void*)) BD
+        : swift::aligned_alloc<2 * alignof(void*)> {
       void *storage[NumWords_DefaultActor];
     };
 

stdlib/public/runtime/Metadata.cpp

@@ -24,6 +24,7 @@
 #include "swift/Runtime/Casting.h"
 #include "swift/Runtime/EnvironmentVariables.h"
 #include "swift/Runtime/ExistentialContainer.h"
+#include "swift/Runtime/Heap.h"
 #include "swift/Runtime/HeapObject.h"
 #include "swift/Runtime/Mutex.h"
 #include "swift/Runtime/Once.h"
@@ -6153,10 +6154,16 @@ void swift::blockOnMetadataDependency(MetadataDependency root,
 #if !SWIFT_STDLIB_PASSTHROUGH_METADATA_ALLOCATOR
 
 namespace {
-  struct alignas(sizeof(uintptr_t) * 2) PoolRange {
+  struct alignas(2 * sizeof(uintptr_t)) PoolRange
+      : swift::aligned_alloc<2 * sizeof(uintptr_t)> {
     static constexpr uintptr_t PageSize = 16 * 1024;
     static constexpr uintptr_t MaxPoolAllocationSize = PageSize / 2;
 
+    constexpr PoolRange(char *Begin, size_t Remaining)
+        : Begin(Begin), Remaining(Remaining) {}
+
+    PoolRange() : Begin(nullptr), Remaining(0) {}
+
     /// The start of the allocation.
     char *Begin;
 

stdlib/public/runtime/MetadataCache.h

@@ -15,6 +15,7 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "swift/Runtime/Concurrent.h"
+#include "swift/Runtime/Heap.h"
 #include "swift/Runtime/Metadata.h"
 #include "swift/Runtime/Mutex.h"
 #include "swift/Runtime/AtomicWaitQueue.h"
@@ -623,8 +624,9 @@ const size_t PrivateMetadataTrackingAlignment = 16;
 
 /// The wait queue object that we create for metadata that are
 /// being actively initialized right now.
-struct alignas(PrivateMetadataTrackingAlignment) MetadataWaitQueue :
-  public AtomicWaitQueue<MetadataWaitQueue, ConcurrencyControl::LockType> {
+struct alignas(PrivateMetadataTrackingAlignment) MetadataWaitQueue
+    : swift::aligned_alloc<PrivateMetadataTrackingAlignment>,
+      public AtomicWaitQueue<MetadataWaitQueue, ConcurrencyControl::LockType> {
 
   /// A pointer to the completion context being used to complete this
   /// metadata.  This is only actually filled in if:
@@ -683,7 +685,8 @@ struct alignas(PrivateMetadataTrackingAlignment) MetadataWaitQueue :
 
 /// A record used to store information about an attempt to
 /// complete a metadata when there's no active worker thread.
-struct alignas(PrivateMetadataTrackingAlignment) SuspendedMetadataCompletion {
+struct alignas(PrivateMetadataTrackingAlignment) SuspendedMetadataCompletion
+    : swift::aligned_alloc<PrivateMetadataTrackingAlignment> {
   MetadataDependency BlockingDependency;
   std::unique_ptr<PrivateMetadataCompletionContext> PersistentContext;