Merge pull request #16799 from mikeash/runtime-conformance-scanning-lockless-array-4.2

[Runtime] 4.2 - Change protocol conformance scanning to use a concurrent array rather than a locked vector.

Author: mikeash

include/swift/Runtime/Atomic.h

@@ -0,0 +1,31 @@
+//===--- Atomic.h - Utilities for atomic operations. ------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for atomic operations, to use with std::atomic.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_RUNTIME_ATOMIC_H
+#define SWIFT_RUNTIME_ATOMIC_H
+
+// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require
+// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed'
+// is formally UB by C++11 language rules, we should be OK because neither
+// the processor model nor the optimizer can realistically reorder our uses
+// of 'consume'.
+#if __arm64__ || __arm__
+#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed)
+#else
+#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume)
+#endif
+
+#endif

include/swift/Runtime/Concurrent.h

@@ -12,10 +12,14 @@
 #ifndef SWIFT_RUNTIME_CONCURRENTUTILS_H
 #define SWIFT_RUNTIME_CONCURRENTUTILS_H
 #include <iterator>
+#include <algorithm>
 #include <atomic>
 #include <functional>
 #include <stdint.h>
 #include "llvm/Support/Allocator.h"
+#include "Atomic.h"
+#include "Debug.h"
+#include "Mutex.h"
 
 #if defined(__FreeBSD__) || defined(__CYGWIN__) || defined(__HAIKU__)
 #include <stdio.h>
@@ -406,6 +410,136 @@ class ConcurrentMap
   }
 };
 
+
+/// An append-only array that can be read without taking locks. Writes
+/// are still locked and serialized, but only with respect to other
+/// writes.
+template <class ElemTy> struct ConcurrentReadableArray {
+private:
+  /// The struct used for the array's storage. The `Elem` member is
+  /// considered to be the first element of a variable-length array,
+  /// whose size is determined by the allocation. The `Capacity` member
+  /// from `ConcurrentReadableArray` indicates how large it can be.
+  struct Storage {
+    std::atomic<size_t> Count;
+    typename std::aligned_storage<sizeof(ElemTy), alignof(ElemTy)>::type Elem;
+
+    static Storage *allocate(size_t capacity) {
+      auto size = sizeof(Storage) + (capacity - 1) * sizeof(Storage().Elem);
+      auto *ptr = reinterpret_cast<Storage *>(malloc(size));
+      if (!ptr) swift::crash("Could not allocate memory.");
+      ptr->Count.store(0, std::memory_order_relaxed);
+      return ptr;
+    }
+
+    void deallocate() {
+      for (size_t i = 0; i < Count; i++) {
+        data()[i].~ElemTy();
+      }
+      free(this);
+    }
+
+    ElemTy *data() {
+      return reinterpret_cast<ElemTy *>(&Elem);
+    }
+  };
+  
+  size_t Capacity;
+  std::atomic<size_t> ReaderCount;
+  std::atomic<Storage *> Elements;
+  Mutex WriterLock;
+  std::vector<Storage *> FreeList;
+  
+  void incrementReaders() {
+    ReaderCount.fetch_add(1, std::memory_order_acquire);
+  }
+  
+  void decrementReaders() {
+    ReaderCount.fetch_sub(1, std::memory_order_release);
+  }
+  
+  void deallocateFreeList() {
+    for (Storage *storage : FreeList)
+      storage->deallocate();
+    FreeList.clear();
+    FreeList.shrink_to_fit();
+  }
+  
+public:
+  struct Snapshot {
+    ConcurrentReadableArray *Array;
+    const ElemTy *Start;
+    size_t Count;
+    
+    Snapshot(ConcurrentReadableArray *array, const ElemTy *start, size_t count)
+      : Array(array), Start(start), Count(count) {}
+    
+    Snapshot(const Snapshot &other)
+      : Array(other.Array), Start(other.Start), Count(other.Count) {
+      Array->incrementReaders();
+    }
+    
+    ~Snapshot() {
+      Array->decrementReaders();
+    }
+    
+    const ElemTy *begin() { return Start; }
+    const ElemTy *end() { return Start + Count; }
+    size_t count() { return Count; }
+  };
+  
+  // This type cannot be safely copied, moved, or deleted.
+  ConcurrentReadableArray(const ConcurrentReadableArray &) = delete;
+  ConcurrentReadableArray(ConcurrentReadableArray &&) = delete;
+  ConcurrentReadableArray &operator=(const ConcurrentReadableArray &) = delete;
+  
+  ConcurrentReadableArray() : Capacity(0), ReaderCount(0), Elements(nullptr) {}
+  
+  ~ConcurrentReadableArray() {
+    assert(ReaderCount.load(std::memory_order_acquire) == 0 &&
+           "deallocating ConcurrentReadableArray with outstanding snapshots");
+    deallocateFreeList();
+  }
+  
+  void push_back(const ElemTy &elem) {
+    ScopedLock guard(WriterLock);
+    
+    auto *storage = Elements.load(std::memory_order_relaxed);
+    auto count = storage ? storage->Count.load(std::memory_order_relaxed) : 0;
+    if (count >= Capacity) {
+      auto newCapacity = std::max((size_t)16, count * 2);
+      auto *newStorage = Storage::allocate(newCapacity);
+      if (storage) {
+        std::copy(storage->data(), storage->data() + count, newStorage->data());
+        newStorage->Count.store(count, std::memory_order_relaxed);
+        FreeList.push_back(storage);
+      }
+      
+      storage = newStorage;
+      Capacity = newCapacity;
+      Elements.store(storage, std::memory_order_release);
+    }
+    
+    new(&storage->data()[count]) ElemTy(elem);
+    storage->Count.store(count + 1, std::memory_order_release);
+    
+    if (ReaderCount.load(std::memory_order_acquire) == 0)
+      deallocateFreeList();
+  }
+  
+  Snapshot snapshot() {
+    incrementReaders();
+    auto *storage = Elements.load(SWIFT_MEMORY_ORDER_CONSUME);
+    if (storage == nullptr) {
+      return Snapshot(this, nullptr, 0);
+    }
+    
+    auto count = storage->Count.load(std::memory_order_acquire);
+    const auto *ptr = storage->data();
+    return Snapshot(this, ptr, count);
+  }
+};
+
 } // end namespace swift
 
 #endif // SWIFT_RUNTIME_CONCURRENTUTILS_H

stdlib/public/SwiftShims/RefCount.h

@@ -35,19 +35,10 @@ typedef InlineRefCountsPlaceholder InlineRefCounts;
 
 #include "llvm/Support/Compiler.h"
 #include "swift/Basic/type_traits.h"
+#include "swift/Runtime/Atomic.h"
 #include "swift/Runtime/Config.h"
 #include "swift/Runtime/Debug.h"
 
-// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require
-// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed'
-// is formally UB by C++11 language rules, we should be OK because neither
-// the processor model nor the optimizer can realistically reorder our uses
-// of 'consume'.
-#if __arm64__ || __arm__
-#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed)
-#else
-#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume)
-#endif
 
 /*
   An object conceptually has three refcounts. These refcounts