[Runtime] 4.2 - Change MetadataLookup section vectors to use ConcurrentReadableArray.

include/swift/Runtime/Atomic.h

@@ -0,0 +1,31 @@
+//===--- Atomic.h - Utilities for atomic operations. ------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for atomic operations, to use with std::atomic.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_RUNTIME_ATOMIC_H
+#define SWIFT_RUNTIME_ATOMIC_H
+
+// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require
+// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed'
+// is formally UB by C++11 language rules, we should be OK because neither
+// the processor model nor the optimizer can realistically reorder our uses
+// of 'consume'.
+#if __arm64__ || __arm__
+#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed)
+#else
+#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume)
+#endif
+
+#endif

include/swift/Runtime/Concurrent.h

@@ -12,10 +12,14 @@
 #ifndef SWIFT_RUNTIME_CONCURRENTUTILS_H
 #define SWIFT_RUNTIME_CONCURRENTUTILS_H
 #include <iterator>
+#include <algorithm>
 #include <atomic>
 #include <functional>
 #include <stdint.h>
 #include "llvm/Support/Allocator.h"
+#include "Atomic.h"
+#include "Debug.h"
+#include "Mutex.h"
 
 #if defined(__FreeBSD__) || defined(__CYGWIN__) || defined(__HAIKU__)
 #include <stdio.h>
@@ -406,6 +410,136 @@ class ConcurrentMap
   }
 };
 
+
+/// An append-only array that can be read without taking locks. Writes
+/// are still locked and serialized, but only with respect to other
+/// writes.
+template <class ElemTy> struct ConcurrentReadableArray {
+private:
+  /// The struct used for the array's storage. The `Elem` member is
+  /// considered to be the first element of a variable-length array,
+  /// whose size is determined by the allocation. The `Capacity` member
+  /// from `ConcurrentReadableArray` indicates how large it can be.
+  struct Storage {
+    std::atomic<size_t> Count;
+    typename std::aligned_storage<sizeof(ElemTy), alignof(ElemTy)>::type Elem;
+
+    static Storage *allocate(size_t capacity) {
+      auto size = sizeof(Storage) + (capacity - 1) * sizeof(Storage().Elem);
+      auto *ptr = reinterpret_cast<Storage *>(malloc(size));
+      if (!ptr) swift::crash("Could not allocate memory.");
+      ptr->Count.store(0, std::memory_order_relaxed);
+      return ptr;
+    }
+
+    void deallocate() {
+      for (size_t i = 0; i < Count; i++) {
+        data()[i].~ElemTy();
+      }
+      free(this);
+    }
+
+    ElemTy *data() {
+      return reinterpret_cast<ElemTy *>(&Elem);
+    }
+  };
+
+  size_t Capacity;
+  std::atomic<size_t> ReaderCount;
+  std::atomic<Storage *> Elements;
+  Mutex WriterLock;
+  std::vector<Storage *> FreeList;
+
+  void incrementReaders() {
+    ReaderCount.fetch_add(1, std::memory_order_acquire);
+  }
+
+  void decrementReaders() {
+    ReaderCount.fetch_sub(1, std::memory_order_release);
+  }
+
+  void deallocateFreeList() {
+    for (Storage *storage : FreeList)
+      storage->deallocate();
+    FreeList.clear();
+    FreeList.shrink_to_fit();
+  }
+
+public:
+  struct Snapshot {
+    ConcurrentReadableArray *Array;
+    const ElemTy *Start;
+    size_t Count;
+
+    Snapshot(ConcurrentReadableArray *array, const ElemTy *start, size_t count)
+      : Array(array), Start(start), Count(count) {}
+
+    Snapshot(const Snapshot &other)
+      : Array(other.Array), Start(other.Start), Count(other.Count) {
+      Array->incrementReaders();
+    }
+
+    ~Snapshot() {
+      Array->decrementReaders();
+    }
+
+    const ElemTy *begin() { return Start; }
+    const ElemTy *end() { return Start + Count; }
+    size_t count() { return Count; }
+  };
+
+  // This type cannot be safely copied, moved, or deleted.
+  ConcurrentReadableArray(const ConcurrentReadableArray &) = delete;
+  ConcurrentReadableArray(ConcurrentReadableArray &&) = delete;
+  ConcurrentReadableArray &operator=(const ConcurrentReadableArray &) = delete;
+
+  ConcurrentReadableArray() : Capacity(0), ReaderCount(0), Elements(nullptr) {}
+
+  ~ConcurrentReadableArray() {
+    assert(ReaderCount.load(std::memory_order_acquire) == 0 &&
+           "deallocating ConcurrentReadableArray with outstanding snapshots");
+    deallocateFreeList();
+  }
+
+  void push_back(const ElemTy &elem) {
+    ScopedLock guard(WriterLock);
+
+    auto *storage = Elements.load(std::memory_order_relaxed);
+    auto count = storage ? storage->Count.load(std::memory_order_relaxed) : 0;
+    if (count >= Capacity) {
+      auto newCapacity = std::max((size_t)16, count * 2);
+      auto *newStorage = Storage::allocate(newCapacity);
+      if (storage) {
+        std::copy(storage->data(), storage->data() + count, newStorage->data());
+        newStorage->Count.store(count, std::memory_order_relaxed);
+        FreeList.push_back(storage);
+      }
+
+      storage = newStorage;
+      Capacity = newCapacity;
+      Elements.store(storage, std::memory_order_release);
+    }
+
+    new(&storage->data()[count]) ElemTy(elem);
+    storage->Count.store(count + 1, std::memory_order_release);
+
+    if (ReaderCount.load(std::memory_order_acquire) == 0)
+      deallocateFreeList();
+  }
+
+  Snapshot snapshot() {
+    incrementReaders();
+    auto *storage = Elements.load(SWIFT_MEMORY_ORDER_CONSUME);
+    if (storage == nullptr) {
+      return Snapshot(this, nullptr, 0);
+    }
+
+    auto count = storage->Count.load(std::memory_order_acquire);
+    const auto *ptr = storage->data();
+    return Snapshot(this, ptr, count);
+  }
+};
+
 } // end namespace swift
 
 #endif // SWIFT_RUNTIME_CONCURRENTUTILS_H

stdlib/public/SwiftShims/RefCount.h

@@ -35,19 +35,10 @@ typedef InlineRefCountsPlaceholder InlineRefCounts;
 
 #include "llvm/Support/Compiler.h"
 #include "swift/Basic/type_traits.h"
+#include "swift/Runtime/Atomic.h"
 #include "swift/Runtime/Config.h"
 #include "swift/Runtime/Debug.h"
 
-// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require
-// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed'
-// is formally UB by C++11 language rules, we should be OK because neither
-// the processor model nor the optimizer can realistically reorder our uses
-// of 'consume'.
-#if __arm64__ || __arm__
-#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed)
-#else
-#  define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume)
-#endif
 
 /*
   An object conceptually has three refcounts. These refcounts 

stdlib/public/runtime/MetadataLookup.cpp

@@ -23,7 +23,6 @@
 #include "swift/Runtime/Concurrent.h"
 #include "swift/Runtime/HeapObject.h"
 #include "swift/Runtime/Metadata.h"
-#include "swift/Runtime/Mutex.h"
 #include "swift/Strings.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Optional.h"
@@ -106,11 +105,9 @@ namespace {
 
 struct TypeMetadataPrivateState {
   ConcurrentMap<NominalTypeDescriptorCacheEntry> NominalCache;
-  std::vector<TypeMetadataSection> SectionsToScan;
-  Mutex SectionsToScanLock;
+  ConcurrentReadableArray<TypeMetadataSection> SectionsToScan;
 
   TypeMetadataPrivateState() {
-    SectionsToScan.reserve(16);
     initializeTypeMetadataRecordLookup();
   }
 
@@ -122,7 +119,6 @@ static void
 _registerTypeMetadataRecords(TypeMetadataPrivateState &T,
                              const TypeMetadataRecord *begin,
                              const TypeMetadataRecord *end) {
-  ScopedLock guard(T.SectionsToScanLock);
   T.SectionsToScan.push_back(TypeMetadataSection{begin, end});
 }
 
@@ -296,12 +292,9 @@ swift::_contextDescriptorMatchesMangling(const ContextDescriptor *context,
 
 // returns the nominal type descriptor for the type named by typeName
 static const TypeContextDescriptor *
-_searchTypeMetadataRecords(const TypeMetadataPrivateState &T,
+_searchTypeMetadataRecords(TypeMetadataPrivateState &T,
                            Demangle::NodePointer node) {
-  unsigned sectionIdx = 0;
-  unsigned endSectionIdx = T.SectionsToScan.size();
-  for (; sectionIdx < endSectionIdx; ++sectionIdx) {
-    auto &section = T.SectionsToScan[sectionIdx];
+  for (auto &section : T.SectionsToScan.snapshot()) {
     for (const auto &record : section) {
       if (auto ntd = record.getTypeContextDescriptor()) {
         if (_contextDescriptorMatchesMangling(ntd, node)) {
@@ -342,9 +335,7 @@ _findNominalTypeDescriptor(Demangle::NodePointer node,
     return Value->getDescription();
 
   // Check type metadata records
-  T.SectionsToScanLock.withLock([&] {
-    foundNominal = _searchTypeMetadataRecords(T, node);
-  });
+  foundNominal = _searchTypeMetadataRecords(T, node);
 
   // Check protocol conformances table. Note that this has no support for
   // resolving generic types yet.
@@ -395,11 +386,9 @@ namespace {
 
   struct ProtocolMetadataPrivateState {
     ConcurrentMap<ProtocolDescriptorCacheEntry> ProtocolCache;
-    std::vector<ProtocolSection> SectionsToScan;
-    Mutex SectionsToScanLock;
+    ConcurrentReadableArray<ProtocolSection> SectionsToScan;
 
     ProtocolMetadataPrivateState() {
-      SectionsToScan.reserve(16);
       initializeProtocolLookup();
     }
   };
@@ -411,7 +400,6 @@ static void
 _registerProtocols(ProtocolMetadataPrivateState &C,
                    const ProtocolRecord *begin,
                    const ProtocolRecord *end) {
-  ScopedLock guard(C.SectionsToScanLock);
   C.SectionsToScan.push_back(ProtocolSection{begin, end});
 }
 
@@ -439,12 +427,9 @@ void swift::swift_registerProtocols(const ProtocolRecord *begin,
 }
 
 static const ProtocolDescriptor *
-_searchProtocolRecords(const ProtocolMetadataPrivateState &C,
+_searchProtocolRecords(ProtocolMetadataPrivateState &C,
                        const llvm::StringRef protocolName){
-  unsigned sectionIdx = 0;
-  unsigned endSectionIdx = C.SectionsToScan.size();
-  for (; sectionIdx < endSectionIdx; ++sectionIdx) {
-    auto &section = C.SectionsToScan[sectionIdx];
+  for (auto &section : C.SectionsToScan.snapshot()) {
     for (const auto &record : section) {
       if (auto protocol = record.Protocol.getPointer()) {
         // Drop the "S$" prefix from the protocol record. It's not used in
@@ -472,9 +457,7 @@ _findProtocolDescriptor(llvm::StringRef mangledName) {
     return Value->getDescription();
 
   // Check type metadata records
-  T.SectionsToScanLock.withLock([&] {
-    foundProtocol = _searchProtocolRecords(T, mangledName);
-  });
+  foundProtocol = _searchProtocolRecords(T, mangledName);
 
   if (foundProtocol) {
     T.ProtocolCache.getOrInsert(mangledName, foundProtocol);
@@ -534,21 +517,18 @@ class DynamicFieldSection {
   DynamicFieldSection(const FieldDescriptor **fields, size_t size)
       : Begin(fields), End(fields + size) {}
 
-  const FieldDescriptor **begin() { return Begin; }
+  const FieldDescriptor **begin() const { return Begin; }
 
   const FieldDescriptor **end() const { return End; }
 };
 
 struct FieldCacheState {
   ConcurrentMap<FieldDescriptorCacheEntry> FieldCache;
 
-  Mutex SectionsLock;
-  std::vector<StaticFieldSection> StaticSections;
-  std::vector<DynamicFieldSection> DynamicSections;
+  ConcurrentReadableArray<StaticFieldSection> StaticSections;
+  ConcurrentReadableArray<DynamicFieldSection> DynamicSections;
 
   FieldCacheState() {
-    StaticSections.reserve(16);
-    DynamicSections.reserve(8);
     initializeTypeFieldLookup();
   }
 };
@@ -559,7 +539,6 @@ static Lazy<FieldCacheState> FieldCache;
 void swift::swift_registerFieldDescriptors(const FieldDescriptor **records,
                                            size_t size) {
   auto &cache = FieldCache.get();
-  ScopedLock guard(cache.SectionsLock);
   cache.DynamicSections.push_back({records, size});
 }
 
@@ -570,7 +549,6 @@ void swift::addImageTypeFieldDescriptorBlockCallback(const void *recordsBegin,
 
   // Field cache should always be sufficiently initialized by this point.
   auto &cache = FieldCache.unsafeGetAlreadyInitialized();
-  ScopedLock guard(cache.SectionsLock);
   cache.StaticSections.push_back({recordsBegin, recordsEnd});
 }
 
@@ -1216,16 +1194,15 @@ void swift::swift_getFieldAt(
     return;
   }
 
-  ScopedLock guard(cache.SectionsLock);
   // Otherwise let's try to find it in one of the sections.
-  for (auto &section : cache.DynamicSections) {
+  for (auto &section : cache.DynamicSections.snapshot()) {
     for (const auto *descriptor : section) {
       if (isRequestedDescriptor(*descriptor))
         return;
     }
   }
 
-  for (const auto &section : cache.StaticSections) {
+  for (const auto &section : cache.StaticSections.snapshot()) {
     for (auto &descriptor : section) {
       if (isRequestedDescriptor(descriptor))
         return;