SIL: add a StackList data structure with zero cost operations.

A StackList is the best choice for things like worklists, etc., if no random access is needed.
Regardless of how large a Stack gets, there is no memory allocation needed (except maybe for the first few uses in the compiler run).
All operations have (almost) zero cost.
The needed memory is managed by the SILModule. Initially, the memory slabs are allocated with the module's bump pointer allocator. In contrast to bump pointer allocated memory, those slabs can be freed again (at zero cost) and then recycled.

StackList is meant to be a replacement for llvm::SmallVector, which needs to malloc after the small size is exceeded.

This is more a usability than a compile time improvement.
Usually we think hard about how to correctly use an llvm::SmallVector to avoid memory allocations: we chose the small size wisely and in many cases we keep a shared instance of a SmallVector to reuse its allocated capacity.

All this is not necessary by using a StackList: no need to select a small size and to share it across usages.

Author: eeckstein

include/swift/SIL/SILModule.h

@@ -49,10 +49,52 @@
 #include "llvm/Support/raw_ostream.h"
 #include <functional>
 
+namespace swift {
+
+/// A fixed size slab of memory, which can be allocated and freed by the
+/// SILModule at (basically) zero cost.
+class FixedSizeSlab : public llvm::ilist_node<FixedSizeSlab>,
+                       public SILAllocated<FixedSizeSlab> {
+public:
+  /// The capacity of the payload.
+  static constexpr size_t capacity = 64 * sizeof(uintptr_t);
+
+private:
+  friend class SILModule;
+
+  /// The magic number which is stored in overflowGuard.
+  static constexpr uintptr_t magicNumber = (uintptr_t)0xdeadbeafdeadbeafull;
+
+  /// The payload.
+  char data[capacity];
+  
+  /// Used for a cheap buffer overflow check - in the spirit of libgmalloc.
+  uintptr_t overflowGuard = magicNumber;
+
+public:
+  /// Returns the payload pointing to \p T.
+  template<typename T> T *dataFor() { return (T *)(&data[0]); }
+
+  /// Returns the payload pointing to const \p T
+  template<typename T> const T *dataFor() const { return (const T *)(&data[0]); }
+};
+
+} // end swift namespace
+
 namespace llvm {
 namespace yaml {
 class Output;
 } // end namespace yaml
+
+template <>
+struct ilist_traits<::swift::FixedSizeSlab> :
+public ilist_node_traits<::swift::FixedSizeSlab> {
+public:
+  static void deleteNode(::swift::FixedSizeSlab *V) {
+    llvm_unreachable("cannot delete a slab");
+  }
+};
+
 } // end namespace llvm
 
 namespace swift {
@@ -129,6 +171,7 @@ class SILModule {
   };
 
   using ActionCallback = std::function<void()>;
+  using SlabList = llvm::ilist<FixedSizeSlab>;
 
 private:
   friend KeyPathPattern;
@@ -151,6 +194,12 @@ class SILModule {
   /// Allocator that manages the memory of all the pieces of the SILModule.
   mutable llvm::BumpPtrAllocator BPA;
 
+  /// The list of freed slabs, which can be reused.
+  SlabList freeSlabs;
+  
+  /// For consistency checking.
+  size_t numAllocatedSlabs = 0;
+
   /// The swift Module associated with this SILModule.
   ModuleDecl *TheSwiftModule;
 
@@ -733,6 +782,22 @@ class SILModule {
     return static_cast<T *>(allocate(sizeof(T) * Count, alignof(T)));
   }
 
+  /// Allocates a slab of memory.
+  ///
+  /// This has (almost) zero cost, because for the first time, the allocation is
+  /// done with the BPA.
+  /// Subsequent allocations are reusing the already freed slabs.
+  FixedSizeSlab *allocSlab();
+  
+  /// Frees a slab.
+  ///
+  /// This has (almost) zero cost, because the slab is just put into the
+  /// freeSlabs list.
+  void freeSlab(FixedSizeSlab *slab);
+  
+  /// Frees all slabs of a list.
+  void freeAllSlabs(SlabList &slabs);
+
   template <typename T>
   MutableArrayRef<T> allocateCopy(ArrayRef<T> Array) const {
     MutableArrayRef<T> result(allocate<T>(Array.size()), Array.size());

include/swift/SIL/StackList.h

@@ -0,0 +1,155 @@
+//===--- StackList.h - defines the StackList data structure -----*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SIL_STACKLIST_H
+#define SWIFT_SIL_STACKLIST_H
+
+#include "swift/SIL/SILModule.h"
+
+namespace swift {
+
+/// A very efficient implementation of a stack, which can also be iterated over.
+///
+/// A StackList is the best choice for things like worklists, etc., if no random
+/// access is needed.
+/// Regardless of how large a Stack gets, there is no memory allocation needed
+/// (except maybe for the first few uses in the compiler run).
+/// All operations have (almost) zero cost.
+template <typename Element> class StackList {
+  /// The capacity of a single slab.
+  static constexpr size_t slabCapacity =
+      FixedSizeSlab::capacity / sizeof(Element);
+  
+  static_assert(slabCapacity > 0, "Element type to large for StackList");
+  static_assert(alignof(FixedSizeSlab) >= alignof(Element),
+                "Element alignment to large for StackList");
+  
+  /// Backlink to the module which manages the slab allocation.
+  SILModule &module;
+
+  /// The list of slabs.
+  ///
+  /// Invariant: there is always free space in the last slab to store at least
+  /// one element.
+  SILModule::SlabList slabs;
+  
+  /// The index of the next free element in endSlab.
+  ///
+  /// Invariant: endIndex < slabCapacity
+  unsigned endIndex = 0;
+
+  FixedSizeSlab *lastSlab() { return &*slabs.rbegin(); }
+  const FixedSizeSlab *lastSlab() const { return &*slabs.rbegin(); }
+
+  void allocSlab() { slabs.push_back(module.allocSlab()); }
+
+  void growIfNeeded() {
+    if (endIndex == slabCapacity) {
+      allocSlab();
+      endIndex = 0;
+    }
+  }
+  
+
+public:
+  /// The Stack's iterator.
+  class iterator {
+    friend StackList<Element>;
+    const FixedSizeSlab *slab;
+    unsigned index;
+
+    iterator(const FixedSizeSlab *slab, unsigned index)
+      : slab(slab), index(index) {}
+      
+  public:
+    const Element &operator*() const {
+      assert(index < slabCapacity);
+      return slab->dataFor<Element>()[index];
+    }
+    const Element &operator->() const { return *this; }
+
+    iterator &operator++() {
+      assert(index < slabCapacity);
+      index++;
+      if (index == slabCapacity) {
+        slab = &*std::next(slab->getIterator());
+        index = 0;
+      }
+      return *this;
+    }
+    
+    iterator operator++(int unused) {
+      iterator copy = *this;
+      ++*this;
+      return copy;
+    }
+
+    friend bool operator==(iterator lhs, iterator rhs) {
+      return lhs.slab == rhs.slab && lhs.index == rhs.index;
+    }
+
+    friend bool operator!=(iterator lhs, iterator rhs) {
+      return !(lhs == rhs);
+    }
+  };
+
+  /// Constructor.
+  StackList(SILFunction *function) : module(function->getModule()) {
+    /// Allocate one slab so that there is free space for inserting the first
+    /// element.
+    allocSlab();
+  }
+
+  ~StackList() {
+    module.freeAllSlabs(slabs);
+  }
+
+  iterator begin() const { return iterator(&*slabs.begin(), 0); }
+  iterator end() const { return iterator(lastSlab(), endIndex); }
+  
+  bool empty() const {
+    return begin() == end();
+  }
+
+  /// Adds a new element at the end.
+  void push_back(const Element &newElement) {
+    assert(endIndex < slabCapacity);
+    lastSlab()->template dataFor<Element>()[endIndex++] = newElement;
+    growIfNeeded();
+  }
+
+  /// Adds a new element at the end.
+  void push_back(Element &&newElement) {
+    assert(endIndex < slabCapacity);
+    lastSlab()->template dataFor<Element>()[endIndex++] = std::move(newElement);
+    growIfNeeded();
+  }
+
+  /// Removes the last element and returns it.
+  Element pop_back_val() {
+    FixedSizeSlab *slab = lastSlab();
+    if (endIndex > 0)
+      return std::move(slab->dataFor<Element>()[--endIndex]);
+
+    assert(!empty());
+  
+    slabs.remove(slab);
+    module.freeSlab(slab);
+    assert(!slabs.empty());
+    endIndex = slabCapacity - 1;
+    return std::move(lastSlab()->template dataFor<Element>()[endIndex]);
+  }
+};
+
+} // namespace swift
+
+#endif

lib/SIL/IR/SILModule.cpp

@@ -35,6 +35,8 @@
 using namespace swift;
 using namespace Lowering;
 
+STATISTIC(NumSlabsAllocated, "number of slabs allocated in SILModule");
+
 class SILModule::SerializationCallback final
     : public DeserializationNotificationHandler {
   void didDeserialize(ModuleDecl *M, SILFunction *fn) override {
@@ -116,6 +118,10 @@ SILModule::SILModule(llvm::PointerUnion<FileUnit *, ModuleDecl *> context,
 SILModule::~SILModule() {
 #ifndef NDEBUG
   checkForLeaks();
+
+  NumSlabsAllocated += numAllocatedSlabs;
+  assert(numAllocatedSlabs == freeSlabs.size() && "leaking slabs in SILModule");
+  freeSlabs.clearAndLeakNodesUnsafely();
 #endif
 
   // Decrement ref count for each SILGlobalVariable with static initializers.
@@ -197,6 +203,26 @@ void *SILModule::allocate(unsigned Size, unsigned Align) const {
   return BPA.Allocate(Size, Align);
 }
 
+FixedSizeSlab *SILModule::allocSlab() {
+  if (freeSlabs.empty()) {
+    numAllocatedSlabs++;
+    return new (*this) FixedSizeSlab();
+  }
+
+  FixedSizeSlab *slab = &*freeSlabs.rbegin();
+  freeSlabs.remove(slab);
+  return slab;
+}
+
+void SILModule::freeSlab(FixedSizeSlab *slab) {
+  freeSlabs.push_back(slab);
+  assert(slab->overflowGuard == FixedSizeSlab::magicNumber);
+}
+
+void SILModule::freeAllSlabs(SlabList &slabs) {
+  freeSlabs.splice(freeSlabs.end(), slabs);
+}
+
 void *SILModule::allocateInst(unsigned Size, unsigned Align) const {
   return AlignedAlloc(Size, Align);
 }