[libc] Use best-fit binary trie to make malloc logarithmic

This reworks the free store implementation in libc's malloc to use a
dlmalloc-style binary trie of circularly linked FIFO free lists. This
data structure can be maintained in logarithmic time, but it still
permits a relatively small implementation compared to other
logarithmic-time ordered maps.

The implementation doesn't do the various bitwise tricks or
optimizations used in actual dlmalloc; it instead optimizes for
(relative) readability and minimum code size. Specific optimization can
be added as necessary given future profiling.

Author: mysterymath

libc/config/linux/x86_64/entrypoints.txt

@@ -202,6 +202,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdlib.aligned_alloc
     libc.src.stdlib.calloc
     libc.src.stdlib.free
+    libc.src.stdlib.freelist_malloc
     libc.src.stdlib.malloc
     libc.src.stdlib.realloc
 

libc/src/__support/block.h

@@ -174,16 +174,32 @@ class Block {
     return inner_size - sizeof(prev_) + BLOCK_OVERHEAD;
   }
 
-  /// @returns The number of usable bytes inside the block.
+  /// @returns The number of usable bytes inside the block were it to be
+  /// allocated.
   size_t inner_size() const {
     if (!next())
       return 0;
     return inner_size(outer_size());
   }
 
+  /// @returns The number of usable bytes inside a block with the given outer
+  /// size were it to be allocated.
   static size_t inner_size(size_t outer_size) {
     // The usable region includes the prev_ field of the next block.
-    return outer_size - BLOCK_OVERHEAD + sizeof(prev_);
+    return inner_size_free(outer_size) + sizeof(prev_);
+  }
+
+  /// @returns The number of usable bytes inside the block if it remains free.
+  size_t inner_size_free() const {
+    if (!next())
+      return 0;
+    return inner_size_free(outer_size());
+  }
+
+  /// @returns The number of usable bytes inside a block with the given outer
+  /// size if it remains free.
+  static size_t inner_size_free(size_t outer_size) {
+    return outer_size - BLOCK_OVERHEAD;
   }
 
   /// @returns A pointer to the usable space inside this block.

libc/src/__support/freelist.h

@@ -1,4 +1,4 @@
-//===-- Interface for freelist_malloc -------------------------------------===//
+//===-- Interface for freelist --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,199 +9,107 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_H
 #define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
 
-#include "src/__support/CPP/array.h"
-#include "src/__support/CPP/cstddef.h"
-#include "src/__support/CPP/new.h"
-#include "src/__support/CPP/span.h"
-#include "src/__support/fixedvector.h"
-#include "src/__support/macros/config.h"
+#include "block.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-using cpp::span;
-
-/// Basic [freelist](https://en.wikipedia.org/wiki/Free_list) implementation
-/// for an allocator. This implementation buckets by chunk size, with a list
-/// of user-provided buckets. Each bucket is a linked list of storage chunks.
-/// Because this freelist uses the added chunks themselves as list nodes, there
-/// is a lower bound of `sizeof(FreeList.FreeListNode)` bytes for chunks which
-/// can be added to this freelist. There is also an implicit bucket for
-/// "everything else", for chunks which do not fit into a bucket.
-///
-/// Each added chunk will be added to the smallest bucket under which it fits.
-/// If it does not fit into any user-provided bucket, it will be added to the
-/// default bucket.
-///
-/// As an example, assume that the `FreeList` is configured with buckets of
-/// sizes {64, 128, 256, and 512} bytes. The internal state may look like the
-/// following:
-///
-/// @code{.unparsed}
-/// bucket[0] (64B) --> chunk[12B] --> chunk[42B] --> chunk[64B] --> NULL
-/// bucket[1] (128B) --> chunk[65B] --> chunk[72B] --> NULL
-/// bucket[2] (256B) --> NULL
-/// bucket[3] (512B) --> chunk[312B] --> chunk[512B] --> chunk[416B] --> NULL
-/// bucket[4] (implicit) --> chunk[1024B] --> chunk[513B] --> NULL
-/// @endcode
+/// A circularly-linked FIFO list storing free Blocks. All Blocks on a list
+/// are the same size.
 ///
-/// Note that added chunks should be aligned to a 4-byte boundary.
-template <size_t NUM_BUCKETS = 6> class FreeList {
+/// Allocating free blocks in FIFO order maximizes the amount of time before a
+/// free block is reused. This in turn maximizes the number of opportunities for
+/// it to be coalesced with an adjacent block, which tends to reduce heap
+/// fragmentation.
+class FreeList {
 public:
-  // Remove copy/move ctors
-  FreeList(const FreeList &other) = delete;
-  FreeList(FreeList &&other) = delete;
-  FreeList &operator=(const FreeList &other) = delete;
-  FreeList &operator=(FreeList &&other) = delete;
-
-  /// Adds a chunk to this freelist.
-  bool add_chunk(cpp::span<cpp::byte> chunk);
-
-  /// Finds an eligible chunk for an allocation of size `size`.
-  ///
-  /// @note This returns the first allocation possible within a given bucket;
-  /// It does not currently optimize for finding the smallest chunk.
-  ///
-  /// @returns
-  /// * On success - A span representing the chunk.
-  /// * On failure (e.g. there were no chunks available for that allocation) -
-  ///   A span with a size of 0.
-  cpp::span<cpp::byte> find_chunk(size_t size) const;
-
-  template <typename Cond> cpp::span<cpp::byte> find_chunk_if(Cond op) const;
-
-  /// Removes a chunk from this freelist.
-  bool remove_chunk(cpp::span<cpp::byte> chunk);
-
-  /// For a given size, find which index into chunks_ the node should be written
-  /// to.
-  constexpr size_t find_chunk_ptr_for_size(size_t size, bool non_null) const;
-
-  struct FreeListNode {
-    FreeListNode *next;
-    size_t size;
-  };
-
-  constexpr void set_freelist_node(FreeListNode &node,
-                                   cpp::span<cpp::byte> chunk);
-
-  constexpr explicit FreeList(const cpp::array<size_t, NUM_BUCKETS> &sizes)
-      : chunks_(NUM_BUCKETS + 1, 0), sizes_(sizes.begin(), sizes.end()) {}
-
-private:
-  FixedVector<FreeList::FreeListNode *, NUM_BUCKETS + 1> chunks_;
-  FixedVector<size_t, NUM_BUCKETS> sizes_;
-};
-
-template <size_t NUM_BUCKETS>
-constexpr void FreeList<NUM_BUCKETS>::set_freelist_node(FreeListNode &node,
-                                                        span<cpp::byte> chunk) {
-  // Add it to the correct list.
-  size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), false);
-  node.size = chunk.size();
-  node.next = chunks_[chunk_ptr];
-  chunks_[chunk_ptr] = &node;
-}
+  class Node {
+  public:
+    /// @returns The block containing this node.
+    Block<> *block() const {
+      return const_cast<Block<> *>(Block<>::from_usable_space(this));
+    }
 
-template <size_t NUM_BUCKETS>
-bool FreeList<NUM_BUCKETS>::add_chunk(span<cpp::byte> chunk) {
-  // Check that the size is enough to actually store what we need
-  if (chunk.size() < sizeof(FreeListNode))
-    return false;
+    /// @returns The inner size of blocks in the list containing this node.
+    size_t size() const { return block()->inner_size(); }
 
-  FreeListNode *node = ::new (chunk.data()) FreeListNode;
-  set_freelist_node(*node, chunk);
+  private:
+    // Circularly linked pointers to adjacent nodes.
+    Node *prev;
+    Node *next;
+    friend class FreeList;
+  };
 
-  return true;
-}
+  constexpr FreeList() : FreeList(nullptr) {}
+  constexpr FreeList(Node *begin) : begin_(begin) {}
 
-template <size_t NUM_BUCKETS>
-template <typename Cond>
-span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk_if(Cond op) const {
-  for (FreeListNode *node : chunks_) {
-    while (node != nullptr) {
-      span<cpp::byte> chunk(reinterpret_cast<cpp::byte *>(node), node->size);
-      if (op(chunk))
-        return chunk;
+  bool empty() const { return !begin_; }
 
-      node = node->next;
-    }
+  /// @returns The inner size of blocks in the list.
+  size_t size() const {
+    LIBC_ASSERT(begin_ && "empty lists have no size");
+    return begin_->size();
   }
 
-  return {};
-}
+  /// @returns The first node in the list.
+  Node *begin() { return begin_; }
 
-template <size_t NUM_BUCKETS>
-span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk(size_t size) const {
-  if (size == 0)
-    return span<cpp::byte>();
+  /// @returns The first block in the list.
+  Block<> *front() { return begin_->block(); }
 
-  size_t chunk_ptr = find_chunk_ptr_for_size(size, true);
+  /// Push a block to the back of the list.
+  /// The block must be large enough to contain a node.
+  void push(Block<> *block);
 
-  // Check that there's data. This catches the case where we run off the
-  // end of the array
-  if (chunks_[chunk_ptr] == nullptr)
-    return span<cpp::byte>();
+  /// Push an already-constructed node to the back of the list.
+  /// This allows pushing derived node types with additional data.
+  void push(Node *node);
 
-  // Now iterate up the buckets, walking each list to find a good candidate
-  for (size_t i = chunk_ptr; i < chunks_.size(); i++) {
-    FreeListNode *node = chunks_[static_cast<unsigned short>(i)];
+  /// Pop the first node from the list.
+  void pop();
 
-    while (node != nullptr) {
-      if (node->size >= size)
-        return span<cpp::byte>(reinterpret_cast<cpp::byte *>(node), node->size);
+  /// Remove an arbitrary node from the list.
+  void remove(Node *node);
 
-      node = node->next;
-    }
-  }
+private:
+  Node *begin_;
+};
 
-  // If we get here, we've checked every block in every bucket. There's
-  // nothing that can support this allocation.
-  return span<cpp::byte>();
+LIBC_INLINE void FreeList::push(Block<> *block) {
+  LIBC_ASSERT(!block->used() && "only free blocks can be placed on free lists");
+  LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList) &&
+              "block too small to accomodate free list node");
+  push(new (block->usable_space()) Node);
 }
 
-template <size_t NUM_BUCKETS>
-bool FreeList<NUM_BUCKETS>::remove_chunk(span<cpp::byte> chunk) {
-  size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), true);
-
-  // Check head first.
-  if (chunks_[chunk_ptr] == nullptr)
-    return false;
-
-  FreeListNode *node = chunks_[chunk_ptr];
-  if (reinterpret_cast<cpp::byte *>(node) == chunk.data()) {
-    chunks_[chunk_ptr] = node->next;
-    return true;
+LIBC_INLINE void FreeList::push(Node *node) {
+  if (begin_) {
+    LIBC_ASSERT(Block<>::from_usable_space(node)->outer_size() ==
+                    begin_->block()->outer_size() &&
+                "freelist entries must have the same size");
+    // Since the list is circular, insert the node immediately before begin_.
+    node->prev = begin_->prev;
+    node->next = begin_;
+    begin_->prev->next = node;
+    begin_->prev = node;
+  } else {
+    begin_ = node->prev = node->next = node;
   }
-
-  // No? Walk the nodes.
-  node = chunks_[chunk_ptr];
-
-  while (node->next != nullptr) {
-    if (reinterpret_cast<cpp::byte *>(node->next) == chunk.data()) {
-      // Found it, remove this node out of the chain
-      node->next = node->next->next;
-      return true;
-    }
-
-    node = node->next;
-  }
-
-  return false;
 }
 
-template <size_t NUM_BUCKETS>
-constexpr size_t
-FreeList<NUM_BUCKETS>::find_chunk_ptr_for_size(size_t size,
-                                               bool non_null) const {
-  size_t chunk_ptr = 0;
-  for (chunk_ptr = 0u; chunk_ptr < sizes_.size(); chunk_ptr++) {
-    if (sizes_[chunk_ptr] >= size &&
-        (!non_null || chunks_[chunk_ptr] != nullptr)) {
-      break;
-    }
+LIBC_INLINE void FreeList::pop() { remove(begin_); }
+
+LIBC_INLINE void FreeList::remove(Node *node) {
+  LIBC_ASSERT(begin_ && "cannot remove from empty list");
+  if (node == node->next) {
+    LIBC_ASSERT(node == begin_ &&
+                "a self-referential node must be the only element");
+    begin_ = nullptr;
+  } else {
+    node->prev->next = node->next;
+    node->next->prev = node->prev;
+    if (begin_ == node)
+      begin_ = node->next;
   }
-
-  return chunk_ptr;
 }
 
 } // namespace LIBC_NAMESPACE_DECL