[sanitizer] re-apply r279572 and r279595 reverted in r279643: change the 64-bit allocator to use a single array for free-d chunks instead of a lock-free linked list of tranfer batches. This change simplifies the code, makes the allocator more 'hardened', and will allow simpler code to release RAM to OS. This may also slowdown malloc stress tests due to lock contension, but I did not observe noticeable slowdown on various real multi-threaded benchmarks.

llvm-svn: 279664

Author: kcc

compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h

@@ -26,8 +26,9 @@ struct SizeClassAllocatorLocalCache
 template <class SizeClassAllocator>
 struct SizeClassAllocator64LocalCache {
   typedef SizeClassAllocator Allocator;
-  typedef typename Allocator::TransferBatch TransferBatch;
   static const uptr kNumClasses = SizeClassAllocator::kNumClasses;
+  typedef typename Allocator::SizeClassMapT SizeClassMap;
+  typedef typename Allocator::CompactPtrT CompactPtrT;
 
   void Init(AllocatorGlobalStats *s) {
     stats_.Init();
@@ -47,9 +48,11 @@ struct SizeClassAllocator64LocalCache {
     stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
     if (UNLIKELY(c->count == 0))
-      Refill(allocator, class_id);
-    void *res = c->batch[--c->count];
-    PREFETCH(c->batch[c->count - 1]);
+      Refill(c, allocator, class_id);
+    CHECK_GT(c->count, 0);
+    CompactPtrT chunk = c->chunks[--c->count];
+    void *res = reinterpret_cast<void *>(allocator->CompactPtrToPointer(
+        allocator->GetRegionBeginBySizeClass(class_id), chunk));
     return res;
   }
 
@@ -63,24 +66,26 @@ struct SizeClassAllocator64LocalCache {
     PerClass *c = &per_class_[class_id];
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
-      Drain(allocator, class_id);
-    c->batch[c->count++] = p;
+      Drain(c, allocator, class_id, c->max_count / 2);
+    CompactPtrT chunk = allocator->PointerToCompactPtr(
+        allocator->GetRegionBeginBySizeClass(class_id),
+        reinterpret_cast<uptr>(p));
+    c->chunks[c->count++] = chunk;
   }
 
   void Drain(SizeClassAllocator *allocator) {
     for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
       PerClass *c = &per_class_[class_id];
       while (c->count > 0)
-        Drain(allocator, class_id);
+        Drain(c, allocator, class_id, c->count);
     }
   }
 
   // private:
-  typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap;
   struct PerClass {
-    uptr count;
-    uptr max_count;
-    void *batch[2 * TransferBatch::kMaxNumCached];
+    u32 count;
+    u32 max_count;
+    CompactPtrT chunks[2 * SizeClassMap::kMaxNumCachedHint];
   };
   PerClass per_class_[kNumClasses];
   AllocatorStats stats_;
@@ -90,77 +95,27 @@ struct SizeClassAllocator64LocalCache {
       return;
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
-      c->max_count = 2 * TransferBatch::MaxCached(i);
+      c->max_count = 2 * SizeClassMap::MaxCachedHint(i);
     }
   }
 
-  // TransferBatch class is declared in SizeClassAllocator.
-  // We transfer chunks between central and thread-local free lists in batches.
-  // For small size classes we allocate batches separately.
-  // For large size classes we may use one of the chunks to store the batch.
-  // sizeof(TransferBatch) must be a power of 2 for more efficient allocation.
-
-  // If kUseSeparateSizeClassForBatch is true,
-  // all TransferBatch objects are allocated from kBatchClassID
-  // size class (except for those that are needed for kBatchClassID itself).
-  // The goal is to have TransferBatches in a totally different region of RAM
-  // to improve security and allow more efficient RAM reclamation.
-  // This is experimental and may currently increase memory usage by up to 3%
-  // in extreme cases.
-  static const bool kUseSeparateSizeClassForBatch = false;
-
-  static uptr SizeClassForTransferBatch(uptr class_id) {
-    if (kUseSeparateSizeClassForBatch)
-      return class_id == SizeClassMap::kBatchClassID
-                 ? 0
-                 : SizeClassMap::kBatchClassID;
-    if (Allocator::ClassIdToSize(class_id) <
-        TransferBatch::AllocationSizeRequiredForNElements(
-            TransferBatch::MaxCached(class_id)))
-      return SizeClassMap::ClassID(sizeof(TransferBatch));
-    return 0;
-  }
-
-  // Returns a TransferBatch suitable for class_id.
-  // For small size classes allocates the batch from the allocator.
-  // For large size classes simply returns b.
-  TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
-                             TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
-      return (TransferBatch*)Allocate(allocator, batch_class_id);
-    return b;
-  }
-
-  // Destroys TransferBatch b.
-  // For small size classes deallocates b to the allocator.
-  // Does notthing for large size classes.
-  void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
-                    TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
-      Deallocate(allocator, batch_class_id, b);
-  }
-
-  NOINLINE void Refill(SizeClassAllocator *allocator, uptr class_id) {
+  NOINLINE void Refill(PerClass *c, SizeClassAllocator *allocator,
+                       uptr class_id) {
     InitCache();
-    PerClass *c = &per_class_[class_id];
-    TransferBatch *b = allocator->AllocateBatch(&stats_, this, class_id);
-    CHECK_GT(b->Count(), 0);
-    b->CopyToArray(c->batch);
-    c->count = b->Count();
-    DestroyBatch(class_id, allocator, b);
+    uptr num_requested_chunks = SizeClassMap::MaxCachedHint(class_id);
+    allocator->GetFromAllocator(&stats_, class_id, c->chunks,
+                                num_requested_chunks);
+    c->count = num_requested_chunks;
   }
 
-  NOINLINE void Drain(SizeClassAllocator *allocator, uptr class_id) {
+  NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id,
+                      uptr count) {
     InitCache();
-    PerClass *c = &per_class_[class_id];
-    uptr cnt = Min(c->max_count / 2, c->count);
-    uptr first_idx_to_drain = c->count - cnt;
-    TransferBatch *b = CreateBatch(
-        class_id, allocator, (TransferBatch *)c->batch[first_idx_to_drain]);
-    b->SetFromArray(allocator->GetRegionBeginBySizeClass(class_id),
-                    &c->batch[first_idx_to_drain], cnt);
-    c->count -= cnt;
-    allocator->DeallocateBatch(&stats_, class_id, b);
+    CHECK_GE(c->count, count);
+    uptr first_idx_to_drain = c->count - count;
+    c->count -= count;
+    allocator->ReturnToAllocator(&stats_, class_id,
+                                 &c->chunks[first_idx_to_drain], count);
   }
 };