replace chunks bool array with bit fields

Author: bratpiorka

src/pool/pool_disjoint.c

@@ -75,28 +75,36 @@ static slab_t *create_slab(bucket_t *bucket) {
     umf_result_t res = UMF_RESULT_SUCCESS;
     umf_memory_provider_handle_t provider = bucket->pool->provider;
 
-    slab_t *slab = umf_ba_global_alloc(sizeof(*slab));
+    size_t num_chunks_total =
+        utils_max(bucket_slab_min_size(bucket) / bucket->size, 1);
+
+    // Calculate the number of 64-bit words needed.
+    size_t num_words =
+        (num_chunks_total + CHUNK_BITMAP_SIZE - 1) / CHUNK_BITMAP_SIZE;
+
+    slab_t *slab = umf_ba_global_alloc(sizeof(*slab) +
+                                       num_words * sizeof(slab->chunks[0]));
     if (slab == NULL) {
         LOG_ERR("allocation of new slab failed!");
         return NULL;
     }
 
     slab->num_chunks_allocated = 0;
-    slab->first_free_chunk_idx = 0;
     slab->bucket = bucket;
 
     slab->iter.val = slab;
     slab->iter.prev = slab->iter.next = NULL;
 
-    slab->num_chunks_total =
-        utils_max(bucket_slab_min_size(bucket) / bucket->size, 1);
-    slab->chunks =
-        umf_ba_global_alloc(sizeof(*slab->chunks) * slab->num_chunks_total);
-    if (slab->chunks == NULL) {
-        LOG_ERR("allocation of slab chunks failed!");
-        goto free_slab;
+    slab->num_chunks_total = num_chunks_total;
+    slab->num_words = num_words;
+
+    // set all chunks as free
+    memset(slab->chunks, ~0, num_words * sizeof(slab->chunks[0]));
+    if (num_chunks_total % CHUNK_BITMAP_SIZE) {
+        // clear remaining bits
+        slab->chunks[num_words - 1] =
+            ((1ULL << (num_chunks_total % CHUNK_BITMAP_SIZE)) - 1);
     }
-    memset(slab->chunks, 0, sizeof(*slab->chunks) * slab->num_chunks_total);
 
     // if slab_min_size is not a multiple of bucket size, we would have some
     // padding at the end of the slab
@@ -108,7 +116,7 @@ static slab_t *create_slab(bucket_t *bucket) {
     res = umfMemoryProviderAlloc(provider, slab->slab_size, 0, &slab->mem_ptr);
     if (res != UMF_RESULT_SUCCESS) {
         LOG_ERR("allocation of slab data failed!");
-        goto free_slab_chunks;
+        goto free_slab;
     }
 
     // raw allocation is not available for user so mark it as inaccessible
@@ -117,9 +125,6 @@ static slab_t *create_slab(bucket_t *bucket) {
     LOG_DEBUG("bucket: %p, slab_size: %zu", (void *)bucket, slab->slab_size);
     return slab;
 
-free_slab_chunks:
-    umf_ba_global_free(slab->chunks);
-
 free_slab:
     umf_ba_global_free(slab);
     return NULL;
@@ -136,25 +141,21 @@ static void destroy_slab(slab_t *slab) {
         LOG_ERR("deallocation of slab data failed!");
     }
 
-    umf_ba_global_free(slab->chunks);
     umf_ba_global_free(slab);
 }
 
-// return the index of the first available chunk, SIZE_MAX otherwise
 static size_t slab_find_first_available_chunk_idx(const slab_t *slab) {
-    // use the first free chunk index as a hint for the search
-    for (bool *chunk = slab->chunks + slab->first_free_chunk_idx;
-         chunk != slab->chunks + slab->num_chunks_total; chunk++) {
-
-        // false means not used
-        if (*chunk == false) {
-            size_t idx = chunk - slab->chunks;
-            LOG_DEBUG("idx: %zu", idx);
-            return idx;
+    for (size_t i = 0; i < slab->num_words; i++) {
+        // NOTE: free chunks are represented as set bits
+        uint64_t word = slab->chunks[i];
+        if (word != 0) {
+            size_t bit_index = utils_lsb64(word);
+            size_t free_chunk = i * CHUNK_BITMAP_SIZE + bit_index;
+            return free_chunk;
         }
     }
 
-    LOG_DEBUG("idx: SIZE_MAX");
+    // No free chunk was found.
     return SIZE_MAX;
 }
 
@@ -167,12 +168,9 @@ static void *slab_get_chunk(slab_t *slab) {
         (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size);
 
     // mark chunk as used
-    slab->chunks[chunk_idx] = true;
+    slab_set_chunk_bit(slab, chunk_idx, false);
     slab->num_chunks_allocated += 1;
 
-    // use the found index as the next hint
-    slab->first_free_chunk_idx = chunk_idx + 1;
-
     return free_chunk;
 }
 
@@ -195,18 +193,9 @@ static void slab_free_chunk(slab_t *slab, void *ptr) {
     size_t chunk_idx = ptr_diff / slab->bucket->size;
 
     // Make sure that the chunk was allocated
-    assert(slab->chunks[chunk_idx] && "double free detected");
-    slab->chunks[chunk_idx] = false;
+    assert(slab_read_chunk_bit(slab, chunk_idx) == 0 && "double free detected");
+    slab_set_chunk_bit(slab, chunk_idx, true);
     slab->num_chunks_allocated -= 1;
-
-    if (chunk_idx < slab->first_free_chunk_idx) {
-        slab->first_free_chunk_idx = chunk_idx;
-    }
-
-    LOG_DEBUG("chunk_idx: %zu, num_chunks_allocated: %zu, "
-              "first_free_chunk_idx: %zu",
-              chunk_idx, slab->num_chunks_allocated,
-              slab->first_free_chunk_idx);
 }
 
 static bool slab_has_avail(const slab_t *slab) {

src/pool/pool_disjoint_internal.h

@@ -15,6 +15,8 @@
 #include "critnib/critnib.h"
 #include "utils_concurrency.h"
 
+#define CHUNK_BITMAP_SIZE 64
+
 typedef struct bucket_t bucket_t;
 typedef struct slab_t slab_t;
 typedef struct slab_list_item_t slab_list_item_t;
@@ -81,23 +83,24 @@ typedef struct slab_t {
     void *mem_ptr;
     size_t slab_size;
 
-    // Represents the current state of each chunk: if the bit is set, the
-    // chunk is allocated; otherwise, the chunk is free for allocation
-    bool *chunks;
     size_t num_chunks_total;
 
+    // Num of 64-bit words needed to store chunk state
+    size_t num_words;
+
     // Total number of allocated chunks at the moment.
     size_t num_chunks_allocated;
 
     // The bucket which the slab belongs to
     bucket_t *bucket;
 
-    // Hints where to start search for free chunk in a slab
-    size_t first_free_chunk_idx;
-
     // Store iterator to the corresponding node in avail/unavail list
     // to achieve O(1) removal
     slab_list_item_t iter;
+
+    // Represents the current state of each chunk: if the bit is clear, the
+    // chunk is allocated; otherwise, the chunk is free for allocation
+    uint64_t chunks[];
 } slab_t;
 
 typedef struct umf_disjoint_pool_shared_limits_t {
@@ -158,4 +161,24 @@ typedef struct disjoint_pool_t {
     size_t provider_min_page_size;
 } disjoint_pool_t;
 
+static inline void slab_set_chunk_bit(slab_t *slab, size_t index, bool value) {
+    assert(index < slab->num_chunks_total && "Index out of range");
+
+    size_t word_index = index / CHUNK_BITMAP_SIZE;
+    unsigned bit_index = index % CHUNK_BITMAP_SIZE;
+    if (value) {
+        slab->chunks[word_index] |= (1ULL << bit_index);
+    } else {
+        slab->chunks[word_index] &= ~(1ULL << bit_index);
+    }
+}
+
+static inline int slab_read_chunk_bit(const slab_t *slab, size_t index) {
+    assert(index < slab->num_chunks_total && "Index out of range");
+
+    size_t word_index = index / CHUNK_BITMAP_SIZE;
+    unsigned bit_index = index % CHUNK_BITMAP_SIZE;
+    return (slab->chunks[word_index] >> bit_index) & 1;
+}
+
 #endif // UMF_POOL_DISJOINT_INTERNAL_H

test/pools/disjoint_pool.cpp

@@ -113,9 +113,8 @@ TEST_F(test, internals) {
     EXPECT_GE(slab->num_chunks_total, slab->slab_size / bucket->size);
 
     // check allocation in slab
-    EXPECT_EQ(slab->chunks[0], true);
-    EXPECT_EQ(slab->chunks[1], false);
-    EXPECT_EQ(slab->first_free_chunk_idx, 1);
+    EXPECT_EQ(slab_read_chunk_bit(slab, 0), false);
+    EXPECT_EQ(slab_read_chunk_bit(slab, 1), true);
 
     // TODO:
     // * multiple alloc + free from single bucket