Skip to content

Commit 82642cc

Browse files
committed
Update to mimalloc 2.0.6
1 parent 95d904a commit 82642cc

18 files changed

+421
-360
lines changed

Include/internal/mimalloc/mimalloc-atomic.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,23 @@ terms of the MIT license. A copy of the license can be found in the file
2323
#define _Atomic(tp) std::atomic<tp>
2424
#define mi_atomic(name) std::atomic_##name
2525
#define mi_memory_order(name) std::memory_order_##name
26+
#if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571
27+
#define MI_ATOMIC_VAR_INIT(x) x
28+
#else
29+
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
30+
#endif
2631
#elif defined(_MSC_VER)
2732
// Use MSVC C wrapper for C11 atomics
2833
#define _Atomic(tp) tp
29-
#define ATOMIC_VAR_INIT(x) x
34+
#define MI_ATOMIC_VAR_INIT(x) x
3035
#define mi_atomic(name) mi_atomic_##name
3136
#define mi_memory_order(name) mi_memory_order_##name
3237
#else
3338
// Use C11 atomics
3439
#include <stdatomic.h>
3540
#define mi_atomic(name) atomic_##name
3641
#define mi_memory_order(name) memory_order_##name
42+
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
3743
#endif
3844

3945
// Various defines for all used memory orders in mimalloc

Include/internal/mimalloc/mimalloc-internal.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,8 @@ mi_msecs_t _mi_clock_start(void);
148148

149149
// "alloc.c"
150150
void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic`
151-
void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero);
152-
void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero);
151+
void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
152+
void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept;
153153
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p);
154154
bool _mi_free_delayed_block(mi_block_t* block);
155155
void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size);
@@ -1035,7 +1035,7 @@ static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
10351035
mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
10361036
void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
10371037
const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
1038-
memcpy(adst, asrc, n);
1038+
_mi_memcpy(adst, asrc, n);
10391039
}
10401040
#else
10411041
// Default fallback on `_mi_memcpy`

Include/internal/mimalloc/mimalloc-types.h

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,6 @@ typedef int32_t mi_ssize_t;
156156
#define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
157157
#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit
158158
#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
159-
#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)
160159

161160
// Maximum number of size classes. (spaced exponentially in 12.5% increments)
162161
#define MI_BIN_HUGE (73U)
@@ -175,7 +174,7 @@ typedef int32_t mi_ssize_t;
175174
#define MI_MAX_SLICE_OFFSET ((MI_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1)
176175

177176
// Used as a special value to encode block sizes in 32 bits.
178-
#define MI_HUGE_BLOCK_SIZE ((uint32_t)MI_HUGE_OBJ_SIZE_MAX)
177+
#define MI_HUGE_BLOCK_SIZE ((uint32_t)(2*MI_GiB))
179178

180179
// blocks up to this size are always allocated aligned
181180
#define MI_MAX_ALIGN_GUARANTEE (8*MI_MAX_ALIGN_SIZE)
@@ -581,9 +580,6 @@ typedef struct mi_segments_tld_s {
581580
size_t peak_count; // peak number of segments
582581
size_t current_size; // current size of all segments
583582
size_t peak_size; // peak size of all segments
584-
size_t cache_count; // number of segments in the cache
585-
size_t cache_size; // total size of all segments in the cache
586-
mi_segment_t* cache; // (small) cache of segments
587583
mi_stats_t* stats; // points to tld stats
588584
mi_os_tld_t* os; // points to os stats
589585
} mi_segments_tld_t;

Include/internal/mimalloc/mimalloc.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
88
#ifndef MIMALLOC_H
99
#define MIMALLOC_H
1010

11-
#define MI_MALLOC_VERSION 205 // major + 2 digits minor
11+
#define MI_MALLOC_VERSION 206 // major + 2 digits minor
1212

1313
// ------------------------------------------------------
1414
// Compiler specific attributes
@@ -256,6 +256,7 @@ typedef struct mi_heap_area_s {
256256
size_t committed; // current available bytes for this area
257257
size_t used; // number of allocated blocks
258258
size_t block_size; // size in bytes of each block
259+
size_t full_block_size; // size in bytes of a full block including padding and metadata.
259260
} mi_heap_area_t;
260261

261262
typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
@@ -315,7 +316,7 @@ typedef enum mi_option_e {
315316
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB) at startup
316317
mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
317318
mi_option_reserve_os_memory, // reserve specified amount of OS memory at startup
318-
mi_option_segment_cache,
319+
mi_option_deprecated_segment_cache,
319320
mi_option_page_reset,
320321
mi_option_abandoned_page_decommit,
321322
mi_option_deprecated_segment_reset,
@@ -326,6 +327,7 @@ typedef enum mi_option_e {
326327
mi_option_os_tag,
327328
mi_option_max_errors,
328329
mi_option_max_warnings,
330+
mi_option_max_segment_reclaim,
329331
mi_option_allow_decommit,
330332
mi_option_segment_decommit_delay,
331333
mi_option_decommit_extend_delay,
@@ -340,6 +342,7 @@ mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
340342
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
341343

342344
mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
345+
mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
343346
mi_decl_export void mi_option_set(mi_option_t option, long value);
344347
mi_decl_export void mi_option_set_default(mi_option_t option, long value);
345348

Include/internal/pycore_mimalloc.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@
173173
#define mi_new _Py_mi_new
174174
#define mi_new_reallocn _Py_mi_new_reallocn
175175
#define mi_new_realloc _Py_mi_new_realloc
176+
#define mi_option_get_clamp _Py_mi_option_get_clamp
176177
#define mi_option_disable _Py_mi_option_disable
177178
#define mi_option_enable _Py_mi_option_enable
178179
#define mi_option_get _Py_mi_option_get

Objects/mimalloc/alloc-override-osx.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) {
6464

6565
static void zone_free(malloc_zone_t* zone, void* p) {
6666
MI_UNUSED(zone);
67-
mi_free(p);
67+
mi_cfree(p);
6868
}
6969

7070
static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
@@ -373,7 +373,7 @@ __attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[]
373373
MI_INTERPOSE_MI(_malloc_fork_child),
374374
MI_INTERPOSE_MI(_malloc_fork_parent),
375375
MI_INTERPOSE_MI(_malloc_fork_prepare),
376-
376+
377377
MI_INTERPOSE_ZONE(zone_batch_free),
378378
MI_INTERPOSE_ZONE(zone_batch_malloc),
379379
MI_INTERPOSE_ZONE(zone_calloc),

Objects/mimalloc/alloc-override.c

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file
1616
#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32))
1717

1818
#if defined(__APPLE__)
19+
#include <AvailabilityMacros.h>
1920
mi_decl_externc void vfree(void* p);
2021
mi_decl_externc size_t malloc_size(const void* p);
2122
mi_decl_externc size_t malloc_good_size(size_t size);
@@ -77,7 +78,9 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
7778
MI_INTERPOSE_MI(valloc),
7879
MI_INTERPOSE_MI(malloc_size),
7980
MI_INTERPOSE_MI(malloc_good_size),
81+
#if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15
8082
MI_INTERPOSE_MI(aligned_alloc),
83+
#endif
8184
#ifdef MI_OSX_ZONE
8285
// we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely
8386
MI_INTERPOSE_MI(free),
@@ -91,15 +94,18 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
9194

9295
#ifdef __cplusplus
9396
extern "C" {
94-
void _ZdlPv(void* p); // delete
95-
void _ZdaPv(void* p); // delete[]
96-
void _ZdlPvm(void* p, size_t n); // delete
97-
void _ZdaPvm(void* p, size_t n); // delete[]
98-
void* _Znwm(size_t n); // new
99-
void* _Znam(size_t n); // new[]
100-
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
101-
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
102-
}
97+
#endif
98+
void _ZdlPv(void* p); // delete
99+
void _ZdaPv(void* p); // delete[]
100+
void _ZdlPvm(void* p, size_t n); // delete
101+
void _ZdaPvm(void* p, size_t n); // delete[]
102+
void* _Znwm(size_t n); // new
103+
void* _Znam(size_t n); // new[]
104+
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
105+
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
106+
#ifdef __cplusplus
107+
}
108+
#endif
103109
__attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) =
104110
{
105111
MI_INTERPOSE_FUN(_ZdlPv,mi_free),
@@ -111,7 +117,6 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
111117
MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),
112118
MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),
113119
};
114-
#endif // __cplusplus
115120

116121
#elif defined(_MSC_VER)
117122
// cannot override malloc unless using a dll.
@@ -161,7 +166,9 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
161166
void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
162167
void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
163168
void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
164-
169+
void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
170+
void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
171+
165172
void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
166173
void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
167174
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }

Objects/mimalloc/alloc-posix.c

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,17 +32,17 @@ terms of the MIT license. A copy of the license can be found in the file
3232
#endif
3333

3434

35-
size_t mi_malloc_size(const void* p) mi_attr_noexcept {
35+
mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
3636
//if (!mi_is_in_heap_region(p)) return 0;
3737
return mi_usable_size(p);
3838
}
3939

40-
size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
40+
mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
4141
//if (!mi_is_in_heap_region(p)) return 0;
4242
return mi_usable_size(p);
4343
}
4444

45-
size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
45+
mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
4646
return mi_good_size(size);
4747
}
4848

@@ -65,24 +65,24 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept
6565
return 0;
6666
}
6767

68-
mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
68+
mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
6969
void* p = mi_malloc_aligned(size, alignment);
7070
mi_assert_internal(((uintptr_t)p % alignment) == 0);
7171
return p;
7272
}
7373

74-
mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
74+
mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
7575
return mi_memalign( _mi_os_page_size(), size );
7676
}
7777

78-
mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
78+
mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
7979
size_t psize = _mi_os_page_size();
8080
if (size >= SIZE_MAX - psize) return NULL; // overflow
8181
size_t asize = _mi_align_up(size, psize);
8282
return mi_malloc_aligned(asize, psize);
8383
}
8484

85-
mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
85+
mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
8686
if (mi_unlikely((size&(alignment-1)) != 0)) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
8787
#if MI_DEBUG > 0
8888
_mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment);
@@ -95,13 +95,13 @@ mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_n
9595
return p;
9696
}
9797

98-
void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD
98+
mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD
9999
void* newp = mi_reallocn(p,count,size);
100100
if (newp==NULL) { errno = ENOMEM; }
101101
return newp;
102102
}
103103

104-
int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD
104+
mi_decl_nodiscard int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD
105105
mi_assert(p != NULL);
106106
if (p == NULL) {
107107
errno = EINVAL;
@@ -120,7 +120,7 @@ void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
120120
return res;
121121
}
122122

123-
mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept {
123+
mi_decl_nodiscard mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept {
124124
if (s==NULL) return NULL;
125125
size_t len;
126126
for(len = 0; s[len] != 0; len++) { }
@@ -132,7 +132,7 @@ mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noex
132132
return p;
133133
}
134134

135-
mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept {
135+
mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept {
136136
return (unsigned char*)mi_strdup((const char*)s);
137137
}
138138

@@ -172,10 +172,10 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name)
172172
#endif
173173
}
174174

175-
void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
175+
mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
176176
return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
177177
}
178178

179-
void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
179+
mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
180180
return mi_recalloc_aligned(p, newcount, size, alignment);
181181
}

0 commit comments

Comments
 (0)