Skip to content

Commit 4df48d3

Browse files
committed
Upgrade to mimalloc 2.12
Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags:
1 parent b437d0c commit 4df48d3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+6043
-3566
lines changed

Include/mimalloc/mimalloc-track.h

Lines changed: 0 additions & 62 deletions
This file was deleted.

Include/mimalloc/mimalloc.h

Lines changed: 45 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/* ----------------------------------------------------------------------------
2-
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
2+
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
33
This is free software; you can redistribute it and/or modify it under the
44
terms of the MIT license. A copy of the license can be found in the file
55
"LICENSE" at the root of this distribution.
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
88
#ifndef MIMALLOC_H
99
#define MIMALLOC_H
1010

11-
#define MI_MALLOC_VERSION 209 // major + 2 digits minor
11+
#define MI_MALLOC_VERSION 212 // major + 2 digits minor
1212

1313
// ------------------------------------------------------
1414
// Compiler specific attributes
@@ -284,7 +284,7 @@ mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node,
284284
mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
285285
mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
286286

287-
#if MI_MALLOC_VERSION >= 200
287+
#if MI_MALLOC_VERSION >= 182
288288
// Create a heap that only allocates in the specified arena
289289
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
290290
#endif
@@ -318,35 +318,40 @@ mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size
318318

319319
typedef enum mi_option_e {
320320
// stable options
321-
mi_option_show_errors,
322-
mi_option_show_stats,
323-
mi_option_verbose,
324-
// some of the following options are experimental
325-
// (deprecated options are kept for binary backward compatibility with v1.x versions)
326-
mi_option_eager_commit,
327-
mi_option_deprecated_eager_region_commit,
328-
mi_option_deprecated_reset_decommits,
329-
mi_option_large_os_pages, // use large (2MiB) OS pages, implies eager commit
330-
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB) at startup
321+
mi_option_show_errors, // print error messages
322+
mi_option_show_stats, // print statistics on termination
323+
mi_option_verbose, // print verbose messages
324+
// the following options are experimental (see src/options.h)
325+
mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1)
326+
mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
327+
mi_option_purge_decommits, // should a memory purge decommit (or only reset) (=1)
328+
mi_option_allow_large_os_pages, // allow large (2MiB) OS pages, implies eager commit
329+
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB/page) at startup
331330
mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
332-
mi_option_reserve_os_memory, // reserve specified amount of OS memory at startup
331+
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup
333332
mi_option_deprecated_segment_cache,
334-
mi_option_page_reset,
335-
mi_option_abandoned_page_decommit,
336-
mi_option_deprecated_segment_reset,
337-
mi_option_eager_commit_delay,
338-
mi_option_decommit_delay,
339-
mi_option_use_numa_nodes, // 0 = use available numa nodes, otherwise use at most N nodes.
340-
mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only reserved arenas)
341-
mi_option_os_tag,
342-
mi_option_max_errors,
343-
mi_option_max_warnings,
344-
mi_option_max_segment_reclaim,
345-
mi_option_allow_decommit,
346-
mi_option_segment_decommit_delay,
347-
mi_option_decommit_extend_delay,
348-
mi_option_destroy_on_exit,
349-
_mi_option_last
333+
mi_option_deprecated_page_reset,
334+
mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination
335+
mi_option_deprecated_segment_reset,
336+
mi_option_eager_commit_delay,
337+
mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all.
338+
mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes.
339+
mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
340+
mi_option_os_tag, // tag used for OS logging (macOS only for now)
341+
mi_option_max_errors, // issue at most N error messages
342+
mi_option_max_warnings, // issue at most N warning messages
343+
mi_option_max_segment_reclaim,
344+
mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe.
345+
mi_option_arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit)
346+
mi_option_arena_purge_mult,
347+
mi_option_purge_extend_delay,
348+
_mi_option_last,
349+
// legacy option names
350+
mi_option_large_os_pages = mi_option_allow_large_os_pages,
351+
mi_option_eager_region_commit = mi_option_arena_eager_commit,
352+
mi_option_reset_decommits = mi_option_purge_decommits,
353+
mi_option_reset_delay = mi_option_purge_delay,
354+
mi_option_abandoned_page_reset = mi_option_abandoned_page_purge
350355
} mi_option_t;
351356

352357

@@ -356,8 +361,9 @@ mi_decl_export void mi_option_disable(mi_option_t option);
356361
mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
357362
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
358363

359-
mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
360-
mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
364+
mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
365+
mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
366+
mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
361367
mi_decl_export void mi_option_set(mi_option_t option, long value);
362368
mi_decl_export void mi_option_set_default(mi_option_t option, long value);
363369

@@ -477,11 +483,13 @@ template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const
477483
template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
478484

479485

480-
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
486+
#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11
487+
#define MI_HAS_HEAP_STL_ALLOCATOR 1
488+
481489
#include <memory> // std::shared_ptr
482490

483491
// Common base class for STL allocators in a specific heap
484-
template<class T, bool destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
492+
template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
485493
using typename _mi_stl_allocator_common<T>::size_type;
486494
using typename _mi_stl_allocator_common<T>::value_type;
487495
using typename _mi_stl_allocator_common<T>::pointer;
@@ -500,18 +508,18 @@ template<class T, bool destroy> struct _mi_heap_stl_allocator_common : public _m
500508
#endif
501509

502510
void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
503-
template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, destroy>& x) const { return (this->heap == x.heap); }
511+
template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
504512

505513
protected:
506514
std::shared_ptr<mi_heap_t> heap;
507515
template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
508516

509517
_mi_heap_stl_allocator_common() {
510518
mi_heap_t* hp = mi_heap_new();
511-
this->heap.reset(hp, (destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */
519+
this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */
512520
}
513521
_mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
514-
template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, destroy>& x) mi_attr_noexcept : heap(x.heap) { }
522+
template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
515523

516524
private:
517525
static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } }

Include/mimalloc/mimalloc-atomic.h renamed to Include/mimalloc/mimalloc/atomic.h

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/* ----------------------------------------------------------------------------
2-
Copyright (c) 2018-2021 Microsoft Research, Daan Leijen
2+
Copyright (c) 2018-2023 Microsoft Research, Daan Leijen
33
This is free software; you can redistribute it and/or modify it under the
44
terms of the MIT license. A copy of the license can be found in the file
55
"LICENSE" at the root of this distribution.
@@ -39,7 +39,11 @@ terms of the MIT license. A copy of the license can be found in the file
3939
#include <stdatomic.h>
4040
#define mi_atomic(name) atomic_##name
4141
#define mi_memory_order(name) memory_order_##name
42-
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
42+
#if !defined(ATOMIC_VAR_INIT) || (__STDC_VERSION__ >= 201710L) // c17, see issue #735
43+
#define MI_ATOMIC_VAR_INIT(x) x
44+
#else
45+
#define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
46+
#endif
4347
#endif
4448

4549
// Various defines for all used memory orders in mimalloc
@@ -113,11 +117,13 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
113117
}
114118

115119
// Used by timers
116-
#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
117-
#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
118-
#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
119-
#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
120+
#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
121+
#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
122+
#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
123+
#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
120124

125+
#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d)
126+
#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i)
121127

122128

123129
#elif defined(_MSC_VER)
@@ -245,6 +251,21 @@ static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t
245251
} while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
246252
}
247253

254+
static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
255+
mi_atomic_addi64_relaxed(p, i);
256+
}
257+
258+
static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
259+
int64_t read = _InterlockedCompareExchange64(p, des, *exp);
260+
if (read == *exp) {
261+
return true;
262+
}
263+
else {
264+
*exp = read;
265+
return false;
266+
}
267+
}
268+
248269
// The pointer macros cast to `uintptr_t`.
249270
#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
250271
#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
@@ -275,6 +296,26 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
275296
return (intptr_t)mi_atomic_addi(p, -sub);
276297
}
277298

299+
typedef _Atomic(uintptr_t) mi_atomic_once_t;
300+
301+
// Returns true only on the first invocation
302+
static inline bool mi_atomic_once( mi_atomic_once_t* once ) {
303+
if (mi_atomic_load_relaxed(once) != 0) return false; // quick test
304+
uintptr_t expected = 0;
305+
return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1
306+
}
307+
308+
typedef _Atomic(uintptr_t) mi_atomic_guard_t;
309+
310+
// Allows only one thread to execute at a time
311+
#define mi_atomic_guard(guard) \
312+
uintptr_t _mi_guard_expected = 0; \
313+
for(bool _mi_guard_once = true; \
314+
_mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \
315+
(mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
316+
317+
318+
278319
// Yield
279320
#if defined(__cplusplus)
280321
#include <thread>
@@ -294,7 +335,7 @@ static inline void mi_atomic_yield(void) {
294335
}
295336
#elif (defined(__GNUC__) || defined(__clang__)) && \
296337
(defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \
297-
defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__))
338+
defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)) || defined(__POWERPC__)
298339
#if defined(__x86_64__) || defined(__i386__)
299340
static inline void mi_atomic_yield(void) {
300341
__asm__ volatile ("pause" ::: "memory");
@@ -307,10 +348,16 @@ static inline void mi_atomic_yield(void) {
307348
static inline void mi_atomic_yield(void) {
308349
__asm__ volatile("yield" ::: "memory");
309350
}
310-
#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)
351+
#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__)
352+
#ifdef __APPLE__
353+
static inline void mi_atomic_yield(void) {
354+
__asm__ volatile ("or r27,r27,r27" ::: "memory");
355+
}
356+
#else
311357
static inline void mi_atomic_yield(void) {
312358
__asm__ __volatile__ ("or 27,27,27" ::: "memory");
313359
}
360+
#endif
314361
#elif defined(__armel__) || defined(__ARMEL__)
315362
static inline void mi_atomic_yield(void) {
316363
__asm__ volatile ("nop" ::: "memory");

0 commit comments

Comments
 (0)