@@ -72,6 +72,14 @@ namespace {
72
72
struct CachedBlock {
73
73
static constexpr u16 CacheIndexMax = UINT16_MAX;
74
74
static constexpr u16 InvalidEntry = CacheIndexMax;
75
+ // * MaxReleasedCachePages default is currently 4
76
+ // - We arrived at this value after noticing that mapping
77
+ // in larger memory regions performs better than releasing
78
+ // memory and forcing a cache hit. According to the data,
79
+ // it suggests that beyond 4 pages, the release execution time is
80
+ // longer than the map execution time. In this way, the default
81
+ // is dependent on the platform.
82
+ static constexpr uptr MaxReleasedCachePages = 4U ;
75
83
76
84
uptr CommitBase = 0 ;
77
85
uptr CommitSize = 0 ;
@@ -90,8 +98,9 @@ struct CachedBlock {
90
98
template <typename Config> class MapAllocatorNoCache {
91
99
public:
92
100
void init (UNUSED s32 ReleaseToOsInterval) {}
93
- CachedBlock retrieve (UNUSED uptr Size, UNUSED uptr Alignment,
94
- UNUSED uptr HeadersSize, UNUSED uptr &EntryHeaderPos) {
101
+ CachedBlock retrieve (UNUSED uptr MaxAllowedFragmentedBytes, UNUSED uptr Size,
102
+ UNUSED uptr Alignment, UNUSED uptr HeadersSize,
103
+ UNUSED uptr &EntryHeaderPos) {
95
104
return {};
96
105
}
97
106
void store (UNUSED Options Options, UNUSED uptr CommitBase,
@@ -121,7 +130,7 @@ template <typename Config> class MapAllocatorNoCache {
121
130
}
122
131
};
123
132
124
- static const uptr MaxUnusedCachePages = 4U ;
133
+ static const uptr MaxUnreleasedCachePages = 4U ;
125
134
126
135
template <typename Config>
127
136
bool mapSecondary (const Options &Options, uptr CommitBase, uptr CommitSize,
@@ -151,9 +160,11 @@ bool mapSecondary(const Options &Options, uptr CommitBase, uptr CommitSize,
151
160
}
152
161
}
153
162
154
- const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * PageSize;
155
- if (useMemoryTagging<Config>(Options) && CommitSize > MaxUnusedCacheBytes) {
156
- const uptr UntaggedPos = Max (AllocPos, CommitBase + MaxUnusedCacheBytes);
163
+ const uptr MaxUnreleasedCacheBytes = MaxUnreleasedCachePages * PageSize;
164
+ if (useMemoryTagging<Config>(Options) &&
165
+ CommitSize > MaxUnreleasedCacheBytes) {
166
+ const uptr UntaggedPos =
167
+ Max (AllocPos, CommitBase + MaxUnreleasedCacheBytes);
157
168
return MemMap.remap (CommitBase, UntaggedPos - CommitBase, " scudo:secondary" ,
158
169
MAP_MEMTAG | Flags) &&
159
170
MemMap.remap (UntaggedPos, CommitBase + CommitSize - UntaggedPos,
@@ -334,61 +345,105 @@ class MapAllocatorCache {
334
345
}
335
346
}
336
347
337
- CachedBlock retrieve (uptr Size, uptr Alignment, uptr HeadersSize,
338
- uptr &EntryHeaderPos) EXCLUDES(Mutex) {
348
+ CachedBlock retrieve (uptr MaxAllowedFragmentedPages, uptr Size,
349
+ uptr Alignment, uptr HeadersSize, uptr &EntryHeaderPos)
350
+ EXCLUDES(Mutex) {
339
351
const uptr PageSize = getPageSizeCached ();
340
352
// 10% of the requested size proved to be the optimal choice for
341
353
// retrieving cached blocks after testing several options.
342
354
constexpr u32 FragmentedBytesDivisor = 10 ;
343
- bool Found = false ;
355
+ bool FoundOptimalFit = false ;
344
356
CachedBlock Entry;
345
357
EntryHeaderPos = 0 ;
346
358
{
347
359
ScopedLock L (Mutex);
348
360
CallsToRetrieve++;
349
361
if (EntriesCount == 0 )
350
362
return {};
351
- u32 OptimalFitIndex = 0 ;
363
+ u16 RetrievedIndex = CachedBlock::InvalidEntry ;
352
364
uptr MinDiff = UINTPTR_MAX;
353
- for (u32 I = LRUHead; I != CachedBlock::InvalidEntry;
365
+
366
+ // Since allocation sizes don't always match cached memory chunk sizes
367
+ // we allow some memory to be unused (called fragmented bytes). The
368
+ // amount of unused bytes is exactly EntryHeaderPos - CommitBase.
369
+ //
370
+ // CommitBase CommitBase + CommitSize
371
+ // V V
372
+ // +---+------------+-----------------+---+
373
+ // | | | | |
374
+ // +---+------------+-----------------+---+
375
+ // ^ ^ ^
376
+ // Guard EntryHeaderPos Guard-page-end
377
+ // page-begin
378
+ //
379
+ // [EntryHeaderPos, CommitBase + CommitSize) contains the user data as
380
+ // well as the header metadata. If EntryHeaderPos - CommitBase exceeds
381
+ // MaxAllowedFragmentedPages * PageSize, the cached memory chunk is
382
+ // not considered valid for retrieval.
383
+ for (u16 I = LRUHead; I != CachedBlock::InvalidEntry;
354
384
I = Entries[I].Next ) {
355
385
const uptr CommitBase = Entries[I].CommitBase ;
356
386
const uptr CommitSize = Entries[I].CommitSize ;
357
387
const uptr AllocPos =
358
388
roundDown (CommitBase + CommitSize - Size, Alignment);
359
389
const uptr HeaderPos = AllocPos - HeadersSize;
360
- if (HeaderPos > CommitBase + CommitSize)
390
+ if (HeaderPos > CommitBase + CommitSize || HeaderPos < CommitBase )
361
391
continue ;
362
- if (HeaderPos < CommitBase ||
363
- AllocPos > CommitBase + PageSize * MaxUnusedCachePages) {
392
+
393
+ const uptr Diff = roundDown (HeaderPos, PageSize) - CommitBase;
394
+
395
+ if (Diff > MaxAllowedFragmentedPages * PageSize || Diff >= MinDiff)
364
396
continue ;
365
- }
366
- Found = true ;
367
- const uptr Diff = HeaderPos - CommitBase ;
368
- // immediately use a cached block if it's size is close enough to the
369
- // requested size.
370
- const uptr MaxAllowedFragmentedBytes =
397
+
398
+ MinDiff = Diff ;
399
+ RetrievedIndex = I ;
400
+ EntryHeaderPos = HeaderPos;
401
+
402
+ const uptr OptimalFitThesholdBytes =
371
403
(CommitBase + CommitSize - HeaderPos) / FragmentedBytesDivisor;
372
- if (Diff <= MaxAllowedFragmentedBytes) {
373
- OptimalFitIndex = I;
374
- EntryHeaderPos = HeaderPos;
404
+ if (Diff <= OptimalFitThesholdBytes) {
405
+ FoundOptimalFit = true ;
375
406
break ;
376
407
}
377
- // keep track of the smallest cached block
378
- // that is greater than (AllocSize + HeaderSize)
379
- if (Diff > MinDiff)
380
- continue ;
381
- OptimalFitIndex = I;
382
- MinDiff = Diff;
383
- EntryHeaderPos = HeaderPos;
384
408
}
385
- if (Found ) {
386
- Entry = Entries[OptimalFitIndex ];
387
- remove (OptimalFitIndex );
409
+ if (RetrievedIndex != CachedBlock::InvalidEntry ) {
410
+ Entry = Entries[RetrievedIndex ];
411
+ remove (RetrievedIndex );
388
412
SuccessfulRetrieves++;
389
413
}
390
414
}
391
415
416
+ // The difference between the retrieved memory chunk and the request
417
+ // size is at most MaxAllowedFragmentedPages
418
+ //
419
+ // / MaxAllowedFragmentedPages * PageSize \
420
+ // +--------------------------+-------------+
421
+ // | | |
422
+ // +--------------------------+-------------+
423
+ // \ Bytes to be released / ^
424
+ // |
425
+ // (may or may not be committed)
426
+ //
427
+ // The maximum number of bytes released to the OS is capped by
428
+ // MaxReleasedCachePages
429
+ //
430
+ // TODO : Consider making MaxReleasedCachePages configurable since
431
+ // the release to OS API can vary across systems.
432
+ if (!FoundOptimalFit && Entry.Time != 0 ) {
433
+ const uptr FragmentedBytes =
434
+ roundDown (EntryHeaderPos, PageSize) - Entry.CommitBase ;
435
+ const uptr MaxUnreleasedCacheBytes = MaxUnreleasedCachePages * PageSize;
436
+ if (FragmentedBytes > MaxUnreleasedCacheBytes) {
437
+ const uptr MaxReleasedCacheBytes =
438
+ CachedBlock::MaxReleasedCachePages * PageSize;
439
+ uptr BytesToRelease =
440
+ roundUp (Min<uptr>(MaxReleasedCacheBytes,
441
+ FragmentedBytes - MaxUnreleasedCacheBytes),
442
+ PageSize);
443
+ Entry.MemMap .releaseAndZeroPagesToOS (Entry.CommitBase , BytesToRelease);
444
+ }
445
+ }
446
+
392
447
return Entry;
393
448
}
394
449
@@ -659,8 +714,18 @@ MapAllocator<Config>::tryAllocateFromCache(const Options &Options, uptr Size,
659
714
FillContentsMode FillContents) {
660
715
CachedBlock Entry;
661
716
uptr EntryHeaderPos;
717
+ uptr MaxAllowedFragmentedPages;
718
+
719
+ if (LIKELY (!useMemoryTagging<Config>(Options))) {
720
+ MaxAllowedFragmentedPages =
721
+ MaxUnreleasedCachePages + CachedBlock::MaxReleasedCachePages;
722
+
723
+ } else {
724
+ MaxAllowedFragmentedPages = MaxUnreleasedCachePages;
725
+ }
662
726
663
- Entry = Cache.retrieve (Size, Alignment, getHeadersSize (), EntryHeaderPos);
727
+ Entry = Cache.retrieve (MaxAllowedFragmentedPages, Size, Alignment,
728
+ getHeadersSize (), EntryHeaderPos);
664
729
if (!Entry.isValid ())
665
730
return nullptr ;
666
731
0 commit comments