Skip to content

Commit 505f5dc

Browse files
ramosian-glidertorvalds
ramosian-glider
authored and
torvalds
committed
mm, kasan: add GFP flags to KASAN API
Add GFP flags to KASAN hooks for future patches to use. This patch is based on the "mm: kasan: unified support for SLUB and SLAB allocators" patch originally prepared by Dmitry Chernenkov. Signed-off-by: Alexander Potapenko <[email protected]> Cc: Christoph Lameter <[email protected]> Cc: Pekka Enberg <[email protected]> Cc: David Rientjes <[email protected]> Cc: Joonsoo Kim <[email protected]> Cc: Andrey Konovalov <[email protected]> Cc: Dmitry Vyukov <[email protected]> Cc: Andrey Ryabinin <[email protected]> Cc: Steven Rostedt <[email protected]> Cc: Konstantin Serebryany <[email protected]> Cc: Dmitry Chernenkov <[email protected]> Signed-off-by: Andrew Morton <[email protected]> Signed-off-by: Linus Torvalds <[email protected]>
1 parent 7ed2f9e commit 505f5dc

File tree

8 files changed

+48
-42
lines changed

8 files changed

+48
-42
lines changed

‎include/linux/kasan.h

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,14 @@ void kasan_poison_slab(struct page *page);
5555
void kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
5656
void kasan_poison_object_data(struct kmem_cache *cache, void *object);
5757

58-
void kasan_kmalloc_large(const void *ptr, size_t size);
58+
void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
5959
void kasan_kfree_large(const void *ptr);
6060
void kasan_kfree(void *ptr);
61-
void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size);
62-
void kasan_krealloc(const void *object, size_t new_size);
61+
void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size,
62+
gfp_t flags);
63+
void kasan_krealloc(const void *object, size_t new_size, gfp_t flags);
6364

64-
void kasan_slab_alloc(struct kmem_cache *s, void *object);
65+
void kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
6566
void kasan_slab_free(struct kmem_cache *s, void *object);
6667

6768
struct kasan_cache {
@@ -94,14 +95,16 @@ static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
9495
static inline void kasan_poison_object_data(struct kmem_cache *cache,
9596
void *object) {}
9697

97-
static inline void kasan_kmalloc_large(void *ptr, size_t size) {}
98+
static inline void kasan_kmalloc_large(void *ptr, size_t size, gfp_t flags) {}
9899
static inline void kasan_kfree_large(const void *ptr) {}
99100
static inline void kasan_kfree(void *ptr) {}
100101
static inline void kasan_kmalloc(struct kmem_cache *s, const void *object,
101-
size_t size) {}
102-
static inline void kasan_krealloc(const void *object, size_t new_size) {}
102+
size_t size, gfp_t flags) {}
103+
static inline void kasan_krealloc(const void *object, size_t new_size,
104+
gfp_t flags) {}
103105

104-
static inline void kasan_slab_alloc(struct kmem_cache *s, void *object) {}
106+
static inline void kasan_slab_alloc(struct kmem_cache *s, void *object,
107+
gfp_t flags) {}
105108
static inline void kasan_slab_free(struct kmem_cache *s, void *object) {}
106109

107110
static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }

‎include/linux/slab.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,7 @@ static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s,
376376
{
377377
void *ret = kmem_cache_alloc(s, flags);
378378

379-
kasan_kmalloc(s, ret, size);
379+
kasan_kmalloc(s, ret, size, flags);
380380
return ret;
381381
}
382382

@@ -387,7 +387,7 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s,
387387
{
388388
void *ret = kmem_cache_alloc_node(s, gfpflags, node);
389389

390-
kasan_kmalloc(s, ret, size);
390+
kasan_kmalloc(s, ret, size, gfpflags);
391391
return ret;
392392
}
393393
#endif /* CONFIG_TRACING */

‎mm/kasan/kasan.c

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -434,9 +434,9 @@ struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
434434
}
435435
#endif
436436

437-
void kasan_slab_alloc(struct kmem_cache *cache, void *object)
437+
void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags)
438438
{
439-
kasan_kmalloc(cache, object, cache->object_size);
439+
kasan_kmalloc(cache, object, cache->object_size, flags);
440440
}
441441

442442
void kasan_slab_free(struct kmem_cache *cache, void *object)
@@ -462,7 +462,8 @@ void kasan_slab_free(struct kmem_cache *cache, void *object)
462462
kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
463463
}
464464

465-
void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size)
465+
void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size,
466+
gfp_t flags)
466467
{
467468
unsigned long redzone_start;
468469
unsigned long redzone_end;
@@ -491,7 +492,7 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size)
491492
}
492493
EXPORT_SYMBOL(kasan_kmalloc);
493494

494-
void kasan_kmalloc_large(const void *ptr, size_t size)
495+
void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
495496
{
496497
struct page *page;
497498
unsigned long redzone_start;
@@ -510,7 +511,7 @@ void kasan_kmalloc_large(const void *ptr, size_t size)
510511
KASAN_PAGE_REDZONE);
511512
}
512513

513-
void kasan_krealloc(const void *object, size_t size)
514+
void kasan_krealloc(const void *object, size_t size, gfp_t flags)
514515
{
515516
struct page *page;
516517

@@ -520,9 +521,9 @@ void kasan_krealloc(const void *object, size_t size)
520521
page = virt_to_head_page(object);
521522

522523
if (unlikely(!PageSlab(page)))
523-
kasan_kmalloc_large(object, size);
524+
kasan_kmalloc_large(object, size, flags);
524525
else
525-
kasan_kmalloc(page->slab_cache, object, size);
526+
kasan_kmalloc(page->slab_cache, object, size, flags);
526527
}
527528

528529
void kasan_kfree(void *ptr)

‎mm/mempool.c

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -112,12 +112,12 @@ static void kasan_poison_element(mempool_t *pool, void *element)
112112
kasan_free_pages(element, (unsigned long)pool->pool_data);
113113
}
114114

115-
static void kasan_unpoison_element(mempool_t *pool, void *element)
115+
static void kasan_unpoison_element(mempool_t *pool, void *element, gfp_t flags)
116116
{
117117
if (pool->alloc == mempool_alloc_slab)
118-
kasan_slab_alloc(pool->pool_data, element);
118+
kasan_slab_alloc(pool->pool_data, element, flags);
119119
if (pool->alloc == mempool_kmalloc)
120-
kasan_krealloc(element, (size_t)pool->pool_data);
120+
kasan_krealloc(element, (size_t)pool->pool_data, flags);
121121
if (pool->alloc == mempool_alloc_pages)
122122
kasan_alloc_pages(element, (unsigned long)pool->pool_data);
123123
}
@@ -130,12 +130,12 @@ static void add_element(mempool_t *pool, void *element)
130130
pool->elements[pool->curr_nr++] = element;
131131
}
132132

133-
static void *remove_element(mempool_t *pool)
133+
static void *remove_element(mempool_t *pool, gfp_t flags)
134134
{
135135
void *element = pool->elements[--pool->curr_nr];
136136

137137
BUG_ON(pool->curr_nr < 0);
138-
kasan_unpoison_element(pool, element);
138+
kasan_unpoison_element(pool, element, flags);
139139
check_element(pool, element);
140140
return element;
141141
}
@@ -154,7 +154,7 @@ void mempool_destroy(mempool_t *pool)
154154
return;
155155

156156
while (pool->curr_nr) {
157-
void *element = remove_element(pool);
157+
void *element = remove_element(pool, GFP_KERNEL);
158158
pool->free(element, pool->pool_data);
159159
}
160160
kfree(pool->elements);
@@ -250,7 +250,7 @@ int mempool_resize(mempool_t *pool, int new_min_nr)
250250
spin_lock_irqsave(&pool->lock, flags);
251251
if (new_min_nr <= pool->min_nr) {
252252
while (new_min_nr < pool->curr_nr) {
253-
element = remove_element(pool);
253+
element = remove_element(pool, GFP_KERNEL);
254254
spin_unlock_irqrestore(&pool->lock, flags);
255255
pool->free(element, pool->pool_data);
256256
spin_lock_irqsave(&pool->lock, flags);
@@ -347,7 +347,7 @@ void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
347347

348348
spin_lock_irqsave(&pool->lock, flags);
349349
if (likely(pool->curr_nr)) {
350-
element = remove_element(pool);
350+
element = remove_element(pool, gfp_temp);
351351
spin_unlock_irqrestore(&pool->lock, flags);
352352
/* paired with rmb in mempool_free(), read comment there */
353353
smp_wmb();

‎mm/slab.c

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3378,7 +3378,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
33783378
{
33793379
void *ret = slab_alloc(cachep, flags, _RET_IP_);
33803380

3381-
kasan_slab_alloc(cachep, ret);
3381+
kasan_slab_alloc(cachep, ret, flags);
33823382
trace_kmem_cache_alloc(_RET_IP_, ret,
33833383
cachep->object_size, cachep->size, flags);
33843384

@@ -3444,7 +3444,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
34443444

34453445
ret = slab_alloc(cachep, flags, _RET_IP_);
34463446

3447-
kasan_kmalloc(cachep, ret, size);
3447+
kasan_kmalloc(cachep, ret, size, flags);
34483448
trace_kmalloc(_RET_IP_, ret,
34493449
size, cachep->size, flags);
34503450
return ret;
@@ -3468,7 +3468,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
34683468
{
34693469
void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
34703470

3471-
kasan_slab_alloc(cachep, ret);
3471+
kasan_slab_alloc(cachep, ret, flags);
34723472
trace_kmem_cache_alloc_node(_RET_IP_, ret,
34733473
cachep->object_size, cachep->size,
34743474
flags, nodeid);
@@ -3486,7 +3486,8 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
34863486
void *ret;
34873487

34883488
ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3489-
kasan_kmalloc(cachep, ret, size);
3489+
3490+
kasan_kmalloc(cachep, ret, size, flags);
34903491
trace_kmalloc_node(_RET_IP_, ret,
34913492
size, cachep->size,
34923493
flags, nodeid);
@@ -3505,7 +3506,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
35053506
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
35063507
return cachep;
35073508
ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
3508-
kasan_kmalloc(cachep, ret, size);
3509+
kasan_kmalloc(cachep, ret, size, flags);
35093510

35103511
return ret;
35113512
}
@@ -3541,7 +3542,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
35413542
return cachep;
35423543
ret = slab_alloc(cachep, flags, caller);
35433544

3544-
kasan_kmalloc(cachep, ret, size);
3545+
kasan_kmalloc(cachep, ret, size, flags);
35453546
trace_kmalloc(caller, ret,
35463547
size, cachep->size, flags);
35473548

@@ -4323,7 +4324,7 @@ size_t ksize(const void *objp)
43234324
/* We assume that ksize callers could use the whole allocated area,
43244325
* so we need to unpoison this area.
43254326
*/
4326-
kasan_krealloc(objp, size);
4327+
kasan_krealloc(objp, size, GFP_NOWAIT);
43274328

43284329
return size;
43294330
}

‎mm/slab.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
405405
kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
406406
kmemleak_alloc_recursive(object, s->object_size, 1,
407407
s->flags, flags);
408-
kasan_slab_alloc(s, object);
408+
kasan_slab_alloc(s, object, flags);
409409
}
410410
memcg_kmem_put_cache(s);
411411
}

‎mm/slab_common.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,7 +1013,7 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
10131013
page = alloc_kmem_pages(flags, order);
10141014
ret = page ? page_address(page) : NULL;
10151015
kmemleak_alloc(ret, size, 1, flags);
1016-
kasan_kmalloc_large(ret, size);
1016+
kasan_kmalloc_large(ret, size, flags);
10171017
return ret;
10181018
}
10191019
EXPORT_SYMBOL(kmalloc_order);
@@ -1192,7 +1192,7 @@ static __always_inline void *__do_krealloc(const void *p, size_t new_size,
11921192
ks = ksize(p);
11931193

11941194
if (ks >= new_size) {
1195-
kasan_krealloc((void *)p, new_size);
1195+
kasan_krealloc((void *)p, new_size, flags);
11961196
return (void *)p;
11971197
}
11981198

‎mm/slub.c

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,7 +1313,7 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node,
13131313
static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
13141314
{
13151315
kmemleak_alloc(ptr, size, 1, flags);
1316-
kasan_kmalloc_large(ptr, size);
1316+
kasan_kmalloc_large(ptr, size, flags);
13171317
}
13181318

13191319
static inline void kfree_hook(const void *x)
@@ -2596,7 +2596,7 @@ void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
25962596
{
25972597
void *ret = slab_alloc(s, gfpflags, _RET_IP_);
25982598
trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2599-
kasan_kmalloc(s, ret, size);
2599+
kasan_kmalloc(s, ret, size, gfpflags);
26002600
return ret;
26012601
}
26022602
EXPORT_SYMBOL(kmem_cache_alloc_trace);
@@ -2624,7 +2624,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
26242624
trace_kmalloc_node(_RET_IP_, ret,
26252625
size, s->size, gfpflags, node);
26262626

2627-
kasan_kmalloc(s, ret, size);
2627+
kasan_kmalloc(s, ret, size, gfpflags);
26282628
return ret;
26292629
}
26302630
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
@@ -3182,7 +3182,8 @@ static void early_kmem_cache_node_alloc(int node)
31823182
init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
31833183
init_tracking(kmem_cache_node, n);
31843184
#endif
3185-
kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node));
3185+
kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
3186+
GFP_KERNEL);
31863187
init_kmem_cache_node(n);
31873188
inc_slabs_node(kmem_cache_node, node, page->objects);
31883189

@@ -3561,7 +3562,7 @@ void *__kmalloc(size_t size, gfp_t flags)
35613562

35623563
trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
35633564

3564-
kasan_kmalloc(s, ret, size);
3565+
kasan_kmalloc(s, ret, size, flags);
35653566

35663567
return ret;
35673568
}
@@ -3606,7 +3607,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
36063607

36073608
trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
36083609

3609-
kasan_kmalloc(s, ret, size);
3610+
kasan_kmalloc(s, ret, size, flags);
36103611

36113612
return ret;
36123613
}
@@ -3635,7 +3636,7 @@ size_t ksize(const void *object)
36353636
size_t size = __ksize(object);
36363637
/* We assume that ksize callers could use whole allocated area,
36373638
so we need unpoison this area. */
3638-
kasan_krealloc(object, size);
3639+
kasan_krealloc(object, size, GFP_NOWAIT);
36393640
return size;
36403641
}
36413642
EXPORT_SYMBOL(ksize);

0 commit comments

Comments
 (0)