Skip to content

Commit a9b3c35

Browse files
kevin-brodsky-armakpm00
kevin-brodsky-arm
authored and
akpm00
committed
asm-generic: pgalloc: provide generic __pgd_{alloc,free}
We already have a generic implementation of alloc/free up to P4D level, as well as pgd_free(). Let's finish the work and add a generic PGD-level alloc helper as well. Unlike at lower levels, almost all architectures need some specific magic at PGD level (typically initialising PGD entries), so introducing a generic pgd_alloc() isn't worth it. Instead we introduce two new helpers, __pgd_alloc() and __pgd_free(), and make use of them in the arch-specific pgd_alloc() and pgd_free() wherever possible. To accommodate as many arch as possible, __pgd_alloc() takes a page allocation order. Because pagetable_alloc() allocates zeroed pages, explicit zeroing in pgd_alloc() becomes redundant and we can get rid of it. Some trivial implementations of pgd_free() also become unnecessary once __pgd_alloc() is used; remove them. Another small improvement is consistent accounting of PGD pages by using GFP_PGTABLE_{USER,KERNEL} as appropriate. Not all PGD allocations can be handled by the generic helpers. In particular, multiple architectures allocate PGDs from a kmem_cache, and those PGDs may not be page-sized. Link: https://lkml.kernel.org/r/[email protected] Signed-off-by: Kevin Brodsky <[email protected]> Acked-by: Dave Hansen <[email protected]> Acked-by: Qi Zheng <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: Catalin Marinas <[email protected]> Cc: Ingo Molnar <[email protected]> Cc: Linus Walleij <[email protected]> Cc: Matthew Wilcox (Oracle) <[email protected]> Cc: Mike Rapoport (Microsoft) <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Ryan Roberts <[email protected]> Cc: Thomas Gleixner <[email protected]> Cc: Will Deacon <[email protected]> Signed-off-by: Andrew Morton <[email protected]>
1 parent 9477102 commit a9b3c35

File tree

19 files changed

+65
-80
lines changed

19 files changed

+65
-80
lines changed

‎arch/alpha/mm/init.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ pgd_alloc(struct mm_struct *mm)
4242
{
4343
pgd_t *ret, *init;
4444

45-
ret = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
45+
ret = __pgd_alloc(mm, 0);
4646
init = pgd_offset(&init_mm, 0UL);
4747
if (ret) {
4848
#ifdef CONFIG_ALPHA_LARGE_VMALLOC

‎arch/arc/include/asm/pgalloc.h

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,19 +53,14 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte_
5353

5454
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
5555
{
56-
pgd_t *ret = (pgd_t *) __get_free_page(GFP_KERNEL);
56+
pgd_t *ret = __pgd_alloc(mm, 0);
5757

5858
if (ret) {
5959
int num, num2;
60-
num = USER_PTRS_PER_PGD + USER_KERNEL_GUTTER / PGDIR_SIZE;
61-
memzero(ret, num * sizeof(pgd_t));
6260

61+
num = USER_PTRS_PER_PGD + USER_KERNEL_GUTTER / PGDIR_SIZE;
6362
num2 = VMALLOC_SIZE / PGDIR_SIZE;
6463
memcpy(ret + num, swapper_pg_dir + num, num2 * sizeof(pgd_t));
65-
66-
memzero(ret + num + num2,
67-
(PTRS_PER_PGD - num - num2) * sizeof(pgd_t));
68-
6964
}
7065
return ret;
7166
}

‎arch/arm/mm/pgd.c

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
#include "mm.h"
1818

1919
#ifdef CONFIG_ARM_LPAE
20-
#define _pgd_alloc(mm) kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL)
20+
#define _pgd_alloc(mm) kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL | __GFP_ZERO)
2121
#define _pgd_free(mm, pgd) kfree(pgd)
2222
#else
23-
#define _pgd_alloc(mm) (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
24-
#define _pgd_free(mm, pgd) free_pages((unsigned long)pgd, 2)
23+
#define _pgd_alloc(mm) __pgd_alloc(mm, 2)
24+
#define _pgd_free(mm, pgd) __pgd_free(mm, pgd)
2525
#endif
2626

2727
/*
@@ -39,8 +39,6 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
3939
if (!new_pgd)
4040
goto no_pgd;
4141

42-
memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
43-
4442
/*
4543
* Copy over the kernel and IO PGD entries
4644
*/

‎arch/arm64/mm/pgd.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,15 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
3333
gfp_t gfp = GFP_PGTABLE_USER;
3434

3535
if (pgdir_is_page_size())
36-
return (pgd_t *)__get_free_page(gfp);
36+
return __pgd_alloc(mm, 0);
3737
else
3838
return kmem_cache_alloc(pgd_cache, gfp);
3939
}
4040

4141
void pgd_free(struct mm_struct *mm, pgd_t *pgd)
4242
{
4343
if (pgdir_is_page_size())
44-
free_page((unsigned long)pgd);
44+
__pgd_free(mm, pgd);
4545
else
4646
kmem_cache_free(pgd_cache, pgd);
4747
}

‎arch/csky/include/asm/pgalloc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
4444
pgd_t *ret;
4545
pgd_t *init;
4646

47-
ret = (pgd_t *) __get_free_page(GFP_KERNEL);
47+
ret = __pgd_alloc(mm, 0);
4848
if (ret) {
4949
init = pgd_offset(&init_mm, 0UL);
5050
pgd_init((unsigned long *)ret);

‎arch/hexagon/include/asm/pgalloc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
2222
{
2323
pgd_t *pgd;
2424

25-
pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
25+
pgd = __pgd_alloc(mm, 0);
2626

2727
/*
2828
* There may be better ways to do this, but to ensure

‎arch/loongarch/mm/pgtable.c

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,10 @@ EXPORT_SYMBOL(tlb_virt_to_page);
2323

2424
pgd_t *pgd_alloc(struct mm_struct *mm)
2525
{
26-
pgd_t *init, *ret = NULL;
27-
struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0);
26+
pgd_t *init, *ret;
2827

29-
if (ptdesc) {
30-
ret = (pgd_t *)ptdesc_address(ptdesc);
28+
ret = __pgd_alloc(mm, 0);
29+
if (ret) {
3130
init = pgd_offset(&init_mm, 0UL);
3231
pgd_init(ret);
3332
memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,

‎arch/m68k/include/asm/sun3_pgalloc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ static inline pgd_t * pgd_alloc(struct mm_struct *mm)
4343
{
4444
pgd_t *new_pgd;
4545

46-
new_pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL);
46+
new_pgd = __pgd_alloc(mm, 0);
4747
memcpy(new_pgd, swapper_pg_dir, PAGE_SIZE);
4848
memset(new_pgd, 0, (PAGE_OFFSET >> PGDIR_SHIFT));
4949
return new_pgd;

‎arch/microblaze/include/asm/pgalloc.h

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,7 @@
2121

2222
extern void __bad_pte(pmd_t *pmd);
2323

24-
static inline pgd_t *get_pgd(void)
25-
{
26-
return (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, 0);
27-
}
28-
29-
#define pgd_alloc(mm) get_pgd()
24+
#define pgd_alloc(mm) __pgd_alloc(mm, 0)
3025

3126
extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
3227

‎arch/mips/include/asm/pgalloc.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
#define __HAVE_ARCH_PMD_ALLOC_ONE
1717
#define __HAVE_ARCH_PUD_ALLOC_ONE
18-
#define __HAVE_ARCH_PGD_FREE
1918
#include <asm-generic/pgalloc.h>
2019

2120
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
@@ -49,11 +48,6 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
4948
extern void pgd_init(void *addr);
5049
extern pgd_t *pgd_alloc(struct mm_struct *mm);
5150

52-
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
53-
{
54-
pagetable_free(virt_to_ptdesc(pgd));
55-
}
56-
5751
#define __pte_free_tlb(tlb, pte, address) \
5852
do { \
5953
pagetable_dtor(page_ptdesc(pte)); \

‎arch/mips/mm/pgtable.c

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,10 @@
1010

1111
pgd_t *pgd_alloc(struct mm_struct *mm)
1212
{
13-
pgd_t *init, *ret = NULL;
14-
struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM,
15-
PGD_TABLE_ORDER);
13+
pgd_t *init, *ret;
1614

17-
if (ptdesc) {
18-
ret = ptdesc_address(ptdesc);
15+
ret = __pgd_alloc(mm, PGD_TABLE_ORDER);
16+
if (ret) {
1917
init = pgd_offset(&init_mm, 0UL);
2018
pgd_init(ret);
2119
memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,

‎arch/nios2/mm/pgtable.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <linux/sched.h>
1212

1313
#include <asm/cpuinfo.h>
14+
#include <asm/pgalloc.h>
1415

1516
/* pteaddr:
1617
* ptbase | vpn* | zero
@@ -54,7 +55,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
5455
{
5556
pgd_t *ret, *init;
5657

57-
ret = (pgd_t *) __get_free_page(GFP_KERNEL);
58+
ret = __pgd_alloc(mm, 0);
5859
if (ret) {
5960
init = pgd_offset(&init_mm, 0UL);
6061
pgd_init(ret);

‎arch/openrisc/include/asm/pgalloc.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,13 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
4141
*/
4242
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
4343
{
44-
pgd_t *ret = (pgd_t *)__get_free_page(GFP_KERNEL);
44+
pgd_t *ret = __pgd_alloc(mm, 0);
4545

46-
if (ret) {
47-
memset(ret, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
46+
if (ret)
4847
memcpy(ret + USER_PTRS_PER_PGD,
4948
swapper_pg_dir + USER_PTRS_PER_PGD,
5049
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
5150

52-
}
5351
return ret;
5452
}
5553

‎arch/parisc/include/asm/pgalloc.h

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,12 @@
1111
#include <asm/cache.h>
1212

1313
#define __HAVE_ARCH_PMD_ALLOC_ONE
14-
#define __HAVE_ARCH_PGD_FREE
1514
#include <asm-generic/pgalloc.h>
1615

1716
/* Allocate the top level pgd (page directory) */
1817
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
1918
{
20-
pgd_t *pgd;
21-
22-
pgd = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_TABLE_ORDER);
23-
if (unlikely(pgd == NULL))
24-
return NULL;
25-
26-
memset(pgd, 0, PAGE_SIZE << PGD_TABLE_ORDER);
27-
28-
return pgd;
29-
}
30-
31-
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
32-
{
33-
free_pages((unsigned long)pgd, PGD_TABLE_ORDER);
19+
return __pgd_alloc(mm, PGD_TABLE_ORDER);
3420
}
3521

3622
#if CONFIG_PGTABLE_LEVELS == 3

‎arch/riscv/include/asm/pgalloc.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,8 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
130130
{
131131
pgd_t *pgd;
132132

133-
pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
133+
pgd = __pgd_alloc(mm, 0);
134134
if (likely(pgd != NULL)) {
135-
memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
136135
/* Copy kernel mappings */
137136
sync_kernel_mappings(pgd);
138137
}

‎arch/um/kernel/mem.c

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -214,14 +214,13 @@ void free_initmem(void)
214214

215215
pgd_t *pgd_alloc(struct mm_struct *mm)
216216
{
217-
pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
217+
pgd_t *pgd = __pgd_alloc(mm, 0);
218218

219-
if (pgd) {
220-
memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
219+
if (pgd)
221220
memcpy(pgd + USER_PTRS_PER_PGD,
222221
swapper_pg_dir + USER_PTRS_PER_PGD,
223222
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
224-
}
223+
225224
return pgd;
226225
}
227226

‎arch/x86/mm/pgtable.c

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -397,15 +397,14 @@ void __init pgtable_cache_init(void)
397397
SLAB_PANIC, NULL);
398398
}
399399

400-
static inline pgd_t *_pgd_alloc(void)
400+
static inline pgd_t *_pgd_alloc(struct mm_struct *mm)
401401
{
402402
/*
403403
* If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain.
404404
* We allocate one page for pgd.
405405
*/
406406
if (!SHARED_KERNEL_PMD)
407-
return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER,
408-
PGD_ALLOCATION_ORDER);
407+
return __pgd_alloc(mm, PGD_ALLOCATION_ORDER);
409408

410409
/*
411410
* Now PAE kernel is not running as a Xen domain. We can allocate
@@ -414,24 +413,23 @@ static inline pgd_t *_pgd_alloc(void)
414413
return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER);
415414
}
416415

417-
static inline void _pgd_free(pgd_t *pgd)
416+
static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd)
418417
{
419418
if (!SHARED_KERNEL_PMD)
420-
free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER);
419+
__pgd_free(mm, pgd);
421420
else
422421
kmem_cache_free(pgd_cache, pgd);
423422
}
424423
#else
425424

426-
static inline pgd_t *_pgd_alloc(void)
425+
static inline pgd_t *_pgd_alloc(struct mm_struct *mm)
427426
{
428-
return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER,
429-
PGD_ALLOCATION_ORDER);
427+
return __pgd_alloc(mm, PGD_ALLOCATION_ORDER);
430428
}
431429

432-
static inline void _pgd_free(pgd_t *pgd)
430+
static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd)
433431
{
434-
free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER);
432+
__pgd_free(mm, pgd);
435433
}
436434
#endif /* CONFIG_X86_PAE */
437435

@@ -441,7 +439,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
441439
pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS];
442440
pmd_t *pmds[MAX_PREALLOCATED_PMDS];
443441

444-
pgd = _pgd_alloc();
442+
pgd = _pgd_alloc(mm);
445443

446444
if (pgd == NULL)
447445
goto out;
@@ -484,7 +482,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
484482
if (sizeof(pmds) != 0)
485483
free_pmds(mm, pmds, PREALLOCATED_PMDS);
486484
out_free_pgd:
487-
_pgd_free(pgd);
485+
_pgd_free(mm, pgd);
488486
out:
489487
return NULL;
490488
}
@@ -494,7 +492,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd)
494492
pgd_mop_up_pmds(mm, pgd);
495493
pgd_dtor(pgd);
496494
paravirt_pgd_free(mm, pgd);
497-
_pgd_free(pgd);
495+
_pgd_free(mm, pgd);
498496
}
499497

500498
/*

‎arch/xtensa/include/asm/pgalloc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
static inline pgd_t*
3030
pgd_alloc(struct mm_struct *mm)
3131
{
32-
return (pgd_t*) __get_free_page(GFP_KERNEL | __GFP_ZERO);
32+
return __pgd_alloc(mm, 0);
3333
}
3434

3535
static inline void ptes_clear(pte_t *ptep)

‎include/asm-generic/pgalloc.h

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,10 +258,35 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
258258

259259
#endif /* CONFIG_PGTABLE_LEVELS > 4 */
260260

261+
static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order)
262+
{
263+
gfp_t gfp = GFP_PGTABLE_USER;
264+
struct ptdesc *ptdesc;
265+
266+
if (mm == &init_mm)
267+
gfp = GFP_PGTABLE_KERNEL;
268+
gfp &= ~__GFP_HIGHMEM;
269+
270+
ptdesc = pagetable_alloc_noprof(gfp, order);
271+
if (!ptdesc)
272+
return NULL;
273+
274+
return ptdesc_address(ptdesc);
275+
}
276+
#define __pgd_alloc(...) alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))
277+
278+
static inline void __pgd_free(struct mm_struct *mm, pgd_t *pgd)
279+
{
280+
struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
281+
282+
BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
283+
pagetable_free(ptdesc);
284+
}
285+
261286
#ifndef __HAVE_ARCH_PGD_FREE
262287
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
263288
{
264-
pagetable_free(virt_to_ptdesc(pgd));
289+
__pgd_free(mm, pgd);
265290
}
266291
#endif
267292

0 commit comments

Comments
 (0)