Skip to content

Commit 1355c31

Browse files
rppttorvalds
rppt
authored and
torvalds
committed
asm-generic: pgalloc: provide generic pmd_alloc_one() and pmd_free_one()
For most architectures that support >2 levels of page tables, pmd_alloc_one() is a wrapper for __get_free_pages(), sometimes with __GFP_ZERO and sometimes followed by memset(0) instead. More elaborate versions on arm64 and x86 account memory for the user page tables and call to pgtable_pmd_page_ctor() as the part of PMD page initialization. Move the arm64 version to include/asm-generic/pgalloc.h and use the generic version on several architectures. The pgtable_pmd_page_ctor() is a NOP when ARCH_ENABLE_SPLIT_PMD_PTLOCK is not enabled, so there is no functional change for most architectures except of the addition of __GFP_ACCOUNT for allocation of user page tables. The pmd_free() is a wrapper for free_page() in all the cases, so no functional change here. Signed-off-by: Mike Rapoport <[email protected]> Signed-off-by: Andrew Morton <[email protected]> Reviewed-by: Pekka Enberg <[email protected]> Cc: Matthew Wilcox <[email protected]> Cc: Abdul Haleem <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: Arnd Bergmann <[email protected]> Cc: Christophe Leroy <[email protected]> Cc: Joerg Roedel <[email protected]> Cc: Joerg Roedel <[email protected]> Cc: Max Filippov <[email protected]> Cc: Peter Zijlstra (Intel) <[email protected]> Cc: Satheesh Rajendran <[email protected]> Cc: Stafford Horne <[email protected]> Cc: Stephen Rothwell <[email protected]> Cc: Steven Rostedt <[email protected]> Cc: Geert Uytterhoeven <[email protected]> Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Linus Torvalds <[email protected]>
1 parent 7278914 commit 1355c31

File tree

13 files changed

+55
-135
lines changed

13 files changed

+55
-135
lines changed

‎arch/alpha/include/asm/pgalloc.h

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
#include <linux/mm.h>
66
#include <linux/mmzone.h>
77

8-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
8+
#include <asm-generic/pgalloc.h>
99

1010
/*
1111
* Allocate and free page tables. The xxx_kernel() versions are
@@ -40,17 +40,4 @@ pgd_free(struct mm_struct *mm, pgd_t *pgd)
4040
free_page((unsigned long)pgd);
4141
}
4242

43-
static inline pmd_t *
44-
pmd_alloc_one(struct mm_struct *mm, unsigned long address)
45-
{
46-
pmd_t *ret = (pmd_t *)__get_free_page(GFP_PGTABLE_USER);
47-
return ret;
48-
}
49-
50-
static inline void
51-
pmd_free(struct mm_struct *mm, pmd_t *pmd)
52-
{
53-
free_page((unsigned long)pmd);
54-
}
55-
5643
#endif /* _ALPHA_PGALLOC_H */

‎arch/arm/include/asm/pgalloc.h

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,6 @@
2222

2323
#ifdef CONFIG_ARM_LPAE
2424

25-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
26-
{
27-
return (pmd_t *)get_zeroed_page(GFP_KERNEL);
28-
}
29-
30-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
31-
{
32-
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
33-
free_page((unsigned long)pmd);
34-
}
35-
3625
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
3726
{
3827
set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));

‎arch/arm64/include/asm/pgalloc.h

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,12 @@
1313
#include <asm/cacheflush.h>
1414
#include <asm/tlbflush.h>
1515

16-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
16+
#include <asm-generic/pgalloc.h>
1717

1818
#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
1919

2020
#if CONFIG_PGTABLE_LEVELS > 2
2121

22-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
23-
{
24-
gfp_t gfp = GFP_PGTABLE_USER;
25-
struct page *page;
26-
27-
if (mm == &init_mm)
28-
gfp = GFP_PGTABLE_KERNEL;
29-
30-
page = alloc_page(gfp);
31-
if (!page)
32-
return NULL;
33-
if (!pgtable_pmd_page_ctor(page)) {
34-
__free_page(page);
35-
return NULL;
36-
}
37-
return page_address(page);
38-
}
39-
40-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp)
41-
{
42-
BUG_ON((unsigned long)pmdp & (PAGE_SIZE-1));
43-
pgtable_pmd_page_dtor(virt_to_page(pmdp));
44-
free_page((unsigned long)pmdp);
45-
}
46-
4722
static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
4823
{
4924
set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot));

‎arch/ia64/include/asm/pgalloc.h

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,6 @@ pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
5959
pud_val(*pud_entry) = __pa(pmd);
6060
}
6161

62-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
63-
{
64-
return (pmd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
65-
}
66-
67-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
68-
{
69-
free_page((unsigned long)pmd);
70-
}
71-
7262
#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd)
7363

7464
static inline void

‎arch/mips/include/asm/pgalloc.h

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
#include <linux/mm.h>
1414
#include <linux/sched.h>
1515

16-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
16+
#define __HAVE_ARCH_PMD_ALLOC_ONE
17+
#include <asm-generic/pgalloc.h>
1718

1819
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
1920
pte_t *pte)
@@ -70,11 +71,6 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
7071
return pmd;
7172
}
7273

73-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
74-
{
75-
free_pages((unsigned long)pmd, PMD_ORDER);
76-
}
77-
7874
#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)
7975

8076
#endif

‎arch/parisc/include/asm/pgalloc.h

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010

1111
#include <asm/cache.h>
1212

13-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
13+
#define __HAVE_ARCH_PMD_FREE
14+
#include <asm-generic/pgalloc.h>
1415

1516
/* Allocate the top level pgd (page directory)
1617
*
@@ -65,14 +66,6 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
6566
(__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT)));
6667
}
6768

68-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
69-
{
70-
pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL, PMD_ORDER);
71-
if (pmd)
72-
memset(pmd, 0, PAGE_SIZE<<PMD_ORDER);
73-
return pmd;
74-
}
75-
7669
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
7770
{
7871
if (pmd_flag(*pmd) & PxD_FLAG_ATTACHED) {

‎arch/riscv/include/asm/pgalloc.h

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
#include <asm/tlb.h>
1212

1313
#ifdef CONFIG_MMU
14-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
14+
#include <asm-generic/pgalloc.h>
1515

1616
static inline void pmd_populate_kernel(struct mm_struct *mm,
1717
pmd_t *pmd, pte_t *pte)
@@ -62,17 +62,6 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
6262

6363
#ifndef __PAGETABLE_PMD_FOLDED
6464

65-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
66-
{
67-
return (pmd_t *)__get_free_page(
68-
GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO);
69-
}
70-
71-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
72-
{
73-
free_page((unsigned long)pmd);
74-
}
75-
7665
#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)
7766

7867
#endif /* __PAGETABLE_PMD_FOLDED */

‎arch/sh/include/asm/pgalloc.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
#define __ASM_SH_PGALLOC_H
44

55
#include <asm/page.h>
6+
7+
#define __HAVE_ARCH_PMD_ALLOC_ONE
8+
#define __HAVE_ARCH_PMD_FREE
69
#include <asm-generic/pgalloc.h>
710

811
extern pgd_t *pgd_alloc(struct mm_struct *);

‎arch/um/include/asm/pgalloc.h

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
#include <linux/mm.h>
1212

13-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
13+
#include <asm-generic/pgalloc.h>
1414

1515
#define pmd_populate_kernel(mm, pmd, pte) \
1616
set_pmd(pmd, __pmd(_PAGE_TABLE + (unsigned long) __pa(pte)))
@@ -34,12 +34,6 @@ do { \
3434
} while (0)
3535

3636
#ifdef CONFIG_3_LEVEL_PGTABLES
37-
38-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
39-
{
40-
free_page((unsigned long)pmd);
41-
}
42-
4337
#define __pmd_free_tlb(tlb,x, address) tlb_remove_page((tlb),virt_to_page(x))
4438
#endif
4539

‎arch/um/include/asm/pgtable-3level.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,6 @@ static inline void pgd_mkuptodate(pgd_t pgd) { pgd_val(pgd) &= ~_PAGE_NEWPAGE; }
7878
#define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval))
7979
#endif
8080

81-
struct mm_struct;
82-
extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
83-
8481
static inline void pud_clear (pud_t *pud)
8582
{
8683
set_pud(pud, __pud(_PAGE_NEWPAGE));

‎arch/um/kernel/mem.c

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -201,18 +201,6 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd)
201201
free_page((unsigned long) pgd);
202202
}
203203

204-
#ifdef CONFIG_3_LEVEL_PGTABLES
205-
pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
206-
{
207-
pmd_t *pmd = (pmd_t *) __get_free_page(GFP_KERNEL);
208-
209-
if (pmd)
210-
memset(pmd, 0, PAGE_SIZE);
211-
212-
return pmd;
213-
}
214-
#endif
215-
216204
void *uml_kmalloc(int size, int flags)
217205
{
218206
return kmalloc(size, flags);

‎arch/x86/include/asm/pgalloc.h

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#include <linux/pagemap.h>
88

99
#define __HAVE_ARCH_PTE_ALLOC_ONE
10-
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
10+
#include <asm-generic/pgalloc.h>
1111

1212
static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }
1313

@@ -86,30 +86,6 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
8686
#define pmd_pgtable(pmd) pmd_page(pmd)
8787

8888
#if CONFIG_PGTABLE_LEVELS > 2
89-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
90-
{
91-
struct page *page;
92-
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
93-
94-
if (mm == &init_mm)
95-
gfp &= ~__GFP_ACCOUNT;
96-
page = alloc_pages(gfp, 0);
97-
if (!page)
98-
return NULL;
99-
if (!pgtable_pmd_page_ctor(page)) {
100-
__free_pages(page, 0);
101-
return NULL;
102-
}
103-
return (pmd_t *)page_address(page);
104-
}
105-
106-
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
107-
{
108-
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
109-
pgtable_pmd_page_dtor(virt_to_page(pmd));
110-
free_page((unsigned long)pmd);
111-
}
112-
11389
extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
11490

11591
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,

‎include/asm-generic/pgalloc.h

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,49 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
102102
__free_page(pte_page);
103103
}
104104

105+
106+
#if CONFIG_PGTABLE_LEVELS > 2
107+
108+
#ifndef __HAVE_ARCH_PMD_ALLOC_ONE
109+
/**
110+
* pmd_alloc_one - allocate a page for PMD-level page table
111+
* @mm: the mm_struct of the current context
112+
*
113+
* Allocates a page and runs the pgtable_pmd_page_ctor().
114+
* Allocations use %GFP_PGTABLE_USER in user context and
115+
* %GFP_PGTABLE_KERNEL in kernel context.
116+
*
117+
* Return: pointer to the allocated memory or %NULL on error
118+
*/
119+
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
120+
{
121+
struct page *page;
122+
gfp_t gfp = GFP_PGTABLE_USER;
123+
124+
if (mm == &init_mm)
125+
gfp = GFP_PGTABLE_KERNEL;
126+
page = alloc_pages(gfp, 0);
127+
if (!page)
128+
return NULL;
129+
if (!pgtable_pmd_page_ctor(page)) {
130+
__free_pages(page, 0);
131+
return NULL;
132+
}
133+
return (pmd_t *)page_address(page);
134+
}
135+
#endif
136+
137+
#ifndef __HAVE_ARCH_PMD_FREE
138+
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
139+
{
140+
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
141+
pgtable_pmd_page_dtor(virt_to_page(pmd));
142+
free_page((unsigned long)pmd);
143+
}
144+
#endif
145+
146+
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
147+
105148
#endif /* CONFIG_MMU */
106149

107150
#endif /* __ASM_GENERIC_PGALLOC_H */

0 commit comments

Comments
 (0)