Skip to content

Commit e0c4f67

Browse files
kirylIngo Molnar
kiryl
authored and
Ingo Molnar
committed
x86/mm: Convert trivial cases of page table walk to 5-level paging
This patch only covers simple cases. Less trivial cases will be converted with separate patches. Signed-off-by: Kirill A. Shutemov <[email protected]> Cc: Andrew Morton <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: Arnd Bergmann <[email protected]> Cc: Borislav Petkov <[email protected]> Cc: Brian Gerst <[email protected]> Cc: Dave Hansen <[email protected]> Cc: Denys Vlasenko <[email protected]> Cc: H. Peter Anvin <[email protected]> Cc: Josh Poimboeuf <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Michal Hocko <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Thomas Gleixner <[email protected]> Cc: [email protected] Cc: [email protected] Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Ingo Molnar <[email protected]>
1 parent fe1e8c3 commit e0c4f67

File tree

9 files changed

+85
-23
lines changed

9 files changed

+85
-23
lines changed

‎arch/x86/kernel/tboot.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,16 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
118118
pgprot_t prot)
119119
{
120120
pgd_t *pgd;
121+
p4d_t *p4d;
121122
pud_t *pud;
122123
pmd_t *pmd;
123124
pte_t *pte;
124125

125126
pgd = pgd_offset(&tboot_mm, vaddr);
126-
pud = pud_alloc(&tboot_mm, pgd, vaddr);
127+
p4d = p4d_alloc(&tboot_mm, pgd, vaddr);
128+
if (!p4d)
129+
return -1;
130+
pud = pud_alloc(&tboot_mm, p4d, vaddr);
127131
if (!pud)
128132
return -1;
129133
pmd = pmd_alloc(&tboot_mm, pud, vaddr);

‎arch/x86/kernel/vm86_32.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ static void mark_screen_rdonly(struct mm_struct *mm)
164164
struct vm_area_struct *vma;
165165
spinlock_t *ptl;
166166
pgd_t *pgd;
167+
p4d_t *p4d;
167168
pud_t *pud;
168169
pmd_t *pmd;
169170
pte_t *pte;
@@ -173,7 +174,10 @@ static void mark_screen_rdonly(struct mm_struct *mm)
173174
pgd = pgd_offset(mm, 0xA0000);
174175
if (pgd_none_or_clear_bad(pgd))
175176
goto out;
176-
pud = pud_offset(pgd, 0xA0000);
177+
p4d = p4d_offset(pgd, 0xA0000);
178+
if (p4d_none_or_clear_bad(p4d))
179+
goto out;
180+
pud = pud_offset(p4d, 0xA0000);
177181
if (pud_none_or_clear_bad(pud))
178182
goto out;
179183
pmd = pmd_offset(pud, 0xA0000);

‎arch/x86/mm/fault.c

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
253253
{
254254
unsigned index = pgd_index(address);
255255
pgd_t *pgd_k;
256+
p4d_t *p4d, *p4d_k;
256257
pud_t *pud, *pud_k;
257258
pmd_t *pmd, *pmd_k;
258259

@@ -265,10 +266,15 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
265266
/*
266267
* set_pgd(pgd, *pgd_k); here would be useless on PAE
267268
* and redundant with the set_pmd() on non-PAE. As would
268-
* set_pud.
269+
* set_p4d/set_pud.
269270
*/
270-
pud = pud_offset(pgd, address);
271-
pud_k = pud_offset(pgd_k, address);
271+
p4d = p4d_offset(pgd, address);
272+
p4d_k = p4d_offset(pgd_k, address);
273+
if (!p4d_present(*p4d_k))
274+
return NULL;
275+
276+
pud = pud_offset(p4d, address);
277+
pud_k = pud_offset(p4d_k, address);
272278
if (!pud_present(*pud_k))
273279
return NULL;
274280

@@ -384,6 +390,8 @@ static void dump_pagetable(unsigned long address)
384390
{
385391
pgd_t *base = __va(read_cr3());
386392
pgd_t *pgd = &base[pgd_index(address)];
393+
p4d_t *p4d;
394+
pud_t *pud;
387395
pmd_t *pmd;
388396
pte_t *pte;
389397

@@ -392,7 +400,9 @@ static void dump_pagetable(unsigned long address)
392400
if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
393401
goto out;
394402
#endif
395-
pmd = pmd_offset(pud_offset(pgd, address), address);
403+
p4d = p4d_offset(pgd, address);
404+
pud = pud_offset(p4d, address);
405+
pmd = pmd_offset(pud, address);
396406
printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
397407

398408
/*
@@ -526,6 +536,7 @@ static void dump_pagetable(unsigned long address)
526536
{
527537
pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
528538
pgd_t *pgd = base + pgd_index(address);
539+
p4d_t *p4d;
529540
pud_t *pud;
530541
pmd_t *pmd;
531542
pte_t *pte;
@@ -538,7 +549,15 @@ static void dump_pagetable(unsigned long address)
538549
if (!pgd_present(*pgd))
539550
goto out;
540551

541-
pud = pud_offset(pgd, address);
552+
p4d = p4d_offset(pgd, address);
553+
if (bad_address(p4d))
554+
goto bad;
555+
556+
printk("P4D %lx ", p4d_val(*p4d));
557+
if (!p4d_present(*p4d) || p4d_large(*p4d))
558+
goto out;
559+
560+
pud = pud_offset(p4d, address);
542561
if (bad_address(pud))
543562
goto bad;
544563

@@ -1082,6 +1101,7 @@ static noinline int
10821101
spurious_fault(unsigned long error_code, unsigned long address)
10831102
{
10841103
pgd_t *pgd;
1104+
p4d_t *p4d;
10851105
pud_t *pud;
10861106
pmd_t *pmd;
10871107
pte_t *pte;
@@ -1104,7 +1124,14 @@ spurious_fault(unsigned long error_code, unsigned long address)
11041124
if (!pgd_present(*pgd))
11051125
return 0;
11061126

1107-
pud = pud_offset(pgd, address);
1127+
p4d = p4d_offset(pgd, address);
1128+
if (!p4d_present(*p4d))
1129+
return 0;
1130+
1131+
if (p4d_large(*p4d))
1132+
return spurious_fault_check(error_code, (pte_t *) p4d);
1133+
1134+
pud = pud_offset(p4d, address);
11081135
if (!pud_present(*pud))
11091136
return 0;
11101137

‎arch/x86/mm/init_32.c

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ bool __read_mostly __vmalloc_start_set = false;
6767
*/
6868
static pmd_t * __init one_md_table_init(pgd_t *pgd)
6969
{
70+
p4d_t *p4d;
7071
pud_t *pud;
7172
pmd_t *pmd_table;
7273

@@ -75,13 +76,15 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
7576
pmd_table = (pmd_t *)alloc_low_page();
7677
paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
7778
set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
78-
pud = pud_offset(pgd, 0);
79+
p4d = p4d_offset(pgd, 0);
80+
pud = pud_offset(p4d, 0);
7981
BUG_ON(pmd_table != pmd_offset(pud, 0));
8082

8183
return pmd_table;
8284
}
8385
#endif
84-
pud = pud_offset(pgd, 0);
86+
p4d = p4d_offset(pgd, 0);
87+
pud = pud_offset(p4d, 0);
8588
pmd_table = pmd_offset(pud, 0);
8689

8790
return pmd_table;
@@ -390,8 +393,11 @@ pte_t *kmap_pte;
390393

391394
static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
392395
{
393-
return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
394-
vaddr), vaddr), vaddr);
396+
pgd_t *pgd = pgd_offset_k(vaddr);
397+
p4d_t *p4d = p4d_offset(pgd, vaddr);
398+
pud_t *pud = pud_offset(p4d, vaddr);
399+
pmd_t *pmd = pmd_offset(pud, vaddr);
400+
return pte_offset_kernel(pmd, vaddr);
395401
}
396402

397403
static void __init kmap_init(void)
@@ -410,6 +416,7 @@ static void __init permanent_kmaps_init(pgd_t *pgd_base)
410416
{
411417
unsigned long vaddr;
412418
pgd_t *pgd;
419+
p4d_t *p4d;
413420
pud_t *pud;
414421
pmd_t *pmd;
415422
pte_t *pte;
@@ -418,7 +425,8 @@ static void __init permanent_kmaps_init(pgd_t *pgd_base)
418425
page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
419426

420427
pgd = swapper_pg_dir + pgd_index(vaddr);
421-
pud = pud_offset(pgd, vaddr);
428+
p4d = p4d_offset(pgd, vaddr);
429+
pud = pud_offset(p4d, vaddr);
422430
pmd = pmd_offset(pud, vaddr);
423431
pte = pte_offset_kernel(pmd, vaddr);
424432
pkmap_page_table = pte;
@@ -450,6 +458,7 @@ void __init native_pagetable_init(void)
450458
{
451459
unsigned long pfn, va;
452460
pgd_t *pgd, *base = swapper_pg_dir;
461+
p4d_t *p4d;
453462
pud_t *pud;
454463
pmd_t *pmd;
455464
pte_t *pte;
@@ -469,7 +478,8 @@ void __init native_pagetable_init(void)
469478
if (!pgd_present(*pgd))
470479
break;
471480

472-
pud = pud_offset(pgd, va);
481+
p4d = p4d_offset(pgd, va);
482+
pud = pud_offset(p4d, va);
473483
pmd = pmd_offset(pud, va);
474484
if (!pmd_present(*pmd))
475485
break;

‎arch/x86/mm/ioremap.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,8 @@ static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
425425
/* Don't assume we're using swapper_pg_dir at this point */
426426
pgd_t *base = __va(read_cr3());
427427
pgd_t *pgd = &base[pgd_index(addr)];
428-
pud_t *pud = pud_offset(pgd, addr);
428+
p4d_t *p4d = p4d_offset(pgd, addr);
429+
pud_t *pud = pud_offset(p4d, addr);
429430
pmd_t *pmd = pmd_offset(pud, addr);
430431

431432
return pmd;

‎arch/x86/mm/pgtable.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,13 +261,15 @@ static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
261261

262262
static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
263263
{
264+
p4d_t *p4d;
264265
pud_t *pud;
265266
int i;
266267

267268
if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
268269
return;
269270

270-
pud = pud_offset(pgd, 0);
271+
p4d = p4d_offset(pgd, 0);
272+
pud = pud_offset(p4d, 0);
271273

272274
for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
273275
pmd_t *pmd = pmds[i];

‎arch/x86/mm/pgtable_32.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ unsigned int __VMALLOC_RESERVE = 128 << 20;
2626
void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
2727
{
2828
pgd_t *pgd;
29+
p4d_t *p4d;
2930
pud_t *pud;
3031
pmd_t *pmd;
3132
pte_t *pte;
@@ -35,7 +36,12 @@ void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
3536
BUG();
3637
return;
3738
}
38-
pud = pud_offset(pgd, vaddr);
39+
p4d = p4d_offset(pgd, vaddr);
40+
if (p4d_none(*p4d)) {
41+
BUG();
42+
return;
43+
}
44+
pud = pud_offset(p4d, vaddr);
3945
if (pud_none(*pud)) {
4046
BUG();
4147
return;

‎arch/x86/platform/efi/efi_64.c

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ void efi_sync_low_kernel_mappings(void)
166166
{
167167
unsigned num_entries;
168168
pgd_t *pgd_k, *pgd_efi;
169+
p4d_t *p4d_k, *p4d_efi;
169170
pud_t *pud_k, *pud_efi;
170171

171172
if (efi_enabled(EFI_OLD_MEMMAP))
@@ -197,16 +198,20 @@ void efi_sync_low_kernel_mappings(void)
197198
BUILD_BUG_ON((EFI_VA_END & ~PUD_MASK) != 0);
198199

199200
pgd_efi = efi_pgd + pgd_index(EFI_VA_END);
200-
pud_efi = pud_offset(pgd_efi, 0);
201+
p4d_efi = p4d_offset(pgd_efi, 0);
202+
pud_efi = pud_offset(p4d_efi, 0);
201203

202204
pgd_k = pgd_offset_k(EFI_VA_END);
203-
pud_k = pud_offset(pgd_k, 0);
205+
p4d_k = p4d_offset(pgd_k, 0);
206+
pud_k = pud_offset(p4d_k, 0);
204207

205208
num_entries = pud_index(EFI_VA_END);
206209
memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);
207210

208-
pud_efi = pud_offset(pgd_efi, EFI_VA_START);
209-
pud_k = pud_offset(pgd_k, EFI_VA_START);
211+
p4d_efi = p4d_offset(pgd_efi, EFI_VA_START);
212+
pud_efi = pud_offset(p4d_efi, EFI_VA_START);
213+
p4d_k = p4d_offset(pgd_k, EFI_VA_START);
214+
pud_k = pud_offset(p4d_k, EFI_VA_START);
210215

211216
num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START);
212217
memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);

‎arch/x86/power/hibernate_32.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pgd_t *resume_pg_dir;
3232
*/
3333
static pmd_t *resume_one_md_table_init(pgd_t *pgd)
3434
{
35+
p4d_t *p4d;
3536
pud_t *pud;
3637
pmd_t *pmd_table;
3738

@@ -41,11 +42,13 @@ static pmd_t *resume_one_md_table_init(pgd_t *pgd)
4142
return NULL;
4243

4344
set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
44-
pud = pud_offset(pgd, 0);
45+
p4d = p4d_offset(pgd, 0);
46+
pud = pud_offset(p4d, 0);
4547

4648
BUG_ON(pmd_table != pmd_offset(pud, 0));
4749
#else
48-
pud = pud_offset(pgd, 0);
50+
p4d = p4d_offset(pgd, 0);
51+
pud = pud_offset(p4d, 0);
4952
pmd_table = pmd_offset(pud, 0);
5053
#endif
5154

0 commit comments

Comments
 (0)