Skip to content

Commit 9d20040

Browse files
torvaldstorvalds
committed
Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 fixes from Will Deacon: "Ryan's been hard at work finding and fixing mm bugs in the arm64 code, so here's a small crop of fixes for -rc5. The main changes are to fix our zapping of non-present PTEs for hugetlb entries created using the contiguous bit in the page-table rather than a block entry at the level above. Prior to these fixes, we were pulling the contiguous bit back out of the PTE in order to determine the size of the hugetlb page but this is clearly bogus if the thing isn't present and consequently both the clearing of the PTE(s) and the TLB invalidation were unreliable. Although the problem was found by code inspection, we really don't want this sitting around waiting to trigger and the changes are CC'd to stable accordingly. Note that the diffstat looks a lot worse than it really is; huge_ptep_get_and_clear() now takes a size argument from the core code and so all the arch implementations of that have been updated in a pretty mechanical fashion. - Fix a sporadic boot failure due to incorrect randomization of the linear map on systems that support it - Fix the zapping (both clearing the entries *and* invalidating the TLB) of hugetlb PTEs constructed using the contiguous bit" * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: arm64: hugetlb: Fix flush_hugetlb_tlb_range() invalidation level arm64: hugetlb: Fix huge_ptep_get_and_clear() for non-present ptes mm: hugetlb: Add huge page size param to huge_ptep_get_and_clear() arm64/mm: Fix Boot panic on Ampere Altra
2 parents b4b215c + eed6bfa commit 9d20040

File tree

17 files changed

+83
-72
lines changed

17 files changed

+83
-72
lines changed

‎arch/arm64/include/asm/hugetlb.h

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ extern int huge_ptep_set_access_flags(struct vm_area_struct *vma,
4242
unsigned long addr, pte_t *ptep,
4343
pte_t pte, int dirty);
4444
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
45-
extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
46-
unsigned long addr, pte_t *ptep);
45+
extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
46+
pte_t *ptep, unsigned long sz);
4747
#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT
4848
extern void huge_ptep_set_wrprotect(struct mm_struct *mm,
4949
unsigned long addr, pte_t *ptep);
@@ -76,12 +76,22 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
7676
{
7777
unsigned long stride = huge_page_size(hstate_vma(vma));
7878

79-
if (stride == PMD_SIZE)
80-
__flush_tlb_range(vma, start, end, stride, false, 2);
81-
else if (stride == PUD_SIZE)
82-
__flush_tlb_range(vma, start, end, stride, false, 1);
83-
else
84-
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, 0);
79+
switch (stride) {
80+
#ifndef __PAGETABLE_PMD_FOLDED
81+
case PUD_SIZE:
82+
__flush_tlb_range(vma, start, end, PUD_SIZE, false, 1);
83+
break;
84+
#endif
85+
case CONT_PMD_SIZE:
86+
case PMD_SIZE:
87+
__flush_tlb_range(vma, start, end, PMD_SIZE, false, 2);
88+
break;
89+
case CONT_PTE_SIZE:
90+
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, 3);
91+
break;
92+
default:
93+
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
94+
}
8595
}
8696

8797
#endif /* __ASM_HUGETLB_H */

‎arch/arm64/mm/hugetlbpage.c

Lines changed: 25 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -100,20 +100,11 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr,
100100

101101
static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
102102
{
103-
int contig_ptes = 0;
103+
int contig_ptes = 1;
104104

105105
*pgsize = size;
106106

107107
switch (size) {
108-
#ifndef __PAGETABLE_PMD_FOLDED
109-
case PUD_SIZE:
110-
if (pud_sect_supported())
111-
contig_ptes = 1;
112-
break;
113-
#endif
114-
case PMD_SIZE:
115-
contig_ptes = 1;
116-
break;
117108
case CONT_PMD_SIZE:
118109
*pgsize = PMD_SIZE;
119110
contig_ptes = CONT_PMDS;
@@ -122,6 +113,8 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
122113
*pgsize = PAGE_SIZE;
123114
contig_ptes = CONT_PTES;
124115
break;
116+
default:
117+
WARN_ON(!__hugetlb_valid_size(size));
125118
}
126119

127120
return contig_ptes;
@@ -163,24 +156,23 @@ static pte_t get_clear_contig(struct mm_struct *mm,
163156
unsigned long pgsize,
164157
unsigned long ncontig)
165158
{
166-
pte_t orig_pte = __ptep_get(ptep);
167-
unsigned long i;
168-
169-
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
170-
pte_t pte = __ptep_get_and_clear(mm, addr, ptep);
171-
172-
/*
173-
* If HW_AFDBM is enabled, then the HW could turn on
174-
* the dirty or accessed bit for any page in the set,
175-
* so check them all.
176-
*/
177-
if (pte_dirty(pte))
178-
orig_pte = pte_mkdirty(orig_pte);
179-
180-
if (pte_young(pte))
181-
orig_pte = pte_mkyoung(orig_pte);
159+
pte_t pte, tmp_pte;
160+
bool present;
161+
162+
pte = __ptep_get_and_clear(mm, addr, ptep);
163+
present = pte_present(pte);
164+
while (--ncontig) {
165+
ptep++;
166+
addr += pgsize;
167+
tmp_pte = __ptep_get_and_clear(mm, addr, ptep);
168+
if (present) {
169+
if (pte_dirty(tmp_pte))
170+
pte = pte_mkdirty(pte);
171+
if (pte_young(tmp_pte))
172+
pte = pte_mkyoung(pte);
173+
}
182174
}
183-
return orig_pte;
175+
return pte;
184176
}
185177

186178
static pte_t get_clear_contig_flush(struct mm_struct *mm,
@@ -396,18 +388,13 @@ void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
396388
__pte_clear(mm, addr, ptep);
397389
}
398390

399-
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
400-
unsigned long addr, pte_t *ptep)
391+
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
392+
pte_t *ptep, unsigned long sz)
401393
{
402394
int ncontig;
403395
size_t pgsize;
404-
pte_t orig_pte = __ptep_get(ptep);
405-
406-
if (!pte_cont(orig_pte))
407-
return __ptep_get_and_clear(mm, addr, ptep);
408-
409-
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
410396

397+
ncontig = num_contig_ptes(sz, &pgsize);
411398
return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
412399
}
413400

@@ -549,6 +536,8 @@ bool __init arch_hugetlb_valid_size(unsigned long size)
549536

550537
pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
551538
{
539+
unsigned long psize = huge_page_size(hstate_vma(vma));
540+
552541
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
553542
/*
554543
* Break-before-make (BBM) is required for all user space mappings
@@ -558,7 +547,7 @@ pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr
558547
if (pte_user_exec(__ptep_get(ptep)))
559548
return huge_ptep_clear_flush(vma, addr, ptep);
560549
}
561-
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
550+
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
562551
}
563552

564553
void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,

‎arch/arm64/mm/init.c

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -279,12 +279,7 @@ void __init arm64_memblock_init(void)
279279

280280
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
281281
extern u16 memstart_offset_seed;
282-
283-
/*
284-
* Use the sanitised version of id_aa64mmfr0_el1 so that linear
285-
* map randomization can be enabled by shrinking the IPA space.
286-
*/
287-
u64 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
282+
u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
288283
int parange = cpuid_feature_extract_unsigned_field(
289284
mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
290285
s64 range = linear_region_size -

‎arch/loongarch/include/asm/hugetlb.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
3636

3737
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
3838
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
39-
unsigned long addr, pte_t *ptep)
39+
unsigned long addr, pte_t *ptep,
40+
unsigned long sz)
4041
{
4142
pte_t clear;
4243
pte_t pte = ptep_get(ptep);
@@ -51,8 +52,9 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
5152
unsigned long addr, pte_t *ptep)
5253
{
5354
pte_t pte;
55+
unsigned long sz = huge_page_size(hstate_vma(vma));
5456

55-
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
57+
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
5658
flush_tlb_page(vma, addr);
5759
return pte;
5860
}

‎arch/mips/include/asm/hugetlb.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ static inline int prepare_hugepage_range(struct file *file,
2727

2828
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
2929
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
30-
unsigned long addr, pte_t *ptep)
30+
unsigned long addr, pte_t *ptep,
31+
unsigned long sz)
3132
{
3233
pte_t clear;
3334
pte_t pte = *ptep;
@@ -42,13 +43,14 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
4243
unsigned long addr, pte_t *ptep)
4344
{
4445
pte_t pte;
46+
unsigned long sz = huge_page_size(hstate_vma(vma));
4547

4648
/*
4749
* clear the huge pte entry firstly, so that the other smp threads will
4850
* not get old pte entry after finishing flush_tlb_page and before
4951
* setting new huge pte entry
5052
*/
51-
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
53+
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
5254
flush_tlb_page(vma, addr);
5355
return pte;
5456
}

‎arch/parisc/include/asm/hugetlb.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
1010

1111
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
1212
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
13-
pte_t *ptep);
13+
pte_t *ptep, unsigned long sz);
1414

1515
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
1616
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,

‎arch/parisc/mm/hugetlbpage.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
126126

127127

128128
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
129-
pte_t *ptep)
129+
pte_t *ptep, unsigned long sz)
130130
{
131131
pte_t entry;
132132

‎arch/powerpc/include/asm/hugetlb.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
4545

4646
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
4747
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
48-
unsigned long addr, pte_t *ptep)
48+
unsigned long addr, pte_t *ptep,
49+
unsigned long sz)
4950
{
5051
return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1));
5152
}
@@ -55,8 +56,9 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
5556
unsigned long addr, pte_t *ptep)
5657
{
5758
pte_t pte;
59+
unsigned long sz = huge_page_size(hstate_vma(vma));
5860

59-
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
61+
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz);
6062
flush_hugetlb_page(vma, addr);
6163
return pte;
6264
}

‎arch/riscv/include/asm/hugetlb.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ void set_huge_pte_at(struct mm_struct *mm,
2828

2929
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
3030
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
31-
unsigned long addr, pte_t *ptep);
31+
unsigned long addr, pte_t *ptep,
32+
unsigned long sz);
3233

3334
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
3435
pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,

‎arch/riscv/mm/hugetlbpage.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
293293

294294
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
295295
unsigned long addr,
296-
pte_t *ptep)
296+
pte_t *ptep, unsigned long sz)
297297
{
298298
pte_t orig_pte = ptep_get(ptep);
299299
int pte_num;

‎arch/s390/include/asm/hugetlb.h

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,16 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
2525
#define __HAVE_ARCH_HUGE_PTEP_GET
2626
pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
2727

28+
pte_t __huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
29+
pte_t *ptep);
30+
2831
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
29-
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
32+
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
33+
unsigned long addr, pte_t *ptep,
34+
unsigned long sz)
35+
{
36+
return __huge_ptep_get_and_clear(mm, addr, ptep);
37+
}
3038

3139
static inline void arch_clear_hugetlb_flags(struct folio *folio)
3240
{
@@ -48,7 +56,7 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
4856
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
4957
unsigned long address, pte_t *ptep)
5058
{
51-
return huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
59+
return __huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
5260
}
5361

5462
#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS
@@ -59,7 +67,7 @@ static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
5967
int changed = !pte_same(huge_ptep_get(vma->vm_mm, addr, ptep), pte);
6068

6169
if (changed) {
62-
huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
70+
__huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
6371
__set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
6472
}
6573
return changed;
@@ -69,7 +77,7 @@ static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
6977
static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
7078
unsigned long addr, pte_t *ptep)
7179
{
72-
pte_t pte = huge_ptep_get_and_clear(mm, addr, ptep);
80+
pte_t pte = __huge_ptep_get_and_clear(mm, addr, ptep);
7381

7482
__set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte));
7583
}

‎arch/s390/mm/hugetlbpage.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,8 +188,8 @@ pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
188188
return __rste_to_pte(pte_val(*ptep));
189189
}
190190

191-
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
192-
unsigned long addr, pte_t *ptep)
191+
pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
192+
unsigned long addr, pte_t *ptep)
193193
{
194194
pte_t pte = huge_ptep_get(mm, addr, ptep);
195195
pmd_t *pmdp = (pmd_t *) ptep;

‎arch/sparc/include/asm/hugetlb.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
2020

2121
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
2222
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
23-
pte_t *ptep);
23+
pte_t *ptep, unsigned long sz);
2424

2525
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
2626
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,

‎arch/sparc/mm/hugetlbpage.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
260260
}
261261

262262
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
263-
pte_t *ptep)
263+
pte_t *ptep, unsigned long sz)
264264
{
265265
unsigned int i, nptes, orig_shift, shift;
266266
unsigned long size;

‎include/asm-generic/hugetlb.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
9090

9191
#ifndef __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
9292
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
93-
unsigned long addr, pte_t *ptep)
93+
unsigned long addr, pte_t *ptep, unsigned long sz)
9494
{
9595
return ptep_get_and_clear(mm, addr, ptep);
9696
}

‎include/linux/hugetlb.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1004,7 +1004,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
10041004
static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
10051005
unsigned long addr, pte_t *ptep)
10061006
{
1007-
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
1007+
unsigned long psize = huge_page_size(hstate_vma(vma));
1008+
1009+
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
10081010
}
10091011
#endif
10101012

‎mm/hugetlb.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5447,7 +5447,7 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
54475447
if (src_ptl != dst_ptl)
54485448
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
54495449

5450-
pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
5450+
pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);
54515451

54525452
if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
54535453
huge_pte_clear(mm, new_addr, dst_pte, sz);
@@ -5622,7 +5622,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
56225622
set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
56235623
}
56245624

5625-
pte = huge_ptep_get_and_clear(mm, address, ptep);
5625+
pte = huge_ptep_get_and_clear(mm, address, ptep, sz);
56265626
tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
56275627
if (huge_pte_dirty(pte))
56285628
set_page_dirty(page);

0 commit comments

Comments
 (0)