Skip to content

Commit 1a472c9

Browse files
kvaneeshmpe
kvaneesh
authored and
mpe
committed
powerpc/mm/radix: Add tlbflush routines
Core kernel doesn't track the page size of the VA range that we are invalidating. Hence we end up flushing TLB for the entire mm here. Later patches will improve this. We also don't flush page walk cache separetly instead use RIC=2 when flushing TLB, because we do a MMU gather flush after freeing page table. MMU_NO_CONTEXT is updated for hash. Signed-off-by: Aneesh Kumar K.V <[email protected]> Signed-off-by: Michael Ellerman <[email protected]>
1 parent 676012a commit 1a472c9

File tree

8 files changed

+310
-5
lines changed

8 files changed

+310
-5
lines changed

‎arch/powerpc/include/asm/book3s/64/mmu-hash.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@
119119
#define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */
120120
#define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */
121121
#define POWER9_TLB_SETS_HASH 256 /* # sets in POWER9 TLB Hash mode */
122+
#define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
122123

123124
#ifndef __ASSEMBLY__
124125

‎arch/powerpc/include/asm/book3s/64/tlbflush-hash.h

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H
22
#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H
33

4-
#define MMU_NO_CONTEXT 0
5-
64
/*
75
* TLB flushing for 64-bit hash-MMU CPUs
86
*/
@@ -29,14 +27,21 @@ extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
2927

3028
static inline void arch_enter_lazy_mmu_mode(void)
3129
{
32-
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
30+
struct ppc64_tlb_batch *batch;
3331

32+
if (radix_enabled())
33+
return;
34+
batch = this_cpu_ptr(&ppc64_tlb_batch);
3435
batch->active = 1;
3536
}
3637

3738
static inline void arch_leave_lazy_mmu_mode(void)
3839
{
39-
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
40+
struct ppc64_tlb_batch *batch;
41+
42+
if (radix_enabled())
43+
return;
44+
batch = this_cpu_ptr(&ppc64_tlb_batch);
4045

4146
if (batch->index)
4247
__flush_tlb_pending(batch);

‎arch/powerpc/include/asm/book3s/64/tlbflush-radix.h

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H
2+
#define _ASM_POWERPC_TLBFLUSH_RADIX_H
3+
4+
struct vm_area_struct;
5+
struct mm_struct;
6+
struct mmu_gather;
7+
8+
static inline int mmu_get_ap(int psize)
9+
{
10+
return mmu_psize_defs[psize].ap;
11+
}
12+
13+
extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
14+
unsigned long end);
15+
extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
16+
17+
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
18+
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
19+
extern void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
20+
unsigned long ap, int nid);
21+
extern void radix__tlb_flush(struct mmu_gather *tlb);
22+
#ifdef CONFIG_SMP
23+
extern void radix__flush_tlb_mm(struct mm_struct *mm);
24+
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
25+
extern void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
26+
unsigned long ap, int nid);
27+
#else
28+
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
29+
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
30+
#define radix___flush_tlb_page(mm,addr,p,i) radix___local_flush_tlb_page(mm,addr,p,i)
31+
#endif
32+
33+
#endif

‎arch/powerpc/include/asm/book3s/64/tlbflush.h

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,71 @@
11
#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
22
#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
33

4+
#define MMU_NO_CONTEXT ~0UL
5+
6+
47
#include <asm/book3s/64/tlbflush-hash.h>
8+
#include <asm/book3s/64/tlbflush-radix.h>
59

610
static inline void flush_tlb_range(struct vm_area_struct *vma,
711
unsigned long start, unsigned long end)
812
{
13+
if (radix_enabled())
14+
return radix__flush_tlb_range(vma, start, end);
915
return hash__flush_tlb_range(vma, start, end);
1016
}
1117

1218
static inline void flush_tlb_kernel_range(unsigned long start,
1319
unsigned long end)
1420
{
21+
if (radix_enabled())
22+
return radix__flush_tlb_kernel_range(start, end);
1523
return hash__flush_tlb_kernel_range(start, end);
1624
}
1725

1826
static inline void local_flush_tlb_mm(struct mm_struct *mm)
1927
{
28+
if (radix_enabled())
29+
return radix__local_flush_tlb_mm(mm);
2030
return hash__local_flush_tlb_mm(mm);
2131
}
2232

2333
static inline void local_flush_tlb_page(struct vm_area_struct *vma,
2434
unsigned long vmaddr)
2535
{
36+
if (radix_enabled())
37+
return radix__local_flush_tlb_page(vma, vmaddr);
2638
return hash__local_flush_tlb_page(vma, vmaddr);
2739
}
2840

2941
static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
3042
unsigned long vmaddr)
3143
{
44+
if (radix_enabled())
45+
return radix__flush_tlb_page(vma, vmaddr);
3246
return hash__flush_tlb_page_nohash(vma, vmaddr);
3347
}
3448

3549
static inline void tlb_flush(struct mmu_gather *tlb)
3650
{
51+
if (radix_enabled())
52+
return radix__tlb_flush(tlb);
3753
return hash__tlb_flush(tlb);
3854
}
3955

4056
#ifdef CONFIG_SMP
4157
static inline void flush_tlb_mm(struct mm_struct *mm)
4258
{
59+
if (radix_enabled())
60+
return radix__flush_tlb_mm(mm);
4361
return hash__flush_tlb_mm(mm);
4462
}
4563

4664
static inline void flush_tlb_page(struct vm_area_struct *vma,
4765
unsigned long vmaddr)
4866
{
67+
if (radix_enabled())
68+
return radix__flush_tlb_page(vma, vmaddr);
4969
return hash__flush_tlb_page(vma, vmaddr);
5070
}
5171
#else

‎arch/powerpc/include/asm/tlbflush.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ extern void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
5858

5959
#elif defined(CONFIG_PPC_STD_MMU_32)
6060

61+
#define MMU_NO_CONTEXT (0)
6162
/*
6263
* TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx
6364
*/

‎arch/powerpc/kernel/mce_power.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ void __flush_tlb_power8(unsigned int action)
7272

7373
void __flush_tlb_power9(unsigned int action)
7474
{
75+
if (radix_enabled())
76+
flush_tlb_206(POWER9_TLB_SETS_RADIX, action);
77+
7578
flush_tlb_206(POWER9_TLB_SETS_HASH, action);
7679
}
7780

‎arch/powerpc/mm/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ obj-$(CONFIG_PPC_BOOK3E) += tlb_low_$(CONFIG_WORD_SIZE)e.o
1515
hash64-$(CONFIG_PPC_NATIVE) := hash_native_64.o
1616
obj-$(CONFIG_PPC_BOOK3E_64) += pgtable-book3e.o
1717
obj-$(CONFIG_PPC_STD_MMU_64) += pgtable-hash64.o hash_utils_64.o slb_low.o slb.o $(hash64-y) mmu_context_book3s64.o
18-
obj-$(CONFIG_PPC_RADIX_MMU) += pgtable-radix.o
18+
obj-$(CONFIG_PPC_RADIX_MMU) += pgtable-radix.o tlb-radix.o
1919
obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o hash_low_32.o mmu_context_hash32.o
2020
obj-$(CONFIG_PPC_STD_MMU) += tlb_hash$(CONFIG_WORD_SIZE).o
2121
ifeq ($(CONFIG_PPC_STD_MMU_64),y)

0 commit comments

Comments
 (0)