Skip to content

Commit 1299ef1

Browse files
amlutoIngo Molnar
authored andcommitted
x86/mm: Rename flush_tlb_single() and flush_tlb_one() to __flush_tlb_one_[user|kernel]()
flush_tlb_single() and flush_tlb_one() sound almost identical, but they really mean "flush one user translation" and "flush one kernel translation". Rename them to flush_tlb_one_user() and flush_tlb_one_kernel() to make the semantics more obvious. [ I was looking at some PTI-related code, and the flush-one-address code is unnecessarily hard to understand because the names of the helpers are uninformative. This came up during PTI review, but no one got around to doing it. ] Signed-off-by: Andy Lutomirski <[email protected]> Acked-by: Peter Zijlstra (Intel) <[email protected]> Cc: Boris Ostrovsky <[email protected]> Cc: Borislav Petkov <[email protected]> Cc: Brian Gerst <[email protected]> Cc: Dave Hansen <[email protected]> Cc: Eduardo Valentin <[email protected]> Cc: Hugh Dickins <[email protected]> Cc: Josh Poimboeuf <[email protected]> Cc: Juergen Gross <[email protected]> Cc: Kees Cook <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Linux-MM <[email protected]> Cc: Rik van Riel <[email protected]> Cc: Thomas Gleixner <[email protected]> Cc: Will Deacon <[email protected]> Link: http://lkml.kernel.org/r/3303b02e3c3d049dc5235d5651e0ae6d29a34354.1517414378.git.luto@kernel.org Signed-off-by: Ingo Molnar <[email protected]>
1 parent ea00f30 commit 1299ef1

File tree

13 files changed

+39
-26
lines changed

13 files changed

+39
-26
lines changed

arch/x86/include/asm/paravirt.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,9 +297,9 @@ static inline void __flush_tlb_global(void)
297297
{
298298
PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
299299
}
300-
static inline void __flush_tlb_single(unsigned long addr)
300+
static inline void __flush_tlb_one_user(unsigned long addr)
301301
{
302-
PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
302+
PVOP_VCALL1(pv_mmu_ops.flush_tlb_one_user, addr);
303303
}
304304

305305
static inline void flush_tlb_others(const struct cpumask *cpumask,

arch/x86/include/asm/paravirt_types.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ struct pv_mmu_ops {
217217
/* TLB operations */
218218
void (*flush_tlb_user)(void);
219219
void (*flush_tlb_kernel)(void);
220-
void (*flush_tlb_single)(unsigned long addr);
220+
void (*flush_tlb_one_user)(unsigned long addr);
221221
void (*flush_tlb_others)(const struct cpumask *cpus,
222222
const struct flush_tlb_info *info);
223223

arch/x86/include/asm/pgtable_32.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ void paging_init(void);
6161
#define kpte_clear_flush(ptep, vaddr) \
6262
do { \
6363
pte_clear(&init_mm, (vaddr), (ptep)); \
64-
__flush_tlb_one((vaddr)); \
64+
__flush_tlb_one_kernel((vaddr)); \
6565
} while (0)
6666

6767
#endif /* !__ASSEMBLY__ */

arch/x86/include/asm/tlbflush.h

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
140140
#else
141141
#define __flush_tlb() __native_flush_tlb()
142142
#define __flush_tlb_global() __native_flush_tlb_global()
143-
#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
143+
#define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
144144
#endif
145145

146146
static inline bool tlb_defer_switch_to_init_mm(void)
@@ -400,7 +400,7 @@ static inline void __native_flush_tlb_global(void)
400400
/*
401401
* flush one page in the user mapping
402402
*/
403-
static inline void __native_flush_tlb_single(unsigned long addr)
403+
static inline void __native_flush_tlb_one_user(unsigned long addr)
404404
{
405405
u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
406406

@@ -437,18 +437,31 @@ static inline void __flush_tlb_all(void)
437437
/*
438438
* flush one page in the kernel mapping
439439
*/
440-
static inline void __flush_tlb_one(unsigned long addr)
440+
static inline void __flush_tlb_one_kernel(unsigned long addr)
441441
{
442442
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
443-
__flush_tlb_single(addr);
443+
444+
/*
445+
* If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
446+
* paravirt equivalent. Even with PCID, this is sufficient: we only
447+
* use PCID if we also use global PTEs for the kernel mapping, and
448+
* INVLPG flushes global translations across all address spaces.
449+
*
450+
* If PTI is on, then the kernel is mapped with non-global PTEs, and
451+
* __flush_tlb_one_user() will flush the given address for the current
452+
* kernel address space and for its usermode counterpart, but it does
453+
* not flush it for other address spaces.
454+
*/
455+
__flush_tlb_one_user(addr);
444456

445457
if (!static_cpu_has(X86_FEATURE_PTI))
446458
return;
447459

448460
/*
449-
* __flush_tlb_single() will have cleared the TLB entry for this ASID,
450-
* but since kernel space is replicated across all, we must also
451-
* invalidate all others.
461+
* See above. We need to propagate the flush to all other address
462+
* spaces. In principle, we only need to propagate it to kernelmode
463+
* address spaces, but the extra bookkeeping we would need is not
464+
* worth it.
452465
*/
453466
invalidate_other_asid();
454467
}

arch/x86/kernel/paravirt.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -200,9 +200,9 @@ static void native_flush_tlb_global(void)
200200
__native_flush_tlb_global();
201201
}
202202

203-
static void native_flush_tlb_single(unsigned long addr)
203+
static void native_flush_tlb_one_user(unsigned long addr)
204204
{
205-
__native_flush_tlb_single(addr);
205+
__native_flush_tlb_one_user(addr);
206206
}
207207

208208
struct static_key paravirt_steal_enabled;
@@ -401,7 +401,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
401401

402402
.flush_tlb_user = native_flush_tlb,
403403
.flush_tlb_kernel = native_flush_tlb_global,
404-
.flush_tlb_single = native_flush_tlb_single,
404+
.flush_tlb_one_user = native_flush_tlb_one_user,
405405
.flush_tlb_others = native_flush_tlb_others,
406406

407407
.pgd_alloc = __paravirt_pgd_alloc,

arch/x86/mm/init_64.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte)
256256
* It's enough to flush this one mapping.
257257
* (PGE mappings get flushed as well)
258258
*/
259-
__flush_tlb_one(vaddr);
259+
__flush_tlb_one_kernel(vaddr);
260260
}
261261

262262
void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte)

arch/x86/mm/ioremap.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -820,5 +820,5 @@ void __init __early_set_fixmap(enum fixed_addresses idx,
820820
set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
821821
else
822822
pte_clear(&init_mm, addr, pte);
823-
__flush_tlb_one(addr);
823+
__flush_tlb_one_kernel(addr);
824824
}

arch/x86/mm/kmmio.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
168168
return -1;
169169
}
170170

171-
__flush_tlb_one(f->addr);
171+
__flush_tlb_one_kernel(f->addr);
172172
return 0;
173173
}
174174

arch/x86/mm/pgtable_32.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
6363
* It's enough to flush this one mapping.
6464
* (PGE mappings get flushed as well)
6565
*/
66-
__flush_tlb_one(vaddr);
66+
__flush_tlb_one_kernel(vaddr);
6767
}
6868

6969
unsigned long __FIXADDR_TOP = 0xfffff000;

arch/x86/mm/tlb.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
492492
* flush that changes context.tlb_gen from 2 to 3. If they get
493493
* processed on this CPU in reverse order, we'll see
494494
* local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL.
495-
* If we were to use __flush_tlb_single() and set local_tlb_gen to
495+
* If we were to use __flush_tlb_one_user() and set local_tlb_gen to
496496
* 3, we'd be break the invariant: we'd update local_tlb_gen above
497497
* 1 without the full flush that's needed for tlb_gen 2.
498498
*
@@ -513,7 +513,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
513513

514514
addr = f->start;
515515
while (addr < f->end) {
516-
__flush_tlb_single(addr);
516+
__flush_tlb_one_user(addr);
517517
addr += PAGE_SIZE;
518518
}
519519
if (local)
@@ -660,7 +660,7 @@ static void do_kernel_range_flush(void *info)
660660

661661
/* flush range by one by one 'invlpg' */
662662
for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
663-
__flush_tlb_one(addr);
663+
__flush_tlb_one_kernel(addr);
664664
}
665665

666666
void flush_tlb_kernel_range(unsigned long start, unsigned long end)

arch/x86/platform/uv/tlb_uv.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ static void bau_process_message(struct msg_desc *mdp, struct bau_control *bcp,
299299
local_flush_tlb();
300300
stat->d_alltlb++;
301301
} else {
302-
__flush_tlb_single(msg->address);
302+
__flush_tlb_one_user(msg->address);
303303
stat->d_onetlb++;
304304
}
305305
stat->d_requestee++;

arch/x86/xen/mmu_pv.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1300,12 +1300,12 @@ static void xen_flush_tlb(void)
13001300
preempt_enable();
13011301
}
13021302

1303-
static void xen_flush_tlb_single(unsigned long addr)
1303+
static void xen_flush_tlb_one_user(unsigned long addr)
13041304
{
13051305
struct mmuext_op *op;
13061306
struct multicall_space mcs;
13071307

1308-
trace_xen_mmu_flush_tlb_single(addr);
1308+
trace_xen_mmu_flush_tlb_one_user(addr);
13091309

13101310
preempt_disable();
13111311

@@ -2370,7 +2370,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
23702370

23712371
.flush_tlb_user = xen_flush_tlb,
23722372
.flush_tlb_kernel = xen_flush_tlb,
2373-
.flush_tlb_single = xen_flush_tlb_single,
2373+
.flush_tlb_one_user = xen_flush_tlb_one_user,
23742374
.flush_tlb_others = xen_flush_tlb_others,
23752375

23762376
.pgd_alloc = xen_pgd_alloc,

include/trace/events/xen.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ TRACE_EVENT(xen_mmu_flush_tlb,
368368
TP_printk("%s", "")
369369
);
370370

371-
TRACE_EVENT(xen_mmu_flush_tlb_single,
371+
TRACE_EVENT(xen_mmu_flush_tlb_one_user,
372372
TP_PROTO(unsigned long addr),
373373
TP_ARGS(addr),
374374
TP_STRUCT__entry(

0 commit comments

Comments
 (0)