Skip to content

Commit 1e02ce4

Browse files
amlutoIngo Molnar
amluto
authored and
Ingo Molnar
committed
x86: Store a per-cpu shadow copy of CR4
Context switches and TLB flushes can change individual bits of CR4. CR4 reads take several cycles, so store a shadow copy of CR4 in a per-cpu variable. To avoid wasting a cache line, I added the CR4 shadow to cpu_tlbstate, which is already touched in switch_mm. The heaviest users of the cr4 shadow will be switch_mm and __switch_to_xtra, and __switch_to_xtra is called shortly after switch_mm during context switch, so the cacheline is likely to be hot. Signed-off-by: Andy Lutomirski <[email protected]> Reviewed-by: Thomas Gleixner <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Cc: Kees Cook <[email protected]> Cc: Andrea Arcangeli <[email protected]> Cc: Vince Weaver <[email protected]> Cc: "hillf.zj" <[email protected]> Cc: Valdis Kletnieks <[email protected]> Cc: Paul Mackerras <[email protected]> Cc: Arnaldo Carvalho de Melo <[email protected]> Cc: Linus Torvalds <[email protected]> Link: http://lkml.kernel.org/r/3a54dd3353fffbf84804398e00dfdc5b7c1afd7d.1414190806.git.luto@amacapital.net Signed-off-by: Ingo Molnar <[email protected]>
1 parent 375074c commit 1e02ce4

File tree

20 files changed

+85
-46
lines changed

20 files changed

+85
-46
lines changed

‎arch/x86/include/asm/paravirt.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,16 +80,16 @@ static inline void write_cr3(unsigned long x)
8080
PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
8181
}
8282

83-
static inline unsigned long read_cr4(void)
83+
static inline unsigned long __read_cr4(void)
8484
{
8585
return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
8686
}
87-
static inline unsigned long read_cr4_safe(void)
87+
static inline unsigned long __read_cr4_safe(void)
8888
{
8989
return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
9090
}
9191

92-
static inline void write_cr4(unsigned long x)
92+
static inline void __write_cr4(unsigned long x)
9393
{
9494
PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
9595
}

‎arch/x86/include/asm/special_insns.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,17 +137,17 @@ static inline void write_cr3(unsigned long x)
137137
native_write_cr3(x);
138138
}
139139

140-
static inline unsigned long read_cr4(void)
140+
static inline unsigned long __read_cr4(void)
141141
{
142142
return native_read_cr4();
143143
}
144144

145-
static inline unsigned long read_cr4_safe(void)
145+
static inline unsigned long __read_cr4_safe(void)
146146
{
147147
return native_read_cr4_safe();
148148
}
149149

150-
static inline void write_cr4(unsigned long x)
150+
static inline void __write_cr4(unsigned long x)
151151
{
152152
native_write_cr4(x);
153153
}

‎arch/x86/include/asm/tlbflush.h

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,56 @@
1515
#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
1616
#endif
1717

18+
struct tlb_state {
19+
#ifdef CONFIG_SMP
20+
struct mm_struct *active_mm;
21+
int state;
22+
#endif
23+
24+
/*
25+
* Access to this CR4 shadow and to H/W CR4 is protected by
26+
* disabling interrupts when modifying either one.
27+
*/
28+
unsigned long cr4;
29+
};
30+
DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
31+
32+
/* Initialize cr4 shadow for this CPU. */
33+
static inline void cr4_init_shadow(void)
34+
{
35+
this_cpu_write(cpu_tlbstate.cr4, __read_cr4());
36+
}
37+
1838
/* Set in this cpu's CR4. */
1939
static inline void cr4_set_bits(unsigned long mask)
2040
{
2141
unsigned long cr4;
2242

23-
cr4 = read_cr4();
24-
cr4 |= mask;
25-
write_cr4(cr4);
43+
cr4 = this_cpu_read(cpu_tlbstate.cr4);
44+
if ((cr4 | mask) != cr4) {
45+
cr4 |= mask;
46+
this_cpu_write(cpu_tlbstate.cr4, cr4);
47+
__write_cr4(cr4);
48+
}
2649
}
2750

2851
/* Clear in this cpu's CR4. */
2952
static inline void cr4_clear_bits(unsigned long mask)
3053
{
3154
unsigned long cr4;
3255

33-
cr4 = read_cr4();
34-
cr4 &= ~mask;
35-
write_cr4(cr4);
56+
cr4 = this_cpu_read(cpu_tlbstate.cr4);
57+
if ((cr4 & ~mask) != cr4) {
58+
cr4 &= ~mask;
59+
this_cpu_write(cpu_tlbstate.cr4, cr4);
60+
__write_cr4(cr4);
61+
}
62+
}
63+
64+
/* Read the CR4 shadow. */
65+
static inline unsigned long cr4_read_shadow(void)
66+
{
67+
return this_cpu_read(cpu_tlbstate.cr4);
3668
}
3769

3870
/*
@@ -61,7 +93,7 @@ static inline void __native_flush_tlb_global_irq_disabled(void)
6193
{
6294
unsigned long cr4;
6395

64-
cr4 = native_read_cr4();
96+
cr4 = this_cpu_read(cpu_tlbstate.cr4);
6597
/* clear PGE */
6698
native_write_cr4(cr4 & ~X86_CR4_PGE);
6799
/* write old PGE again and flush TLBs */
@@ -221,12 +253,6 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
221253
#define TLBSTATE_OK 1
222254
#define TLBSTATE_LAZY 2
223255

224-
struct tlb_state {
225-
struct mm_struct *active_mm;
226-
int state;
227-
};
228-
DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
229-
230256
static inline void reset_lazy_tlbstate(void)
231257
{
232258
this_cpu_write(cpu_tlbstate.state, 0);

‎arch/x86/include/asm/virtext.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ static inline void cpu_vmxoff(void)
4646

4747
static inline int cpu_vmx_enabled(void)
4848
{
49-
return read_cr4() & X86_CR4_VMXE;
49+
return __read_cr4() & X86_CR4_VMXE;
5050
}
5151

5252
/** Disable VMX if it is enabled on the current CPU

‎arch/x86/kernel/acpi/sleep.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ int x86_acpi_suspend_lowlevel(void)
7878

7979
header->pmode_cr0 = read_cr0();
8080
if (__this_cpu_read(cpu_info.cpuid_level) >= 0) {
81-
header->pmode_cr4 = read_cr4();
81+
header->pmode_cr4 = __read_cr4();
8282
header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_CR4);
8383
}
8484
if (!rdmsr_safe(MSR_IA32_MISC_ENABLE,

‎arch/x86/kernel/cpu/common.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include <asm/archrandom.h>
2020
#include <asm/hypervisor.h>
2121
#include <asm/processor.h>
22+
#include <asm/tlbflush.h>
2223
#include <asm/debugreg.h>
2324
#include <asm/sections.h>
2425
#include <asm/vsyscall.h>
@@ -1293,6 +1294,12 @@ void cpu_init(void)
12931294

12941295
wait_for_master_cpu(cpu);
12951296

1297+
/*
1298+
* Initialize the CR4 shadow before doing anything that could
1299+
* try to read it.
1300+
*/
1301+
cr4_init_shadow();
1302+
12961303
/*
12971304
* Load microcode on this cpu if a valid microcode is available.
12981305
* This is early microcode loading procedure.

‎arch/x86/kernel/cpu/mtrr/cyrix.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,8 @@ static void prepare_set(void)
138138

139139
/* Save value of CR4 and clear Page Global Enable (bit 7) */
140140
if (cpu_has_pge) {
141-
cr4 = read_cr4();
142-
write_cr4(cr4 & ~X86_CR4_PGE);
141+
cr4 = __read_cr4();
142+
__write_cr4(cr4 & ~X86_CR4_PGE);
143143
}
144144

145145
/*
@@ -171,7 +171,7 @@ static void post_set(void)
171171

172172
/* Restore value of CR4 */
173173
if (cpu_has_pge)
174-
write_cr4(cr4);
174+
__write_cr4(cr4);
175175
}
176176

177177
static void cyrix_set_arr(unsigned int reg, unsigned long base,

‎arch/x86/kernel/cpu/mtrr/generic.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -678,8 +678,8 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
678678

679679
/* Save value of CR4 and clear Page Global Enable (bit 7) */
680680
if (cpu_has_pge) {
681-
cr4 = read_cr4();
682-
write_cr4(cr4 & ~X86_CR4_PGE);
681+
cr4 = __read_cr4();
682+
__write_cr4(cr4 & ~X86_CR4_PGE);
683683
}
684684

685685
/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
@@ -708,7 +708,7 @@ static void post_set(void) __releases(set_atomicity_lock)
708708

709709
/* Restore value of CR4 */
710710
if (cpu_has_pge)
711-
write_cr4(cr4);
711+
__write_cr4(cr4);
712712
raw_spin_unlock(&set_atomicity_lock);
713713
}
714714

‎arch/x86/kernel/head32.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ static void __init i386_default_early_setup(void)
3131

3232
asmlinkage __visible void __init i386_start_kernel(void)
3333
{
34+
cr4_init_shadow();
3435
sanitize_boot_params(&boot_params);
3536

3637
/* Call the subarch specific early setup function */

‎arch/x86/kernel/head64.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
155155
(__START_KERNEL & PGDIR_MASK)));
156156
BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
157157

158+
cr4_init_shadow();
159+
158160
/* Kill off the identity-map trampoline */
159161
reset_early_page_tables();
160162

‎arch/x86/kernel/process_32.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ void __show_regs(struct pt_regs *regs, int all)
101101
cr0 = read_cr0();
102102
cr2 = read_cr2();
103103
cr3 = read_cr3();
104-
cr4 = read_cr4_safe();
104+
cr4 = __read_cr4_safe();
105105
printk(KERN_DEFAULT "CR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
106106
cr0, cr2, cr3, cr4);
107107

‎arch/x86/kernel/process_64.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ void __show_regs(struct pt_regs *regs, int all)
9393
cr0 = read_cr0();
9494
cr2 = read_cr2();
9595
cr3 = read_cr3();
96-
cr4 = read_cr4();
96+
cr4 = __read_cr4();
9797

9898
printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
9999
fs, fsindex, gs, gsindex, shadowgs);

‎arch/x86/kernel/setup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1178,7 +1178,7 @@ void __init setup_arch(char **cmdline_p)
11781178

11791179
if (boot_cpu_data.cpuid_level >= 0) {
11801180
/* A CPU has %cr4 if and only if it has CPUID */
1181-
mmu_cr4_features = read_cr4();
1181+
mmu_cr4_features = __read_cr4();
11821182
if (trampoline_cr4_features)
11831183
*trampoline_cr4_features = mmu_cr4_features;
11841184
}

‎arch/x86/kvm/svm.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1583,7 +1583,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
15831583

15841584
static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
15851585
{
1586-
unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
1586+
unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
15871587
unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
15881588

15891589
if (cr4 & X86_CR4_VMXE)

‎arch/x86/kvm/vmx.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2785,7 +2785,7 @@ static int hardware_enable(void)
27852785
u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
27862786
u64 old, test_bits;
27872787

2788-
if (read_cr4() & X86_CR4_VMXE)
2788+
if (cr4_read_shadow() & X86_CR4_VMXE)
27892789
return -EBUSY;
27902790

27912791
INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
@@ -4255,7 +4255,7 @@ static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
42554255
vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
42564256

42574257
/* Save the most likely value for this task's CR4 in the VMCS. */
4258-
cr4 = read_cr4();
4258+
cr4 = cr4_read_shadow();
42594259
vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */
42604260
vmx->host_state.vmcs_host_cr4 = cr4;
42614261

@@ -7784,7 +7784,7 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
77847784
if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
77857785
vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
77867786

7787-
cr4 = read_cr4();
7787+
cr4 = cr4_read_shadow();
77887788
if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
77897789
vmcs_writel(HOST_CR4, cr4);
77907790
vmx->host_state.vmcs_host_cr4 = cr4;

‎arch/x86/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -600,7 +600,7 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code,
600600
printk(nx_warning, from_kuid(&init_user_ns, current_uid()));
601601
if (pte && pte_present(*pte) && pte_exec(*pte) &&
602602
(pgd_flags(*pgd) & _PAGE_USER) &&
603-
(read_cr4() & X86_CR4_SMEP))
603+
(__read_cr4() & X86_CR4_SMEP))
604604
printk(smep_warning, from_kuid(&init_user_ns, current_uid()));
605605
}
606606

‎arch/x86/mm/init.c

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,15 @@ void __init zone_sizes_init(void)
713713
free_area_init_nodes(max_zone_pfns);
714714
}
715715

716+
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = {
717+
#ifdef CONFIG_SMP
718+
.active_mm = &init_mm,
719+
.state = 0,
720+
#endif
721+
.cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */
722+
};
723+
EXPORT_SYMBOL_GPL(cpu_tlbstate);
724+
716725
void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
717726
{
718727
/* entry 0 MUST be WB (hardwired to speed up translations) */

‎arch/x86/mm/tlb.c

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,6 @@
1414
#include <asm/uv/uv.h>
1515
#include <linux/debugfs.h>
1616

17-
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
18-
= { &init_mm, 0, };
19-
2017
/*
2118
* Smarter SMP flushing macros.
2219
* c/o Linus Torvalds.

‎arch/x86/power/cpu.c

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -105,11 +105,8 @@ static void __save_processor_state(struct saved_context *ctxt)
105105
ctxt->cr0 = read_cr0();
106106
ctxt->cr2 = read_cr2();
107107
ctxt->cr3 = read_cr3();
108-
#ifdef CONFIG_X86_32
109-
ctxt->cr4 = read_cr4_safe();
110-
#else
111-
/* CONFIG_X86_64 */
112-
ctxt->cr4 = read_cr4();
108+
ctxt->cr4 = __read_cr4_safe();
109+
#ifdef CONFIG_X86_64
113110
ctxt->cr8 = read_cr8();
114111
#endif
115112
ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
@@ -175,12 +172,12 @@ static void notrace __restore_processor_state(struct saved_context *ctxt)
175172
/* cr4 was introduced in the Pentium CPU */
176173
#ifdef CONFIG_X86_32
177174
if (ctxt->cr4)
178-
write_cr4(ctxt->cr4);
175+
__write_cr4(ctxt->cr4);
179176
#else
180177
/* CONFIG X86_64 */
181178
wrmsrl(MSR_EFER, ctxt->efer);
182179
write_cr8(ctxt->cr8);
183-
write_cr4(ctxt->cr4);
180+
__write_cr4(ctxt->cr4);
184181
#endif
185182
write_cr3(ctxt->cr3);
186183
write_cr2(ctxt->cr2);

‎arch/x86/realmode/init.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ void __init setup_real_mode(void)
8181

8282
trampoline_header->start = (u64) secondary_startup_64;
8383
trampoline_cr4_features = &trampoline_header->cr4;
84-
*trampoline_cr4_features = read_cr4();
84+
*trampoline_cr4_features = __read_cr4();
8585

8686
trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
8787
trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;

0 commit comments

Comments
 (0)