Skip to content

Commit 35754c9

Browse files
bonzinibonzini
committed
KVM: x86: introduce lapic_in_kernel
Avoid pointer chasing and memory barriers, and simplify the code when split irqchip (LAPIC in kernel, IOAPIC/PIC in userspace) is introduced. Signed-off-by: Paolo Bonzini <[email protected]>
1 parent d50ab6c commit 35754c9

File tree

7 files changed

+45
-43
lines changed

7 files changed

+45
-43
lines changed

‎arch/x86/kvm/irq.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ static int kvm_cpu_has_extint(struct kvm_vcpu *v)
5757
*/
5858
int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v)
5959
{
60-
if (!irqchip_in_kernel(v->kvm))
60+
if (!lapic_in_kernel(v))
6161
return v->arch.interrupt.pending;
6262

6363
if (kvm_cpu_has_extint(v))
@@ -75,7 +75,7 @@ int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v)
7575
*/
7676
int kvm_cpu_has_interrupt(struct kvm_vcpu *v)
7777
{
78-
if (!irqchip_in_kernel(v->kvm))
78+
if (!lapic_in_kernel(v))
7979
return v->arch.interrupt.pending;
8080

8181
if (kvm_cpu_has_extint(v))
@@ -103,7 +103,7 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v)
103103
{
104104
int vector;
105105

106-
if (!irqchip_in_kernel(v->kvm))
106+
if (!lapic_in_kernel(v))
107107
return v->arch.interrupt.nr;
108108

109109
vector = kvm_cpu_get_extint(v);

‎arch/x86/kvm/irq.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,14 @@ static inline int irqchip_in_kernel(struct kvm *kvm)
9292
return vpic != NULL;
9393
}
9494

95+
static inline int lapic_in_kernel(struct kvm_vcpu *vcpu)
96+
{
97+
/* Same as irqchip_in_kernel(vcpu->kvm), but with less
98+
* pointer chasing and no unnecessary memory barriers.
99+
*/
100+
return vcpu->arch.apic != NULL;
101+
}
102+
95103
void kvm_pic_reset(struct kvm_kpic_state *s);
96104

97105
void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu);

‎arch/x86/kvm/lapic.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1985,7 +1985,7 @@ int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
19851985
struct kvm_lapic *apic = vcpu->arch.apic;
19861986
u32 reg = (msr - APIC_BASE_MSR) << 4;
19871987

1988-
if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic))
1988+
if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
19891989
return 1;
19901990

19911991
if (reg == APIC_ICR2)
@@ -2002,7 +2002,7 @@ int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
20022002
struct kvm_lapic *apic = vcpu->arch.apic;
20032003
u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
20042004

2005-
if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic))
2005+
if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
20062006
return 1;
20072007

20082008
if (reg == APIC_DFR || reg == APIC_ICR2) {

‎arch/x86/kvm/mmu.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3427,7 +3427,7 @@ static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
34273427

34283428
static bool can_do_async_pf(struct kvm_vcpu *vcpu)
34293429
{
3430-
if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
3430+
if (unlikely(!lapic_in_kernel(vcpu) ||
34313431
kvm_event_needs_reinjection(vcpu)))
34323432
return false;
34333433

‎arch/x86/kvm/svm.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3060,7 +3060,7 @@ static int cr8_write_interception(struct vcpu_svm *svm)
30603060
u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
30613061
/* instruction emulation calls kvm_set_cr8() */
30623062
r = cr_interception(svm);
3063-
if (irqchip_in_kernel(svm->vcpu.kvm))
3063+
if (lapic_in_kernel(&svm->vcpu))
30643064
return r;
30653065
if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
30663066
return r;
@@ -3305,7 +3305,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm)
33053305
* If the user space waits to inject interrupts, exit as soon as
33063306
* possible
33073307
*/
3308-
if (!irqchip_in_kernel(svm->vcpu.kvm) &&
3308+
if (!lapic_in_kernel(&svm->vcpu) &&
33093309
kvm_run->request_interrupt_window &&
33103310
!kvm_cpu_has_interrupt(&svm->vcpu)) {
33113311
kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;

‎arch/x86/kvm/vmx.c

Lines changed: 20 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -809,7 +809,6 @@ static void kvm_cpu_vmxon(u64 addr);
809809
static void kvm_cpu_vmxoff(void);
810810
static bool vmx_mpx_supported(void);
811811
static bool vmx_xsaves_supported(void);
812-
static int vmx_vm_has_apicv(struct kvm *kvm);
813812
static int vmx_cpu_uses_apicv(struct kvm_vcpu *vcpu);
814813
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
815814
static void vmx_set_segment(struct kvm_vcpu *vcpu,
@@ -947,9 +946,9 @@ static inline bool cpu_has_vmx_tpr_shadow(void)
947946
return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
948947
}
949948

950-
static inline bool vm_need_tpr_shadow(struct kvm *kvm)
949+
static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
951950
{
952-
return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
951+
return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
953952
}
954953

955954
static inline bool cpu_has_secondary_exec_ctrls(void)
@@ -1063,9 +1062,9 @@ static inline bool cpu_has_vmx_ple(void)
10631062
SECONDARY_EXEC_PAUSE_LOOP_EXITING;
10641063
}
10651064

1066-
static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
1065+
static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
10671066
{
1068-
return flexpriority_enabled && irqchip_in_kernel(kvm);
1067+
return flexpriority_enabled && lapic_in_kernel(vcpu);
10691068
}
10701069

10711070
static inline bool cpu_has_vmx_vpid(void)
@@ -2378,7 +2377,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
23782377
vmx->nested.nested_vmx_pinbased_ctls_high |=
23792378
PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
23802379
PIN_BASED_VMX_PREEMPTION_TIMER;
2381-
if (vmx_vm_has_apicv(vmx->vcpu.kvm))
2380+
if (vmx_cpu_uses_apicv(&vmx->vcpu))
23822381
vmx->nested.nested_vmx_pinbased_ctls_high |=
23832382
PIN_BASED_POSTED_INTR;
23842383

@@ -4333,14 +4332,9 @@ static void vmx_disable_intercept_msr_write_x2apic(u32 msr)
43334332
msr, MSR_TYPE_W);
43344333
}
43354334

4336-
static int vmx_vm_has_apicv(struct kvm *kvm)
4337-
{
4338-
return enable_apicv && irqchip_in_kernel(kvm);
4339-
}
4340-
43414335
static int vmx_cpu_uses_apicv(struct kvm_vcpu *vcpu)
43424336
{
4343-
return vmx_vm_has_apicv(vcpu->kvm);
4337+
return enable_apicv && lapic_in_kernel(vcpu);
43444338
}
43454339

43464340
static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
@@ -4520,7 +4514,7 @@ static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
45204514
{
45214515
u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
45224516

4523-
if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
4517+
if (!vmx_cpu_uses_apicv(&vmx->vcpu))
45244518
pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
45254519
return pin_based_exec_ctrl;
45264520
}
@@ -4532,7 +4526,7 @@ static u32 vmx_exec_control(struct vcpu_vmx *vmx)
45324526
if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
45334527
exec_control &= ~CPU_BASED_MOV_DR_EXITING;
45344528

4535-
if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
4529+
if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
45364530
exec_control &= ~CPU_BASED_TPR_SHADOW;
45374531
#ifdef CONFIG_X86_64
45384532
exec_control |= CPU_BASED_CR8_STORE_EXITING |
@@ -4549,7 +4543,7 @@ static u32 vmx_exec_control(struct vcpu_vmx *vmx)
45494543
static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
45504544
{
45514545
u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
4552-
if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
4546+
if (!cpu_need_virtualize_apic_accesses(&vmx->vcpu))
45534547
exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
45544548
if (vmx->vpid == 0)
45554549
exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
@@ -4563,7 +4557,7 @@ static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
45634557
exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
45644558
if (!ple_gap)
45654559
exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
4566-
if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
4560+
if (!vmx_cpu_uses_apicv(&vmx->vcpu))
45674561
exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
45684562
SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
45694563
exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
@@ -4624,7 +4618,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
46244618
vmx_secondary_exec_control(vmx));
46254619
}
46264620

4627-
if (vmx_vm_has_apicv(vmx->vcpu.kvm)) {
4621+
if (vmx_cpu_uses_apicv(&vmx->vcpu)) {
46284622
vmcs_write64(EOI_EXIT_BITMAP0, 0);
46294623
vmcs_write64(EOI_EXIT_BITMAP1, 0);
46304624
vmcs_write64(EOI_EXIT_BITMAP2, 0);
@@ -4768,15 +4762,15 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
47684762

47694763
if (cpu_has_vmx_tpr_shadow() && !init_event) {
47704764
vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
4771-
if (vm_need_tpr_shadow(vcpu->kvm))
4765+
if (cpu_need_tpr_shadow(vcpu))
47724766
vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
47734767
__pa(vcpu->arch.apic->regs));
47744768
vmcs_write32(TPR_THRESHOLD, 0);
47754769
}
47764770

47774771
kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
47784772

4779-
if (vmx_vm_has_apicv(vcpu->kvm))
4773+
if (vmx_cpu_uses_apicv(vcpu))
47804774
memset(&vmx->pi_desc, 0, sizeof(struct pi_desc));
47814775

47824776
if (vmx->vpid != 0)
@@ -5316,7 +5310,7 @@ static int handle_cr(struct kvm_vcpu *vcpu)
53165310
u8 cr8 = (u8)val;
53175311
err = kvm_set_cr8(vcpu, cr8);
53185312
kvm_complete_insn_gp(vcpu, err);
5319-
if (irqchip_in_kernel(vcpu->kvm))
5313+
if (lapic_in_kernel(vcpu))
53205314
return 1;
53215315
if (cr8_prev <= cr8)
53225316
return 1;
@@ -5535,7 +5529,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
55355529
* If the user space waits to inject interrupts, exit as soon as
55365530
* possible
55375531
*/
5538-
if (!irqchip_in_kernel(vcpu->kvm) &&
5532+
if (!lapic_in_kernel(vcpu) &&
55395533
vcpu->run->request_interrupt_window &&
55405534
!kvm_cpu_has_interrupt(vcpu)) {
55415535
vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
@@ -7944,10 +7938,10 @@ static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
79447938
* apicv
79457939
*/
79467940
if (!cpu_has_vmx_virtualize_x2apic_mode() ||
7947-
!vmx_vm_has_apicv(vcpu->kvm))
7941+
!vmx_cpu_uses_apicv(vcpu))
79487942
return;
79497943

7950-
if (!vm_need_tpr_shadow(vcpu->kvm))
7944+
if (!cpu_need_tpr_shadow(vcpu))
79517945
return;
79527946

79537947
sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
@@ -8052,7 +8046,7 @@ static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
80528046
static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu)
80538047
{
80548048
u64 *eoi_exit_bitmap = vcpu->arch.eoi_exit_bitmap;
8055-
if (!vmx_vm_has_apicv(vcpu->kvm))
8049+
if (!vmx_cpu_uses_apicv(vcpu))
80568050
return;
80578051

80588052
vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
@@ -8551,7 +8545,7 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
85518545
put_cpu();
85528546
if (err)
85538547
goto free_vmcs;
8554-
if (vm_need_virtualize_apic_accesses(kvm)) {
8548+
if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
85558549
err = alloc_apic_access_page(kvm);
85568550
if (err)
85578551
goto free_vmcs;
@@ -9344,7 +9338,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
93449338
vmcs_write64(APIC_ACCESS_ADDR,
93459339
page_to_phys(vmx->nested.apic_access_page));
93469340
} else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
9347-
(vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))) {
9341+
cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
93489342
exec_control |=
93499343
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
93509344
kvm_vcpu_reload_apic_access_page(vcpu);

‎arch/x86/kvm/x86.c

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -788,7 +788,7 @@ int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
788788
{
789789
if (cr8 & CR8_RESERVED_BITS)
790790
return 1;
791-
if (irqchip_in_kernel(vcpu->kvm))
791+
if (lapic_in_kernel(vcpu))
792792
kvm_lapic_set_tpr(vcpu, cr8);
793793
else
794794
vcpu->arch.cr8 = cr8;
@@ -798,7 +798,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr8);
798798

799799
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
800800
{
801-
if (irqchip_in_kernel(vcpu->kvm))
801+
if (lapic_in_kernel(vcpu))
802802
return kvm_lapic_get_cr8(vcpu);
803803
else
804804
return vcpu->arch.cr8;
@@ -3175,7 +3175,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
31753175
struct kvm_vapic_addr va;
31763176

31773177
r = -EINVAL;
3178-
if (!irqchip_in_kernel(vcpu->kvm))
3178+
if (!lapic_in_kernel(vcpu))
31793179
goto out;
31803180
r = -EFAULT;
31813181
if (copy_from_user(&va, argp, sizeof va))
@@ -5666,7 +5666,7 @@ void kvm_arch_exit(void)
56665666
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
56675667
{
56685668
++vcpu->stat.halt_exits;
5669-
if (irqchip_in_kernel(vcpu->kvm)) {
5669+
if (lapic_in_kernel(vcpu)) {
56705670
vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
56715671
return 1;
56725672
} else {
@@ -6162,7 +6162,7 @@ void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
61626162
{
61636163
struct page *page = NULL;
61646164

6165-
if (!irqchip_in_kernel(vcpu->kvm))
6165+
if (!lapic_in_kernel(vcpu))
61666166
return;
61676167

61686168
if (!kvm_x86_ops->set_apic_access_page_addr)
@@ -6200,7 +6200,7 @@ void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
62006200
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
62016201
{
62026202
int r;
6203-
bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
6203+
bool req_int_win = !lapic_in_kernel(vcpu) &&
62046204
vcpu->run->request_interrupt_window;
62056205
bool req_immediate_exit = false;
62066206

@@ -6597,7 +6597,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
65976597
}
65986598

65996599
/* re-sync apic's tpr */
6600-
if (!irqchip_in_kernel(vcpu->kvm)) {
6600+
if (!lapic_in_kernel(vcpu)) {
66016601
if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
66026602
r = -EINVAL;
66036603
goto out;
@@ -7297,7 +7297,7 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
72977297

72987298
bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
72997299
{
7300-
return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
7300+
return irqchip_in_kernel(vcpu->kvm) == lapic_in_kernel(vcpu);
73017301
}
73027302

73037303
struct static_key kvm_no_apic_vcpu __read_mostly;
@@ -7391,7 +7391,7 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
73917391
kvm_mmu_destroy(vcpu);
73927392
srcu_read_unlock(&vcpu->kvm->srcu, idx);
73937393
free_page((unsigned long)vcpu->arch.pio_data);
7394-
if (!irqchip_in_kernel(vcpu->kvm))
7394+
if (!lapic_in_kernel(vcpu))
73957395
static_key_slow_dec(&kvm_no_apic_vcpu);
73967396
}
73977397

0 commit comments

Comments
 (0)