Lines Matching +full:is +full:- +full:decoded +full:- +full:cs
18 #include <linux/amd-iommu.h>
24 #include <linux/psp-sev.h>
40 #include <asm/spec-ctrl.h>
57 MODULE_DESCRIPTION("KVM support for SVM (AMD-V) extensions");
87 bool always; /* True if intercept is initially cleared */
133 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
150 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
153 * count value. On VMRUN this value is loaded into an internal counter.
154 * Each time a pause instruction is executed, this counter is decremented
155 * until it reaches zero at which time a #VMEXIT is generated if pause
156 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause
162 * the amount of time a guest is allowed to execute in a pause loop.
163 * In this mode, a 16-bit pause filter threshold field is added in the
164 * VMCB. The threshold value is a cycle count that is used to reset the
169 * If the elapsed cycle count is greater than the pause filter threshold,
170 * then the internal pause count is reloaded from the VMCB and execution
171 * continues. If the elapsed cycle count is less than the pause filter
172 * threshold, then the internal pause count is decremented. If the count
173 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
174 * triggered. If advanced pause filtering is supported and pause filter
175 * threshold field is set to zero, the filter will operate in the simpler,
185 /* Default doubles per-vcpu window every exit. */
189 /* Default resets per-vcpu window every exit to pause_filter_count. */
193 /* Default is to compute the maximum so we can never overflow. */
253 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via
259 static int tsc_aux_uret_slot __read_mostly = -1;
277 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ in svm_msrpm_offset()
280 /* Now we have the u8 offset - but need the u32 offset */ in svm_msrpm_offset()
300 u64 old_efer = vcpu->arch.efer; in svm_set_efer()
301 vcpu->arch.efer = efer; in svm_set_efer()
315 /* #GP intercept is still needed for vmware backdoor */ in svm_set_efer()
331 vcpu->arch.efer = old_efer; in svm_set_efer()
339 if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm)) in svm_set_efer()
344 svm->vmcb->save.efer = efer | EFER_SVME; in svm_set_efer()
345 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in svm_set_efer()
354 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) in svm_get_interrupt_shadow()
364 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; in svm_set_interrupt_shadow()
366 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; in svm_set_interrupt_shadow()
377 * SEV-ES does not expose the next RIP. The RIP update is controlled by in __svm_skip_emulated_instruction()
380 if (sev_es_guest(vcpu->kvm)) in __svm_skip_emulated_instruction()
383 if (nrips && svm->vmcb->control.next_rip != 0) { in __svm_skip_emulated_instruction()
385 svm->next_rip = svm->vmcb->control.next_rip; in __svm_skip_emulated_instruction()
388 if (!svm->next_rip) { in __svm_skip_emulated_instruction()
390 old_rflags = svm->vmcb->save.rflags; in __svm_skip_emulated_instruction()
396 svm->vmcb->save.rflags = old_rflags; in __svm_skip_emulated_instruction()
398 kvm_rip_write(vcpu, svm->next_rip); in __svm_skip_emulated_instruction()
422 * the instruction even if NextRIP is supported to acquire the next in svm_update_soft_interrupt_rip()
426 * retry the instruction, it's the least awful option. If NRIPS is in svm_update_soft_interrupt_rip()
431 return -EIO; in svm_update_soft_interrupt_rip()
437 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection in svm_update_soft_interrupt_rip()
438 * doesn't complete due to a VM-Exit occurring while the CPU is in svm_update_soft_interrupt_rip()
440 * work as there may be no backing instruction, e.g. if the event is in svm_update_soft_interrupt_rip()
441 * being injected by L1 for L2, or if the guest is patching INT3 into in svm_update_soft_interrupt_rip()
444 svm->soft_int_injected = true; in svm_update_soft_interrupt_rip()
445 svm->soft_int_csbase = svm->vmcb->save.cs.base; in svm_update_soft_interrupt_rip()
446 svm->soft_int_old_rip = old_rip; in svm_update_soft_interrupt_rip()
447 svm->soft_int_next_rip = rip; in svm_update_soft_interrupt_rip()
453 svm->vmcb->control.next_rip = rip; in svm_update_soft_interrupt_rip()
460 struct kvm_queued_exception *ex = &vcpu->arch.exception; in svm_inject_exception()
465 if (kvm_exception_is_soft(ex->vector) && in svm_inject_exception()
469 svm->vmcb->control.event_inj = ex->vector in svm_inject_exception()
471 | (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0) in svm_inject_exception()
473 svm->vmcb->control.event_inj_err = ex->error_code; in svm_inject_exception()
506 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; in svm_init_osvw()
507 vcpu->arch.osvw.status = osvw_status & ~(6ULL); in svm_init_osvw()
511 * all osvw.status bits inside that length, including bit 0 (which is in svm_init_osvw()
513 * osvw_len is 0 then osvw_status[0] carries no information. We need to in svm_init_osvw()
515 * is present (because we really don't know). in svm_init_osvw()
518 vcpu->arch.osvw.status |= 1; in svm_init_osvw()
526 if (c->x86_vendor != X86_VENDOR_AMD && in __kvm_is_svm_supported()
527 c->x86_vendor != X86_VENDOR_HYGON) { in __kvm_is_svm_supported()
538 pr_info("KVM is unsupported when running as an SEV guest\n"); in __kvm_is_svm_supported()
559 return -EIO; in svm_check_processor_compat()
575 return &sd->save_area->host_sev_es_save; in sev_es_host_save_area()
621 return -EBUSY; in svm_enable_virtualization_cpu()
624 sd->asid_generation = 1; in svm_enable_virtualization_cpu()
625 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; in svm_enable_virtualization_cpu()
626 sd->next_asid = sd->max_asid + 1; in svm_enable_virtualization_cpu()
627 sd->min_asid = max_sev_asid + 1; in svm_enable_virtualization_cpu()
631 wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); in svm_enable_virtualization_cpu()
645 * Note that it is possible to have a system with mixed processor in svm_enable_virtualization_cpu()
648 * is present on one processor and not on another then assume that the in svm_enable_virtualization_cpu()
649 * erratum is present everywhere). in svm_enable_virtualization_cpu()
666 osvw_status &= (1ULL << osvw_len) - 1; in svm_enable_virtualization_cpu()
676 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type in svm_enable_virtualization_cpu()
677 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests. in svm_enable_virtualization_cpu()
684 rdmsr(MSR_TSC_AUX, sev_es_host_save_area(sd)->tsc_aux, msr_hi); in svm_enable_virtualization_cpu()
694 if (!sd->save_area) in svm_cpu_uninit()
697 kfree(sd->sev_vmcbs); in svm_cpu_uninit()
698 __free_page(__sme_pa_to_page(sd->save_area_pa)); in svm_cpu_uninit()
699 sd->save_area_pa = 0; in svm_cpu_uninit()
700 sd->save_area = NULL; in svm_cpu_uninit()
707 int ret = -ENOMEM; in svm_cpu_init()
718 sd->save_area = page_address(save_area_page); in svm_cpu_init()
719 sd->save_area_pa = __sme_page_pa(save_area_page); in svm_cpu_init()
730 struct vmcb *vmcb = svm->vmcb01.ptr; in set_dr_intercepts()
732 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ); in set_dr_intercepts()
733 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ); in set_dr_intercepts()
734 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ); in set_dr_intercepts()
735 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ); in set_dr_intercepts()
736 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ); in set_dr_intercepts()
737 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ); in set_dr_intercepts()
738 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ); in set_dr_intercepts()
739 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE); in set_dr_intercepts()
740 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE); in set_dr_intercepts()
741 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE); in set_dr_intercepts()
742 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE); in set_dr_intercepts()
743 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE); in set_dr_intercepts()
744 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE); in set_dr_intercepts()
745 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE); in set_dr_intercepts()
746 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ); in set_dr_intercepts()
747 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE); in set_dr_intercepts()
754 struct vmcb *vmcb = svm->vmcb01.ptr; in clr_dr_intercepts()
756 vmcb->control.intercepts[INTERCEPT_DR] = 0; in clr_dr_intercepts()
769 return -ENOENT; in direct_access_msr_slot()
778 if (slot == -ENOENT) in set_shadow_msr_intercept()
783 set_bit(slot, svm->shadow_msr_intercept.read); in set_shadow_msr_intercept()
785 clear_bit(slot, svm->shadow_msr_intercept.read); in set_shadow_msr_intercept()
788 set_bit(slot, svm->shadow_msr_intercept.write); in set_shadow_msr_intercept()
790 clear_bit(slot, svm->shadow_msr_intercept.write); in set_shadow_msr_intercept()
795 return direct_access_msr_slot(index) != -ENOENT; in valid_msr_intercept()
806 * For non-nested case: in msr_write_intercepted()
814 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: in msr_write_intercepted()
815 to_svm(vcpu)->msrpm; in msr_write_intercepted()
860 svm->nested.force_msr_bitmap_recalc = true; in set_msr_interception_bitmap()
900 if (intercept == svm->x2avic_msrs_intercepted) in svm_set_x2apic_msr_interception()
912 set_msr_interception(&svm->vcpu, svm->msrpm, index, in svm_set_x2apic_msr_interception()
916 svm->x2avic_msrs_intercepted = intercept; in svm_set_x2apic_msr_interception()
936 u32 read = test_bit(i, svm->shadow_msr_intercept.read); in svm_msr_filter_changed()
937 u32 write = test_bit(i, svm->shadow_msr_intercept.write); in svm_msr_filter_changed()
939 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write); in svm_msr_filter_changed()
988 to_vmcb->save.dbgctl = from_vmcb->save.dbgctl; in svm_copy_lbrs()
989 to_vmcb->save.br_from = from_vmcb->save.br_from; in svm_copy_lbrs()
990 to_vmcb->save.br_to = from_vmcb->save.br_to; in svm_copy_lbrs()
991 to_vmcb->save.last_excp_from = from_vmcb->save.last_excp_from; in svm_copy_lbrs()
992 to_vmcb->save.last_excp_to = from_vmcb->save.last_excp_to; in svm_copy_lbrs()
1001 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; in svm_enable_lbrv()
1002 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); in svm_enable_lbrv()
1003 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); in svm_enable_lbrv()
1004 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); in svm_enable_lbrv()
1005 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); in svm_enable_lbrv()
1007 if (sev_es_guest(vcpu->kvm)) in svm_enable_lbrv()
1008 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1); in svm_enable_lbrv()
1012 svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr); in svm_enable_lbrv()
1019 KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm); in svm_disable_lbrv()
1021 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; in svm_disable_lbrv()
1022 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); in svm_disable_lbrv()
1023 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); in svm_disable_lbrv()
1024 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); in svm_disable_lbrv()
1025 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); in svm_disable_lbrv()
1032 svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb); in svm_disable_lbrv()
1038 * If LBR virtualization is disabled, the LBR MSRs are always kept in in svm_get_lbr_vmcb()
1039 * vmcb01. If LBR virtualization is enabled and L1 is running VMs of in svm_get_lbr_vmcb()
1042 return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb : in svm_get_lbr_vmcb()
1043 svm->vmcb01.ptr; in svm_get_lbr_vmcb()
1049 bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK; in svm_update_lbrv()
1050 bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) || in svm_update_lbrv()
1052 (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK)); in svm_update_lbrv()
1065 svm->nmi_singlestep = false; in disable_nmi_singlestep()
1067 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { in disable_nmi_singlestep()
1069 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) in disable_nmi_singlestep()
1070 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; in disable_nmi_singlestep()
1071 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) in disable_nmi_singlestep()
1072 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; in disable_nmi_singlestep()
1079 struct vmcb_control_area *control = &svm->vmcb->control; in grow_ple_window()
1080 int old = control->pause_filter_count; in grow_ple_window()
1082 if (kvm_pause_in_guest(vcpu->kvm)) in grow_ple_window()
1085 control->pause_filter_count = __grow_ple_window(old, in grow_ple_window()
1090 if (control->pause_filter_count != old) { in grow_ple_window()
1091 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); in grow_ple_window()
1092 trace_kvm_ple_window_update(vcpu->vcpu_id, in grow_ple_window()
1093 control->pause_filter_count, old); in grow_ple_window()
1100 struct vmcb_control_area *control = &svm->vmcb->control; in shrink_ple_window()
1101 int old = control->pause_filter_count; in shrink_ple_window()
1103 if (kvm_pause_in_guest(vcpu->kvm)) in shrink_ple_window()
1106 control->pause_filter_count = in shrink_ple_window()
1111 if (control->pause_filter_count != old) { in shrink_ple_window()
1112 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); in shrink_ple_window()
1113 trace_kvm_ple_window_update(vcpu->vcpu_id, in shrink_ple_window()
1114 control->pause_filter_count, old); in shrink_ple_window()
1133 seg->selector = 0; in init_seg()
1134 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | in init_seg()
1136 seg->limit = 0xffff; in init_seg()
1137 seg->base = 0; in init_seg()
1142 seg->selector = 0; in init_sys_seg()
1143 seg->attrib = SVM_SELECTOR_P_MASK | type; in init_sys_seg()
1144 seg->limit = 0xffff; in init_sys_seg()
1145 seg->base = 0; in init_sys_seg()
1152 return svm->nested.ctl.tsc_offset; in svm_get_l2_tsc_offset()
1159 return svm->tsc_ratio_msr; in svm_get_l2_tsc_multiplier()
1166 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; in svm_write_tsc_offset()
1167 svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; in svm_write_tsc_offset()
1168 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); in svm_write_tsc_offset()
1174 if (to_svm(vcpu)->guest_state_loaded) in svm_write_tsc_multiplier()
1175 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); in svm_write_tsc_multiplier()
1184 * Intercept INVPCID if shadow paging is enabled to sync/free shadow in svm_recalc_instruction_intercepts()
1185 * roots, or if INVPCID is disabled in the guest to inject #UD. in svm_recalc_instruction_intercepts()
1189 !guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_INVPCID)) in svm_recalc_instruction_intercepts()
1215 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; in init_vmcb_after_set_cpuid()
1217 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); in init_vmcb_after_set_cpuid()
1218 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); in init_vmcb_after_set_cpuid()
1227 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; in init_vmcb_after_set_cpuid()
1230 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); in init_vmcb_after_set_cpuid()
1231 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); in init_vmcb_after_set_cpuid()
1238 struct vmcb *vmcb = svm->vmcb01.ptr; in init_vmcb()
1239 struct vmcb_control_area *control = &vmcb->control; in init_vmcb()
1240 struct vmcb_save_area *save = &vmcb->save; in init_vmcb()
1295 if (!kvm_mwait_in_guest(vcpu->kvm)) { in init_vmcb()
1300 if (!kvm_hlt_in_guest(vcpu->kvm)) { in init_vmcb()
1307 control->iopm_base_pa = iopm_base; in init_vmcb()
1308 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); in init_vmcb()
1309 control->int_ctl = V_INTR_MASKING_MASK; in init_vmcb()
1311 init_seg(&save->es); in init_vmcb()
1312 init_seg(&save->ss); in init_vmcb()
1313 init_seg(&save->ds); in init_vmcb()
1314 init_seg(&save->fs); in init_vmcb()
1315 init_seg(&save->gs); in init_vmcb()
1317 save->cs.selector = 0xf000; in init_vmcb()
1318 save->cs.base = 0xffff0000; in init_vmcb()
1320 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | in init_vmcb()
1322 save->cs.limit = 0xffff; in init_vmcb()
1324 save->gdtr.base = 0; in init_vmcb()
1325 save->gdtr.limit = 0xffff; in init_vmcb()
1326 save->idtr.base = 0; in init_vmcb()
1327 save->idtr.limit = 0xffff; in init_vmcb()
1329 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); in init_vmcb()
1330 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); in init_vmcb()
1334 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; in init_vmcb()
1339 save->g_pat = vcpu->arch.pat; in init_vmcb()
1340 save->cr3 = 0; in init_vmcb()
1342 svm->current_vmcb->asid_generation = 0; in init_vmcb()
1343 svm->asid = 0; in init_vmcb()
1345 svm->nested.vmcb12_gpa = INVALID_GPA; in init_vmcb()
1346 svm->nested.last_vmcb12_gpa = INVALID_GPA; in init_vmcb()
1348 if (!kvm_pause_in_guest(vcpu->kvm)) { in init_vmcb()
1349 control->pause_filter_count = pause_filter_count; in init_vmcb()
1351 control->pause_filter_thresh = pause_filter_thresh; in init_vmcb()
1364 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); in init_vmcb()
1370 svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK; in init_vmcb()
1375 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; in init_vmcb()
1378 if (sev_guest(vcpu->kvm)) in init_vmcb()
1393 svm_vcpu_init_msrpm(vcpu, svm->msrpm); in __svm_vcpu_reset()
1397 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) in __svm_vcpu_reset()
1398 vcpu->arch.microcode_version = 0x01000065; in __svm_vcpu_reset()
1399 svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio; in __svm_vcpu_reset()
1401 svm->nmi_masked = false; in __svm_vcpu_reset()
1402 svm->awaiting_iret_completion = false; in __svm_vcpu_reset()
1404 if (sev_es_guest(vcpu->kvm)) in __svm_vcpu_reset()
1412 svm->spec_ctrl = 0; in svm_vcpu_reset()
1413 svm->virt_spec_ctrl = 0; in svm_vcpu_reset()
1426 svm->current_vmcb = target_vmcb; in svm_switch_vmcb()
1427 svm->vmcb = target_vmcb->ptr; in svm_switch_vmcb()
1440 err = -ENOMEM; in svm_vcpu_create()
1445 if (sev_es_guest(vcpu->kvm)) { in svm_vcpu_create()
1447 * SEV-ES guests require a separate VMSA page used to contain in svm_vcpu_create()
1459 svm->msrpm = svm_vcpu_alloc_msrpm(); in svm_vcpu_create()
1460 if (!svm->msrpm) { in svm_vcpu_create()
1461 err = -ENOMEM; in svm_vcpu_create()
1465 svm->x2avic_msrs_intercepted = true; in svm_vcpu_create()
1467 svm->vmcb01.ptr = page_address(vmcb01_page); in svm_vcpu_create()
1468 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); in svm_vcpu_create()
1469 svm_switch_vmcb(svm, &svm->vmcb01); in svm_vcpu_create()
1472 svm->sev_es.vmsa = page_address(vmsa_page); in svm_vcpu_create()
1474 svm->guest_state_loaded = false; in svm_vcpu_create()
1504 svm_clear_current_vmcb(svm->vmcb); in svm_vcpu_free()
1511 __free_page(__sme_pa_to_page(svm->vmcb01.pa)); in svm_vcpu_free()
1512 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); in svm_vcpu_free()
1523 if (!sd->bp_spec_reduce_set) in svm_srso_clear_bp_spec_reduce()
1527 sd->bp_spec_reduce_set = false; in svm_srso_clear_bp_spec_reduce()
1557 * transition, i.e. destroying the last VM, is fully complete, e.g. so in svm_srso_vm_init()
1575 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); in svm_prepare_switch_to_guest()
1577 if (sev_es_guest(vcpu->kvm)) in svm_prepare_switch_to_guest()
1580 if (svm->guest_state_loaded) in svm_prepare_switch_to_guest()
1584 * Save additional host state that will be restored on VMEXIT (sev-es) in svm_prepare_switch_to_guest()
1587 vmsave(sd->save_area_pa); in svm_prepare_switch_to_guest()
1588 if (sev_es_guest(vcpu->kvm)) in svm_prepare_switch_to_guest()
1592 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); in svm_prepare_switch_to_guest()
1595 * TSC_AUX is always virtualized for SEV-ES guests when the feature is in svm_prepare_switch_to_guest()
1596 * available. The user return MSR support is not required in this case in svm_prepare_switch_to_guest()
1597 * because TSC_AUX is restored on #VMEXIT from the host save area in svm_prepare_switch_to_guest()
1601 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm))) in svm_prepare_switch_to_guest()
1602 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull); in svm_prepare_switch_to_guest()
1605 !sd->bp_spec_reduce_set) { in svm_prepare_switch_to_guest()
1606 sd->bp_spec_reduce_set = true; in svm_prepare_switch_to_guest()
1609 svm->guest_state_loaded = true; in svm_prepare_switch_to_guest()
1614 to_svm(vcpu)->guest_state_loaded = false; in svm_prepare_host_switch()
1622 if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm)) in svm_vcpu_load()
1625 if (sd->current_vmcb != svm->vmcb) { in svm_vcpu_load()
1626 sd->current_vmcb = svm->vmcb; in svm_vcpu_load()
1643 ++vcpu->stat.host_state_reload; in svm_vcpu_put()
1649 unsigned long rflags = svm->vmcb->save.rflags; in svm_get_rflags()
1651 if (svm->nmi_singlestep) { in svm_get_rflags()
1653 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) in svm_get_rflags()
1655 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) in svm_get_rflags()
1663 if (to_svm(vcpu)->nmi_singlestep) in svm_set_rflags()
1667 * Any change of EFLAGS.VM is accompanied by a reload of SS in svm_set_rflags()
1668 * (caused by either a task switch or an inter-privilege IRET), in svm_set_rflags()
1671 to_svm(vcpu)->vmcb->save.rflags = rflags; in svm_set_rflags()
1676 struct vmcb *vmcb = to_svm(vcpu)->vmcb; in svm_get_if_flag()
1678 return sev_es_guest(vcpu->kvm) in svm_get_if_flag()
1679 ? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK in svm_get_if_flag()
1690 * When !npt_enabled, mmu->pdptrs[] is already available since in svm_cache_reg()
1691 * it is always updated per SDM when moving to CRs. in svm_cache_reg()
1697 KVM_BUG_ON(1, vcpu->kvm); in svm_cache_reg()
1706 * The following fields are ignored when AVIC is enabled in svm_set_vintr()
1708 WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu)); in svm_set_vintr()
1714 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF in svm_set_vintr()
1715 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN. in svm_set_vintr()
1716 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as in svm_set_vintr()
1717 * interrupts will never be unblocked while L2 is running. in svm_set_vintr()
1723 * This is just a dummy VINTR to actually cause a vmexit to happen. in svm_set_vintr()
1726 control = &svm->vmcb->control; in svm_set_vintr()
1727 control->int_vector = 0x0; in svm_set_vintr()
1728 control->int_ctl &= ~V_INTR_PRIO_MASK; in svm_set_vintr()
1729 control->int_ctl |= V_IRQ_MASK | in svm_set_vintr()
1730 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); in svm_set_vintr()
1731 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); in svm_set_vintr()
1739 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; in svm_clear_vintr()
1740 if (is_guest_mode(&svm->vcpu)) { in svm_clear_vintr()
1741 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; in svm_clear_vintr()
1743 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != in svm_clear_vintr()
1744 (svm->nested.ctl.int_ctl & V_TPR_MASK)); in svm_clear_vintr()
1746 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & in svm_clear_vintr()
1749 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; in svm_clear_vintr()
1752 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); in svm_clear_vintr()
1757 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; in svm_seg()
1758 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; in svm_seg()
1761 case VCPU_SREG_CS: return &save->cs; in svm_seg()
1762 case VCPU_SREG_DS: return &save->ds; in svm_seg()
1763 case VCPU_SREG_ES: return &save->es; in svm_seg()
1764 case VCPU_SREG_FS: return &save01->fs; in svm_seg()
1765 case VCPU_SREG_GS: return &save01->gs; in svm_seg()
1766 case VCPU_SREG_SS: return &save->ss; in svm_seg()
1767 case VCPU_SREG_TR: return &save01->tr; in svm_seg()
1768 case VCPU_SREG_LDTR: return &save01->ldtr; in svm_seg()
1778 return s->base; in svm_get_segment_base()
1786 var->base = s->base; in svm_get_segment()
1787 var->limit = s->limit; in svm_get_segment()
1788 var->selector = s->selector; in svm_get_segment()
1789 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; in svm_get_segment()
1790 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; in svm_get_segment()
1791 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; in svm_get_segment()
1792 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; in svm_get_segment()
1793 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; in svm_get_segment()
1794 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; in svm_get_segment()
1795 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; in svm_get_segment()
1798 * AMD CPUs circa 2014 track the G bit for all segments except CS. in svm_get_segment()
1799 * However, the SVM spec states that the G bit is not observed by the in svm_get_segment()
1802 * running KVM nested. It also helps cross-vendor migration, because in svm_get_segment()
1805 var->g = s->limit > 0xfffff; in svm_get_segment()
1811 var->unusable = !var->present; in svm_get_segment()
1819 var->type |= 0x2; in svm_get_segment()
1830 * cross-vendor migration. in svm_get_segment()
1832 if (!var->unusable) in svm_get_segment()
1833 var->type |= 0x1; in svm_get_segment()
1838 * descriptor is left as 1, although the whole segment has in svm_get_segment()
1842 if (var->unusable) in svm_get_segment()
1843 var->db = 0; in svm_get_segment()
1844 /* This is symmetric with svm_set_segment() */ in svm_get_segment()
1845 var->dpl = to_svm(vcpu)->vmcb->save.cpl; in svm_get_segment()
1852 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; in svm_get_cpl()
1854 return save->cpl; in svm_get_cpl()
1859 struct kvm_segment cs; in svm_get_cs_db_l_bits() local
1861 svm_get_segment(vcpu, &cs, VCPU_SREG_CS); in svm_get_cs_db_l_bits()
1862 *db = cs.db; in svm_get_cs_db_l_bits()
1863 *l = cs.l; in svm_get_cs_db_l_bits()
1870 dt->size = svm->vmcb->save.idtr.limit; in svm_get_idt()
1871 dt->address = svm->vmcb->save.idtr.base; in svm_get_idt()
1878 svm->vmcb->save.idtr.limit = dt->size; in svm_set_idt()
1879 svm->vmcb->save.idtr.base = dt->address ; in svm_set_idt()
1880 vmcb_mark_dirty(svm->vmcb, VMCB_DT); in svm_set_idt()
1887 dt->size = svm->vmcb->save.gdtr.limit; in svm_get_gdt()
1888 dt->address = svm->vmcb->save.gdtr.base; in svm_get_gdt()
1895 svm->vmcb->save.gdtr.limit = dt->size; in svm_set_gdt()
1896 svm->vmcb->save.gdtr.base = dt->address ; in svm_set_gdt()
1897 vmcb_mark_dirty(svm->vmcb, VMCB_DT); in svm_set_gdt()
1905 * For guests that don't set guest_state_protected, the cr3 update is in sev_post_set_cr3()
1907 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to in sev_post_set_cr3()
1912 if (sev_es_guest(vcpu->kvm)) { in sev_post_set_cr3()
1913 svm->vmcb->save.cr3 = cr3; in sev_post_set_cr3()
1914 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in sev_post_set_cr3()
1930 if (vcpu->arch.efer & EFER_LME) { in svm_set_cr0()
1932 vcpu->arch.efer |= EFER_LMA; in svm_set_cr0()
1933 if (!vcpu->arch.guest_state_protected) in svm_set_cr0()
1934 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; in svm_set_cr0()
1938 vcpu->arch.efer &= ~EFER_LMA; in svm_set_cr0()
1939 if (!vcpu->arch.guest_state_protected) in svm_set_cr0()
1940 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); in svm_set_cr0()
1944 vcpu->arch.cr0 = cr0; in svm_set_cr0()
1953 * re-enable caching here because the QEMU bios in svm_set_cr0()
1954 * does not do it - this results in some delay at in svm_set_cr0()
1957 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) in svm_set_cr0()
1960 svm->vmcb->save.cr0 = hcr0; in svm_set_cr0()
1961 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in svm_set_cr0()
1964 * SEV-ES guests must always keep the CR intercepts cleared. CR in svm_set_cr0()
1965 * tracking is done using the CR write traps. in svm_set_cr0()
1967 if (sev_es_guest(vcpu->kvm)) in svm_set_cr0()
1988 unsigned long old_cr4 = vcpu->arch.cr4; in svm_set_cr4()
1990 vcpu->arch.cr4 = cr4; in svm_set_cr4()
1998 to_svm(vcpu)->vmcb->save.cr4 = cr4; in svm_set_cr4()
1999 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); in svm_set_cr4()
2002 vcpu->arch.cpuid_dynamic_bits_dirty = true; in svm_set_cr4()
2011 s->base = var->base; in svm_set_segment()
2012 s->limit = var->limit; in svm_set_segment()
2013 s->selector = var->selector; in svm_set_segment()
2014 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); in svm_set_segment()
2015 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; in svm_set_segment()
2016 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; in svm_set_segment()
2017 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; in svm_set_segment()
2018 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; in svm_set_segment()
2019 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; in svm_set_segment()
2020 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; in svm_set_segment()
2021 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; in svm_set_segment()
2024 * This is always accurate, except if SYSRET returned to a segment in svm_set_segment()
2030 /* This is symmetric with svm_get_segment() */ in svm_set_segment()
2031 svm->vmcb->save.cpl = (var->dpl & 3); in svm_set_segment()
2033 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); in svm_set_segment()
2042 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { in svm_update_exception_bitmap()
2043 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) in svm_update_exception_bitmap()
2050 if (sd->next_asid > sd->max_asid) { in new_asid()
2051 ++sd->asid_generation; in new_asid()
2052 sd->next_asid = sd->min_asid; in new_asid()
2053 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; in new_asid()
2054 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); in new_asid()
2057 svm->current_vmcb->asid_generation = sd->asid_generation; in new_asid()
2058 svm->asid = sd->next_asid++; in new_asid()
2063 struct vmcb *vmcb = to_svm(vcpu)->vmcb; in svm_set_dr6()
2065 if (vcpu->arch.guest_state_protected) in svm_set_dr6()
2068 if (unlikely(value != vmcb->save.dr6)) { in svm_set_dr6()
2069 vmcb->save.dr6 = value; in svm_set_dr6()
2078 if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm))) in svm_sync_dirty_debug_regs()
2081 get_debugreg(vcpu->arch.db[0], 0); in svm_sync_dirty_debug_regs()
2082 get_debugreg(vcpu->arch.db[1], 1); in svm_sync_dirty_debug_regs()
2083 get_debugreg(vcpu->arch.db[2], 2); in svm_sync_dirty_debug_regs()
2084 get_debugreg(vcpu->arch.db[3], 3); in svm_sync_dirty_debug_regs()
2086 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, in svm_sync_dirty_debug_regs()
2089 vcpu->arch.dr6 = svm->vmcb->save.dr6; in svm_sync_dirty_debug_regs()
2090 vcpu->arch.dr7 = svm->vmcb->save.dr7; in svm_sync_dirty_debug_regs()
2091 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; in svm_sync_dirty_debug_regs()
2099 if (vcpu->arch.guest_state_protected) in svm_set_dr7()
2102 svm->vmcb->save.dr7 = value; in svm_set_dr7()
2103 vmcb_mark_dirty(svm->vmcb, VMCB_DR); in svm_set_dr7()
2110 u64 fault_address = svm->vmcb->control.exit_info_2; in pf_interception()
2111 u64 error_code = svm->vmcb->control.exit_info_1; in pf_interception()
2115 svm->vmcb->control.insn_bytes : NULL, in pf_interception()
2116 svm->vmcb->control.insn_len); in pf_interception()
2124 u64 fault_address = svm->vmcb->control.exit_info_2; in npf_interception()
2125 u64 error_code = svm->vmcb->control.exit_info_1; in npf_interception()
2129 * with KVM-defined sythentic flags. Clear the flags and continue on, in npf_interception()
2136 if (sev_snp_guest(vcpu->kvm) && (error_code & PFERR_GUEST_ENC_MASK)) in npf_interception()
2142 svm->vmcb->control.insn_bytes : NULL, in npf_interception()
2143 svm->vmcb->control.insn_len); in npf_interception()
2153 struct kvm_run *kvm_run = vcpu->run; in db_interception()
2156 if (!(vcpu->guest_debug & in db_interception()
2158 !svm->nmi_singlestep) { in db_interception()
2159 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; in db_interception()
2164 if (svm->nmi_singlestep) { in db_interception()
2170 if (vcpu->guest_debug & in db_interception()
2172 kvm_run->exit_reason = KVM_EXIT_DEBUG; in db_interception()
2173 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; in db_interception()
2174 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; in db_interception()
2175 kvm_run->debug.arch.pc = in db_interception()
2176 svm->vmcb->save.cs.base + svm->vmcb->save.rip; in db_interception()
2177 kvm_run->debug.arch.exception = DB_VECTOR; in db_interception()
2187 struct kvm_run *kvm_run = vcpu->run; in bp_interception()
2189 kvm_run->exit_reason = KVM_EXIT_DEBUG; in bp_interception()
2190 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; in bp_interception()
2191 kvm_run->debug.arch.exception = BP_VECTOR; in bp_interception()
2239 /* Flush tlb to evict multi-match entries */ in is_erratum_383()
2249 * Erratum 383 triggered. Guest state is corrupt so kill the in svm_handle_mce()
2260 * On an #MC intercept the MCE handler is not called automatically in in svm_handle_mce()
2273 struct kvm_run *kvm_run = vcpu->run; in shutdown_interception()
2278 * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put in shutdown_interception()
2281 * userspace. At a platform view, INIT is acceptable behavior as in shutdown_interception()
2285 * The VM save area for SEV-ES guests has already been encrypted so it in shutdown_interception()
2286 * cannot be reinitialized, i.e. synthesizing INIT is futile. in shutdown_interception()
2288 if (!sev_es_guest(vcpu->kvm)) { in shutdown_interception()
2289 clear_page(svm->vmcb); in shutdown_interception()
2297 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; in shutdown_interception()
2304 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ in io_interception()
2308 ++vcpu->stat.io_exits; in io_interception()
2315 if (sev_es_guest(vcpu->kvm)) in io_interception()
2321 svm->next_rip = svm->vmcb->control.exit_info_2; in io_interception()
2338 ++vcpu->stat.irq_exits; in intr_interception()
2352 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); in vmload_vmsave_interception()
2354 if (ret == -EINVAL) in vmload_vmsave_interception()
2364 svm_copy_vmloadsave_state(svm->vmcb, vmcb12); in vmload_vmsave_interception()
2365 svm->sysenter_eip_hi = 0; in vmload_vmsave_interception()
2366 svm->sysenter_esp_hi = 0; in vmload_vmsave_interception()
2368 svm_copy_vmloadsave_state(vmcb12, svm->vmcb); in vmload_vmsave_interception()
2404 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in svm_instr_opcode()
2406 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) in svm_instr_opcode()
2409 switch (ctxt->modrm) { in svm_instr_opcode()
2439 /* Returns '1' or -errno on failure, '0' on success. */ in emulate_svm_instr()
2451 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2459 u32 error_code = svm->vmcb->control.exit_info_1; in gp_interception()
2485 if (svm->vmcb->save.rax & ~PAGE_MASK) in gp_interception()
2500 * If VGIF is enabled, the STGI intercept is only added to in svm_set_gif()
2511 if (svm->vcpu.arch.smi_pending || in svm_set_gif()
2512 svm->vcpu.arch.nmi_pending || in svm_set_gif()
2513 kvm_cpu_has_injectable_intr(&svm->vcpu) || in svm_set_gif()
2514 kvm_apic_has_pending_init_or_sipi(&svm->vcpu)) in svm_set_gif()
2515 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); in svm_set_gif()
2520 * After a CLGI no interrupts should come. But if vGIF is in svm_set_gif()
2562 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); in invlpga_interception()
2572 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); in skinit_interception()
2583 int int_type = svm->vmcb->control.exit_int_info & in task_switch_interception()
2585 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; in task_switch_interception()
2587 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; in task_switch_interception()
2589 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; in task_switch_interception()
2593 tss_selector = (u16)svm->vmcb->control.exit_info_1; in task_switch_interception()
2595 if (svm->vmcb->control.exit_info_2 & in task_switch_interception()
2598 else if (svm->vmcb->control.exit_info_2 & in task_switch_interception()
2609 vcpu->arch.nmi_injected = false; in task_switch_interception()
2612 if (svm->vmcb->control.exit_info_2 & in task_switch_interception()
2616 (u32)svm->vmcb->control.exit_info_2; in task_switch_interception()
2638 int_vec = -1; in task_switch_interception()
2646 if (!sev_es_guest(svm->vcpu.kvm)) in svm_clr_iret_intercept()
2652 if (!sev_es_guest(svm->vcpu.kvm)) in svm_set_iret_intercept()
2660 WARN_ON_ONCE(sev_es_guest(vcpu->kvm)); in iret_interception()
2662 ++vcpu->stat.nmi_window_exits; in iret_interception()
2663 svm->awaiting_iret_completion = true; in iret_interception()
2666 svm->nmi_iret_rip = kvm_rip_read(vcpu); in iret_interception()
2677 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); in invlpg_interception()
2695 unsigned long cr0 = vcpu->arch.cr0; in check_selective_cr0_intercepted()
2699 (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) in check_selective_cr0_intercepted()
2706 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; in check_selective_cr0_intercepted()
2725 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) in cr_interception()
2728 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; in cr_interception()
2729 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) in cr_interception()
2730 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; in cr_interception()
2732 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; in cr_interception()
2736 cr -= 16; in cr_interception()
2767 val = vcpu->arch.cr2; in cr_interception()
2796 new_value = (unsigned long)svm->vmcb->control.exit_info_1; in cr_trap()
2798 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; in cr_trap()
2831 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT in dr_interception()
2834 if (sev_es_guest(vcpu->kvm)) in dr_interception()
2837 if (vcpu->guest_debug == 0) { in dr_interception()
2844 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; in dr_interception()
2851 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; in dr_interception()
2852 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; in dr_interception()
2854 dr -= 16; in dr_interception()
2874 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; in cr8_write_interception()
2886 * whether the guest has X86_FEATURE_SVM - this avoids a failure if in efer_trap()
2891 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; in efer_trap()
2916 return sev_es_guest(vcpu->kvm) && in sev_es_prevent_msr_access()
2917 vcpu->arch.guest_state_protected && in sev_es_prevent_msr_access()
2918 svm_msrpm_offset(msr_info->index) != MSR_INVALID && in sev_es_prevent_msr_access()
2919 !msr_write_intercepted(vcpu, msr_info->index); in sev_es_prevent_msr_access()
2927 msr_info->data = 0; in svm_get_msr()
2928 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in svm_get_msr()
2931 switch (msr_info->index) { in svm_get_msr()
2933 if (!msr_info->host_initiated && in svm_get_msr()
2936 msr_info->data = svm->tsc_ratio_msr; in svm_get_msr()
2939 msr_info->data = svm->vmcb01.ptr->save.star; in svm_get_msr()
2943 msr_info->data = svm->vmcb01.ptr->save.lstar; in svm_get_msr()
2946 msr_info->data = svm->vmcb01.ptr->save.cstar; in svm_get_msr()
2949 msr_info->data = svm->vmcb01.ptr->save.gs.base; in svm_get_msr()
2952 msr_info->data = svm->vmcb01.ptr->save.fs.base; in svm_get_msr()
2955 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; in svm_get_msr()
2958 msr_info->data = svm->vmcb01.ptr->save.sfmask; in svm_get_msr()
2962 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; in svm_get_msr()
2965 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; in svm_get_msr()
2967 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; in svm_get_msr()
2970 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; in svm_get_msr()
2972 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; in svm_get_msr()
2975 msr_info->data = svm->tsc_aux; in svm_get_msr()
2978 msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl; in svm_get_msr()
2981 msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from; in svm_get_msr()
2984 msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to; in svm_get_msr()
2987 msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from; in svm_get_msr()
2990 msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to; in svm_get_msr()
2993 msr_info->data = svm->nested.hsave_msr; in svm_get_msr()
2996 msr_info->data = svm->nested.vm_cr_msr; in svm_get_msr()
2999 if (!msr_info->host_initiated && in svm_get_msr()
3004 msr_info->data = svm->vmcb->save.spec_ctrl; in svm_get_msr()
3006 msr_info->data = svm->spec_ctrl; in svm_get_msr()
3009 if (!msr_info->host_initiated && in svm_get_msr()
3013 msr_info->data = svm->virt_spec_ctrl; in svm_get_msr()
3025 msr_info->data = 0; in svm_get_msr()
3029 msr_info->data = 0x1E; in svm_get_msr()
3033 msr_info->data = svm->msr_decfg; in svm_get_msr()
3044 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb)) in svm_complete_emulated_msr()
3061 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) in svm_set_vm_cr()
3064 svm->nested.vm_cr_msr &= ~chg_mask; in svm_set_vm_cr()
3065 svm->nested.vm_cr_msr |= (data & chg_mask); in svm_set_vm_cr()
3067 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; in svm_set_vm_cr()
3069 /* check for svm_disable while efer.svme is set */ in svm_set_vm_cr()
3070 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) in svm_set_vm_cr()
3081 u32 ecx = msr->index; in svm_set_msr()
3082 u64 data = msr->data; in svm_set_msr()
3085 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in svm_set_msr()
3092 if (!msr->host_initiated) in svm_set_msr()
3095 * In case TSC scaling is not enabled, always in svm_set_msr()
3099 * this msr to 0 if tsc scaling is not enabled. in svm_set_msr()
3102 if (data != 0 && data != svm->tsc_ratio_msr) in svm_set_msr()
3110 svm->tsc_ratio_msr = data; in svm_set_msr()
3122 svm->vmcb01.ptr->save.g_pat = data; in svm_set_msr()
3125 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); in svm_set_msr()
3128 if (!msr->host_initiated && in svm_set_msr()
3136 svm->vmcb->save.spec_ctrl = data; in svm_set_msr()
3138 svm->spec_ctrl = data; in svm_set_msr()
3143 * For non-nested: in svm_set_msr()
3144 * When it's written (to non-zero) for the first time, pass in svm_set_msr()
3148 * The handling of the MSR bitmap for L2 guests is done in in svm_set_msr()
3153 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); in svm_set_msr()
3156 if (!msr->host_initiated && in svm_set_msr()
3163 svm->virt_spec_ctrl = data; in svm_set_msr()
3166 svm->vmcb01.ptr->save.star = data; in svm_set_msr()
3170 svm->vmcb01.ptr->save.lstar = data; in svm_set_msr()
3173 svm->vmcb01.ptr->save.cstar = data; in svm_set_msr()
3176 svm->vmcb01.ptr->save.gs.base = data; in svm_set_msr()
3179 svm->vmcb01.ptr->save.fs.base = data; in svm_set_msr()
3182 svm->vmcb01.ptr->save.kernel_gs_base = data; in svm_set_msr()
3185 svm->vmcb01.ptr->save.sfmask = data; in svm_set_msr()
3189 svm->vmcb01.ptr->save.sysenter_cs = data; in svm_set_msr()
3192 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; in svm_set_msr()
3200 svm->sysenter_eip_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0; in svm_set_msr()
3203 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; in svm_set_msr()
3204 svm->sysenter_esp_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0; in svm_set_msr()
3208 * TSC_AUX is always virtualized for SEV-ES guests when the in svm_set_msr()
3209 * feature is available. The user return MSR support is not in svm_set_msr()
3210 * required in this case because TSC_AUX is restored on #VMEXIT in svm_set_msr()
3214 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm)) in svm_set_msr()
3218 * TSC_AUX is usually changed only during boot and never read in svm_set_msr()
3223 ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull); in svm_set_msr()
3228 svm->tsc_aux = data; in svm_set_msr()
3240 * and bits 5:3 are reserved-to-zero. Sadly, old KVM allowed in svm_set_msr()
3242 * Performance-Monitoring/Breakpoint external pins. Drop bits in svm_set_msr()
3259 svm_get_lbr_vmcb(svm)->save.dbgctl = data; in svm_set_msr()
3269 if (!msr->host_initiated && !page_address_valid(vcpu, data)) in svm_set_msr()
3272 svm->nested.hsave_msr = data & PAGE_MASK; in svm_set_msr()
3288 svm->msr_decfg = data; in svm_set_msr()
3299 if (to_svm(vcpu)->vmcb->control.exit_info_1) in msr_interception()
3311 * If not running nested, for AVIC, the only reason to end up here is ExtINTs. in interrupt_window_interception()
3313 * requesting the IRQ window and we have to re-enable it. in interrupt_window_interception()
3320 * AVIC still inhibited due to per-cpu AVIC inhibition. in interrupt_window_interception()
3322 kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); in interrupt_window_interception()
3324 ++vcpu->stat.irq_window_exits; in interrupt_window_interception()
3332 * CPL is not made available for an SEV-ES guest, therefore in pause_interception()
3333 * vcpu->arch.preempted_in_kernel can never be true. Just in pause_interception()
3336 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; in pause_interception()
3360 type = svm->vmcb->control.exit_info_2; in invpcid_interception()
3361 gva = svm->vmcb->control.exit_info_1; in invpcid_interception()
3364 * FIXME: Perform segment checks for 32-bit mode, and inject #SS if the in invpcid_interception()
3365 * stack segment is used. The intercept takes priority over all in invpcid_interception()
3367 * address? The APM is sorely lacking. in invpcid_interception()
3457 struct vmcb_control_area *control = &svm->vmcb->control; in dump_vmcb()
3458 struct vmcb_save_area *save = &svm->vmcb->save; in dump_vmcb()
3459 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; in dump_vmcb()
3467 svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); in dump_vmcb()
3469 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); in dump_vmcb()
3470 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); in dump_vmcb()
3471 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); in dump_vmcb()
3472 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); in dump_vmcb()
3473 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); in dump_vmcb()
3474 pr_err("%-20s%08x %08x\n", "intercepts:", in dump_vmcb()
3475 control->intercepts[INTERCEPT_WORD3], in dump_vmcb()
3476 control->intercepts[INTERCEPT_WORD4]); in dump_vmcb()
3477 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); in dump_vmcb()
3478 pr_err("%-20s%d\n", "pause filter threshold:", in dump_vmcb()
3479 control->pause_filter_thresh); in dump_vmcb()
3480 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); in dump_vmcb()
3481 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); in dump_vmcb()
3482 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); in dump_vmcb()
3483 pr_err("%-20s%d\n", "asid:", control->asid); in dump_vmcb()
3484 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); in dump_vmcb()
3485 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); in dump_vmcb()
3486 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); in dump_vmcb()
3487 pr_err("%-20s%08x\n", "int_state:", control->int_state); in dump_vmcb()
3488 pr_err("%-20s%08x\n", "exit_code:", control->exit_code); in dump_vmcb()
3489 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); in dump_vmcb()
3490 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); in dump_vmcb()
3491 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); in dump_vmcb()
3492 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); in dump_vmcb()
3493 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); in dump_vmcb()
3494 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); in dump_vmcb()
3495 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); in dump_vmcb()
3496 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); in dump_vmcb()
3497 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); in dump_vmcb()
3498 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); in dump_vmcb()
3499 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); in dump_vmcb()
3500 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); in dump_vmcb()
3501 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); in dump_vmcb()
3502 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); in dump_vmcb()
3503 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); in dump_vmcb()
3504 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); in dump_vmcb()
3506 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3508 save->es.selector, save->es.attrib, in dump_vmcb()
3509 save->es.limit, save->es.base); in dump_vmcb()
3510 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3511 "cs:", in dump_vmcb()
3512 save->cs.selector, save->cs.attrib, in dump_vmcb()
3513 save->cs.limit, save->cs.base); in dump_vmcb()
3514 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3516 save->ss.selector, save->ss.attrib, in dump_vmcb()
3517 save->ss.limit, save->ss.base); in dump_vmcb()
3518 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3520 save->ds.selector, save->ds.attrib, in dump_vmcb()
3521 save->ds.limit, save->ds.base); in dump_vmcb()
3522 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3524 save01->fs.selector, save01->fs.attrib, in dump_vmcb()
3525 save01->fs.limit, save01->fs.base); in dump_vmcb()
3526 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3528 save01->gs.selector, save01->gs.attrib, in dump_vmcb()
3529 save01->gs.limit, save01->gs.base); in dump_vmcb()
3530 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3532 save->gdtr.selector, save->gdtr.attrib, in dump_vmcb()
3533 save->gdtr.limit, save->gdtr.base); in dump_vmcb()
3534 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3536 save01->ldtr.selector, save01->ldtr.attrib, in dump_vmcb()
3537 save01->ldtr.limit, save01->ldtr.base); in dump_vmcb()
3538 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3540 save->idtr.selector, save->idtr.attrib, in dump_vmcb()
3541 save->idtr.limit, save->idtr.base); in dump_vmcb()
3542 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3544 save01->tr.selector, save01->tr.attrib, in dump_vmcb()
3545 save01->tr.limit, save01->tr.base); in dump_vmcb()
3547 save->vmpl, save->cpl, save->efer); in dump_vmcb()
3548 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3549 "cr0:", save->cr0, "cr2:", save->cr2); in dump_vmcb()
3550 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3551 "cr3:", save->cr3, "cr4:", save->cr4); in dump_vmcb()
3552 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3553 "dr6:", save->dr6, "dr7:", save->dr7); in dump_vmcb()
3554 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3555 "rip:", save->rip, "rflags:", save->rflags); in dump_vmcb()
3556 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3557 "rsp:", save->rsp, "rax:", save->rax); in dump_vmcb()
3558 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3559 "star:", save01->star, "lstar:", save01->lstar); in dump_vmcb()
3560 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3561 "cstar:", save01->cstar, "sfmask:", save01->sfmask); in dump_vmcb()
3562 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3563 "kernel_gs_base:", save01->kernel_gs_base, in dump_vmcb()
3564 "sysenter_cs:", save01->sysenter_cs); in dump_vmcb()
3565 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3566 "sysenter_esp:", save01->sysenter_esp, in dump_vmcb()
3567 "sysenter_eip:", save01->sysenter_eip); in dump_vmcb()
3568 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3569 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); in dump_vmcb()
3570 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3571 "br_from:", save->br_from, "br_to:", save->br_to); in dump_vmcb()
3572 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3573 "excp_from:", save->last_excp_from, in dump_vmcb()
3574 "excp_to:", save->last_excp_to); in dump_vmcb()
3587 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in svm_handle_invalid_exit()
3588 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; in svm_handle_invalid_exit()
3589 vcpu->run->internal.ndata = 2; in svm_handle_invalid_exit()
3590 vcpu->run->internal.data[0] = exit_code; in svm_handle_invalid_exit()
3591 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; in svm_handle_invalid_exit()
3619 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; in svm_get_exit_info()
3621 *reason = control->exit_code; in svm_get_exit_info()
3622 *info1 = control->exit_info_1; in svm_get_exit_info()
3623 *info2 = control->exit_info_2; in svm_get_exit_info()
3624 *intr_info = control->exit_int_info; in svm_get_exit_info()
3627 *error_code = control->exit_int_info_err; in svm_get_exit_info()
3635 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; in svm_get_entry_info()
3637 *intr_info = control->event_inj; in svm_get_entry_info()
3641 *error_code = control->event_inj_err; in svm_get_entry_info()
3650 struct kvm_run *kvm_run = vcpu->run; in svm_handle_exit()
3651 u32 exit_code = svm->vmcb->control.exit_code; in svm_handle_exit()
3653 /* SEV-ES guests must use the CR write traps to track CR registers. */ in svm_handle_exit()
3654 if (!sev_es_guest(vcpu->kvm)) { in svm_handle_exit()
3656 vcpu->arch.cr0 = svm->vmcb->save.cr0; in svm_handle_exit()
3658 vcpu->arch.cr3 = svm->vmcb->save.cr3; in svm_handle_exit()
3675 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { in svm_handle_exit()
3676 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; in svm_handle_exit()
3677 kvm_run->fail_entry.hardware_entry_failure_reason in svm_handle_exit()
3678 = svm->vmcb->control.exit_code; in svm_handle_exit()
3679 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; in svm_handle_exit()
3692 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); in pre_svm_run()
3700 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { in pre_svm_run()
3701 svm->current_vmcb->asid_generation = 0; in pre_svm_run()
3702 vmcb_mark_all_dirty(svm->vmcb); in pre_svm_run()
3703 svm->current_vmcb->cpu = vcpu->cpu; in pre_svm_run()
3706 if (sev_guest(vcpu->kvm)) in pre_svm_run()
3707 return pre_sev_run(svm, vcpu->cpu); in pre_svm_run()
3710 if (svm->current_vmcb->asid_generation != sd->asid_generation) in pre_svm_run()
3720 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; in svm_inject_nmi()
3722 if (svm->nmi_l1_to_l2) in svm_inject_nmi()
3726 * No need to manually track NMI masking when vNMI is enabled, hardware in svm_inject_nmi()
3731 svm->nmi_masked = true; in svm_inject_nmi()
3734 ++vcpu->stat.nmi_injections; in svm_inject_nmi()
3744 return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK); in svm_is_vnmi_pending()
3754 if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK) in svm_set_vnmi_pending()
3757 svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK; in svm_set_vnmi_pending()
3758 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); in svm_set_vnmi_pending()
3761 * Because the pending NMI is serviced by hardware, KVM can't know when in svm_set_vnmi_pending()
3762 * the NMI is "injected", but for all intents and purposes, passing the in svm_set_vnmi_pending()
3765 ++vcpu->stat.nmi_injections; in svm_set_vnmi_pending()
3775 if (vcpu->arch.interrupt.soft) { in svm_inject_irq()
3784 trace_kvm_inj_virq(vcpu->arch.interrupt.nr, in svm_inject_irq()
3785 vcpu->arch.interrupt.soft, reinjected); in svm_inject_irq()
3786 ++vcpu->stat.irq_injections; in svm_inject_irq()
3788 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | in svm_inject_irq()
3796 * apic->apicv_active must be read after vcpu->mode. in svm_complete_interrupt_delivery()
3799 bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE); in svm_complete_interrupt_delivery()
3801 /* Note, this is called iff the local APIC is in-kernel. */ in svm_complete_interrupt_delivery()
3802 if (!READ_ONCE(vcpu->arch.apic->apicv_active)) { in svm_complete_interrupt_delivery()
3809 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); in svm_complete_interrupt_delivery()
3832 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in in svm_deliver_interrupt()
3833 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before in svm_deliver_interrupt()
3839 svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector); in svm_deliver_interrupt()
3847 * SEV-ES guests must always keep the CR intercepts cleared. CR in svm_update_cr8_intercept()
3848 * tracking is done using the CR write traps. in svm_update_cr8_intercept()
3850 if (sev_es_guest(vcpu->kvm)) in svm_update_cr8_intercept()
3858 if (irr == -1) in svm_update_cr8_intercept()
3870 return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK; in svm_get_nmi_mask()
3872 return svm->nmi_masked; in svm_get_nmi_mask()
3881 svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK; in svm_set_nmi_mask()
3883 svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK; in svm_set_nmi_mask()
3886 svm->nmi_masked = masked; in svm_set_nmi_mask()
3897 struct vmcb *vmcb = svm->vmcb; in svm_nmi_blocked()
3908 return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK; in svm_nmi_blocked()
3914 if (svm->nested.nested_run_pending) in svm_nmi_allowed()
3915 return -EBUSY; in svm_nmi_allowed()
3920 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ in svm_nmi_allowed()
3922 return -EBUSY; in svm_nmi_allowed()
3929 struct vmcb *vmcb = svm->vmcb; in svm_interrupt_blocked()
3936 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) in svm_interrupt_blocked()
3937 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) in svm_interrupt_blocked()
3949 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); in svm_interrupt_blocked()
3956 if (svm->nested.nested_run_pending) in svm_interrupt_allowed()
3957 return -EBUSY; in svm_interrupt_allowed()
3963 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, in svm_interrupt_allowed()
3967 return -EBUSY; in svm_interrupt_allowed()
3980 * we'll get the vintr intercept. However, if the vGIF feature is in svm_enable_irq_window()
3986 * IRQ window is not needed when AVIC is enabled, in svm_enable_irq_window()
3991 * If running nested, AVIC is already locally inhibited in svm_enable_irq_window()
3992 * on this vCPU, therefore there is no need to request in svm_enable_irq_window()
3996 kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); in svm_enable_irq_window()
4007 * If NMIs are outright masked, i.e. the vCPU is already handling an in svm_enable_nmi_window()
4008 * NMI, and KVM has not yet intercepted an IRET, then there is nothing in svm_enable_nmi_window()
4010 * If KVM has already intercepted IRET, then single-step over the IRET, in svm_enable_nmi_window()
4013 * If vNMI is enabled, KVM should never request an NMI window if NMIs in svm_enable_nmi_window()
4014 * are masked, as KVM allows at most one to-be-injected NMI and one in svm_enable_nmi_window()
4018 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately in svm_enable_nmi_window()
4019 * inject the NMI. In those situations, KVM needs to single-step over in svm_enable_nmi_window()
4025 if (!svm->awaiting_iret_completion) in svm_enable_nmi_window()
4030 * SEV-ES guests are responsible for signaling when a vCPU is ready to in svm_enable_nmi_window()
4031 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e. in svm_enable_nmi_window()
4032 * KVM can't intercept and single-step IRET to detect when NMIs are in svm_enable_nmi_window()
4035 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware in svm_enable_nmi_window()
4036 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not in svm_enable_nmi_window()
4039 if (sev_es_guest(vcpu->kvm)) in svm_enable_nmi_window()
4052 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); in svm_enable_nmi_window()
4053 svm->nmi_singlestep = true; in svm_enable_nmi_window()
4054 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); in svm_enable_nmi_window()
4064 * entries, and thus is a superset of Hyper-V's fine grained flushing. in svm_flush_tlb_asid()
4072 * unconditionally does a TLB flush on both nested VM-Enter and nested in svm_flush_tlb_asid()
4073 * VM-Exit (via kvm_mmu_reset_context()). in svm_flush_tlb_asid()
4076 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; in svm_flush_tlb_asid()
4078 svm->current_vmcb->asid_generation--; in svm_flush_tlb_asid()
4083 hpa_t root_tdp = vcpu->arch.mmu->root.hpa; in svm_flush_tlb_current()
4086 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly in svm_flush_tlb_current()
4100 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB in svm_flush_tlb_all()
4107 hv_flush_remote_tlbs(vcpu->kvm); in svm_flush_tlb_all()
4116 invlpga(gva, svm->vmcb->control.asid); in svm_flush_tlb_gva()
4127 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; in sync_cr8_to_lapic()
4142 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; in sync_lapic_to_cr8()
4143 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; in sync_lapic_to_cr8()
4154 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's in svm_complete_soft_interrupt()
4155 * associated with the original soft exception/interrupt. next_rip is in svm_complete_soft_interrupt()
4157 * needs to manually set next_rip for re-injection. Unlike the !nrips in svm_complete_soft_interrupt()
4158 * case below, this needs to be done if and only if KVM is re-injecting in svm_complete_soft_interrupt()
4159 * the same event, i.e. if the event is a soft exception/interrupt, in svm_complete_soft_interrupt()
4160 * otherwise next_rip is unused on VMRUN. in svm_complete_soft_interrupt()
4163 kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase)) in svm_complete_soft_interrupt()
4164 svm->vmcb->control.next_rip = svm->soft_int_next_rip; in svm_complete_soft_interrupt()
4174 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase)) in svm_complete_soft_interrupt()
4175 kvm_rip_write(vcpu, svm->soft_int_old_rip); in svm_complete_soft_interrupt()
4183 u32 exitintinfo = svm->vmcb->control.exit_int_info; in svm_complete_interrupts()
4184 bool nmi_l1_to_l2 = svm->nmi_l1_to_l2; in svm_complete_interrupts()
4185 bool soft_int_injected = svm->soft_int_injected; in svm_complete_interrupts()
4187 svm->nmi_l1_to_l2 = false; in svm_complete_interrupts()
4188 svm->soft_int_injected = false; in svm_complete_interrupts()
4194 if (svm->awaiting_iret_completion && in svm_complete_interrupts()
4195 kvm_rip_read(vcpu) != svm->nmi_iret_rip) { in svm_complete_interrupts()
4196 svm->awaiting_iret_completion = false; in svm_complete_interrupts()
4197 svm->nmi_masked = false; in svm_complete_interrupts()
4201 vcpu->arch.nmi_injected = false; in svm_complete_interrupts()
4218 vcpu->arch.nmi_injected = true; in svm_complete_interrupts()
4219 svm->nmi_l1_to_l2 = nmi_l1_to_l2; in svm_complete_interrupts()
4225 * Never re-inject a #VC exception. in svm_complete_interrupts()
4231 error_code = svm->vmcb->control.exit_int_info_err; in svm_complete_interrupts()
4253 struct vmcb_control_area *control = &svm->vmcb->control; in svm_cancel_injection()
4255 control->exit_int_info = control->event_inj; in svm_cancel_injection()
4256 control->exit_int_info_err = control->event_inj_err; in svm_cancel_injection()
4257 control->event_inj = 0; in svm_cancel_injection()
4263 if (to_kvm_sev_info(vcpu->kvm)->need_init) in svm_vcpu_pre_run()
4264 return -EINVAL; in svm_vcpu_pre_run()
4276 switch (svm->vmcb->control.exit_code) { in svm_exit_handlers_fastpath()
4278 if (!svm->vmcb->control.exit_info_1) in svm_exit_handlers_fastpath()
4292 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); in svm_vcpu_enter_exit()
4311 if (sev_es_guest(vcpu->kvm)) in svm_vcpu_enter_exit()
4330 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; in svm_vcpu_run()
4331 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; in svm_vcpu_run()
4332 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; in svm_vcpu_run()
4340 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { in svm_vcpu_run()
4344 * is enough to force an immediate vmexit. in svm_vcpu_run()
4351 smp_send_reschedule(vcpu->cpu); in svm_vcpu_run()
4354 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; in svm_vcpu_run()
4355 vcpu->run->fail_entry.hardware_entry_failure_reason = SVM_EXIT_ERR; in svm_vcpu_run()
4356 vcpu->run->fail_entry.cpu = vcpu->cpu; in svm_vcpu_run()
4362 if (unlikely(svm->asid != svm->vmcb->control.asid)) { in svm_vcpu_run()
4363 svm->vmcb->control.asid = svm->asid; in svm_vcpu_run()
4364 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); in svm_vcpu_run()
4366 svm->vmcb->save.cr2 = vcpu->arch.cr2; in svm_vcpu_run()
4368 svm_hv_update_vp_id(svm->vmcb, vcpu); in svm_vcpu_run()
4371 * Run with all-zero DR6 unless needed, so that we can get the exact cause in svm_vcpu_run()
4374 if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))) in svm_vcpu_run()
4381 * Hardware only context switches DEBUGCTL if LBR virtualization is in svm_vcpu_run()
4383 * VM-Exit), as running with the host's DEBUGCTL can negatively affect in svm_vcpu_run()
4386 if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) && in svm_vcpu_run()
4387 vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl) in svm_vcpu_run()
4388 update_debugctlmsr(svm->vmcb->save.dbgctl); in svm_vcpu_run()
4394 * it's non-zero. Since vmentry is serialising on affected CPUs, there in svm_vcpu_run()
4395 * is no need to worry about the conditional branch over the wrmsr in svm_vcpu_run()
4399 x86_spec_ctrl_set_guest(svm->virt_spec_ctrl); in svm_vcpu_run()
4404 x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); in svm_vcpu_run()
4406 if (!sev_es_guest(vcpu->kvm)) { in svm_vcpu_run()
4407 vcpu->arch.cr2 = svm->vmcb->save.cr2; in svm_vcpu_run()
4408 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; in svm_vcpu_run()
4409 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; in svm_vcpu_run()
4410 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; in svm_vcpu_run()
4412 vcpu->arch.regs_dirty = 0; in svm_vcpu_run()
4414 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) in svm_vcpu_run()
4417 if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) && in svm_vcpu_run()
4418 vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl) in svm_vcpu_run()
4419 update_debugctlmsr(vcpu->arch.host_debugctl); in svm_vcpu_run()
4426 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) in svm_vcpu_run()
4431 svm->next_rip = 0; in svm_vcpu_run()
4436 if (svm->nested.nested_run_pending && in svm_vcpu_run()
4437 svm->vmcb->control.exit_code != SVM_EXIT_ERR) in svm_vcpu_run()
4438 ++vcpu->stat.nested_run; in svm_vcpu_run()
4440 svm->nested.nested_run_pending = 0; in svm_vcpu_run()
4443 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; in svm_vcpu_run()
4444 vmcb_mark_all_clean(svm->vmcb); in svm_vcpu_run()
4447 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) in svm_vcpu_run()
4448 vcpu->arch.apf.host_apf_flags = in svm_vcpu_run()
4451 vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET; in svm_vcpu_run()
4457 if (unlikely(svm->vmcb->control.exit_code == in svm_vcpu_run()
4475 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); in svm_load_mmu_pgd()
4476 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); in svm_load_mmu_pgd()
4480 cr3 = vcpu->arch.cr3; in svm_load_mmu_pgd()
4484 /* PCID in the guest should be impossible with a 32-bit MMU. */ in svm_load_mmu_pgd()
4489 svm->vmcb->save.cr3 = cr3; in svm_load_mmu_pgd()
4490 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in svm_load_mmu_pgd()
4517 /* SEV-ES guests do not support SMM, so report false */ in svm_has_emulated_msr()
4539 * XSS on VM-Enter/VM-Exit. Failure to do so would effectively give in svm_vcpu_after_set_cpuid()
4547 * Intercept VMLOAD if the vCPU model is Intel in order to emulate that in svm_vcpu_after_set_cpuid()
4549 * SVM on Intel is bonkers and extremely unlikely to work). in svm_vcpu_after_set_cpuid()
4557 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, in svm_vcpu_after_set_cpuid()
4561 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0, in svm_vcpu_after_set_cpuid()
4564 if (sev_guest(vcpu->kvm)) in svm_vcpu_after_set_cpuid()
4647 struct vmcb *vmcb = svm->vmcb; in svm_check_intercept()
4649 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) in svm_check_intercept()
4652 icpt_info = x86_intercept_map[info->intercept]; in svm_check_intercept()
4659 if (info->intercept == x86_intercept_cr_read) in svm_check_intercept()
4660 icpt_info.exit_code += info->modrm_reg; in svm_check_intercept()
4665 if (info->intercept == x86_intercept_cr_write) in svm_check_intercept()
4666 icpt_info.exit_code += info->modrm_reg; in svm_check_intercept()
4669 info->intercept == x86_intercept_clts) in svm_check_intercept()
4672 if (!(vmcb12_is_intercept(&svm->nested.ctl, in svm_check_intercept()
4676 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; in svm_check_intercept()
4677 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; in svm_check_intercept()
4679 if (info->intercept == x86_intercept_lmsw) { in svm_check_intercept()
4682 /* lmsw can't clear PE - catch this here */ in svm_check_intercept()
4694 icpt_info.exit_code += info->modrm_reg; in svm_check_intercept()
4697 if (info->intercept == x86_intercept_wrmsr) in svm_check_intercept()
4698 vmcb->control.exit_info_1 = 1; in svm_check_intercept()
4700 vmcb->control.exit_info_1 = 0; in svm_check_intercept()
4705 * is rep not, check this here in svm_check_intercept()
4707 if (info->rep_prefix != REPE_PREFIX) in svm_check_intercept()
4714 if (info->intercept == x86_intercept_in || in svm_check_intercept()
4715 info->intercept == x86_intercept_ins) { in svm_check_intercept()
4716 exit_info = ((info->src_val & 0xffff) << 16) | in svm_check_intercept()
4718 bytes = info->dst_bytes; in svm_check_intercept()
4720 exit_info = (info->dst_val & 0xffff) << 16; in svm_check_intercept()
4721 bytes = info->src_bytes; in svm_check_intercept()
4724 if (info->intercept == x86_intercept_outs || in svm_check_intercept()
4725 info->intercept == x86_intercept_ins) in svm_check_intercept()
4728 if (info->rep_prefix) in svm_check_intercept()
4735 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); in svm_check_intercept()
4737 vmcb->control.exit_info_1 = exit_info; in svm_check_intercept()
4738 vmcb->control.exit_info_2 = info->next_rip; in svm_check_intercept()
4748 vmcb->control.next_rip = info->next_rip; in svm_check_intercept()
4749 vmcb->control.exit_code = icpt_info.exit_code; in svm_check_intercept()
4761 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR) in svm_handle_exit_irqoff()
4762 vcpu->arch.at_instruction_boundary = true; in svm_handle_exit_irqoff()
4768 vcpu->arch.mcg_cap &= 0x1ff; in svm_setup_mce()
4786 if (svm->nested.nested_run_pending) in svm_smi_allowed()
4787 return -EBUSY; in svm_smi_allowed()
4792 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ in svm_smi_allowed()
4794 return -EBUSY; in svm_smi_allowed()
4809 * 32-bit SMRAM format doesn't preserve EFER and SVM state. Userspace is in svm_enter_smm()
4816 smram->smram64.svm_guest_flag = 1; in svm_enter_smm()
4817 smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa; in svm_enter_smm()
4819 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; in svm_enter_smm()
4820 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; in svm_enter_smm()
4821 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; in svm_enter_smm()
4829 * VMCB01 is going to be used during SMM and thus the state will in svm_enter_smm()
4830 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save in svm_enter_smm()
4832 * format of the area is identical to guest save area offsetted in svm_enter_smm()
4839 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) in svm_enter_smm()
4845 &svm->vmcb01.ptr->save); in svm_enter_smm()
4858 const struct kvm_smram_state_64 *smram64 = &smram->smram64; in svm_leave_smm()
4863 /* Non-zero if SMI arrived while vCPU was in guest mode. */ in svm_leave_smm()
4864 if (!smram64->svm_guest_flag) in svm_leave_smm()
4870 if (!(smram64->efer & EFER_SVME)) in svm_leave_smm()
4873 if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map)) in svm_leave_smm()
4877 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) in svm_leave_smm()
4888 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400); in svm_leave_smm()
4894 vmcb_mark_all_dirty(svm->vmcb01.ptr); in svm_leave_smm()
4897 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); in svm_leave_smm()
4898 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); in svm_leave_smm()
4899 ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false); in svm_leave_smm()
4904 svm->nested.nested_run_pending = 1; in svm_leave_smm()
4934 /* Check that emulation is possible during event vectoring */ in svm_check_emulate_instruction()
4935 if ((svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK) && in svm_check_emulate_instruction()
4939 /* Emulation is always possible when KVM has access to all guest state. */ in svm_check_emulate_instruction()
4940 if (!sev_guest(vcpu->kvm)) in svm_check_emulate_instruction()
4949 * Emulation is impossible for SEV-ES guests as KVM doesn't have access in svm_check_emulate_instruction()
4952 if (sev_es_guest(vcpu->kvm)) in svm_check_emulate_instruction()
4956 * Emulation is possible if the instruction is already decoded, e.g. in svm_check_emulate_instruction()
4963 * Emulation is possible for SEV guests if and only if a prefilled in svm_check_emulate_instruction()
4964 * buffer containing the bytes of the intercepted instruction is in svm_check_emulate_instruction()
4965 * available. SEV guest memory is encrypted with a guest specific key in svm_check_emulate_instruction()
4969 * If KVM is NOT trying to simply skip an instruction, inject #UD if in svm_check_emulate_instruction()
4971 * this path should never be hit by a well-behaved guest, e.g. KVM in svm_check_emulate_instruction()
4972 * doesn't intercept #UD or #GP for SEV guests, but this path is still in svm_check_emulate_instruction()
4973 * theoretically reachable, e.g. via unaccelerated fault-like AVIC in svm_check_emulate_instruction()
4975 * into an infinite loop. Injecting #UD is somewhat arbitrary, but in svm_check_emulate_instruction()
4978 * If KVM is trying to skip an instruction, simply resume the guest. in svm_check_emulate_instruction()
4979 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM in svm_check_emulate_instruction()
4980 * will attempt to re-inject the INT3/INTO and skip the instruction. in svm_check_emulate_instruction()
4982 * make forward progress is the only option that has a chance of in svm_check_emulate_instruction()
4994 * Emulate for SEV guests if the insn buffer is not empty. The buffer in svm_check_emulate_instruction()
4998 * table used to translate CS:RIP resides in emulated MMIO. in svm_check_emulate_instruction()
5007 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is in svm_check_emulate_instruction()
5009 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly in svm_check_emulate_instruction()
5010 * be '0'. This happens because microcode reads CS:RIP using a _data_ in svm_check_emulate_instruction()
5014 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU in svm_check_emulate_instruction()
5020 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate in svm_check_emulate_instruction()
5022 * encountered a reserved/not-present #PF. in svm_check_emulate_instruction()
5029 * 3. The #NPF is not due to a code fetch, in which case failure to in svm_check_emulate_instruction()
5030 * retrieve the instruction bytes is legitimate (see abvoe). in svm_check_emulate_instruction()
5035 error_code = svm->vmcb->control.exit_info_1; in svm_check_emulate_instruction()
5052 * In practice, the triple fault is moot as no sane SEV kernel in svm_check_emulate_instruction()
5066 * if the fault is at CPL=0, it's the lesser of all evils. Exiting to in svm_check_emulate_instruction()
5070 * Simply resuming the guest is technically not a violation of the SEV in svm_check_emulate_instruction()
5072 * accesses for SEV guest are encrypted, regardless of the C-Bit. The in svm_check_emulate_instruction()
5075 * the guest spin is technically "ignoring" the access. in svm_check_emulate_instruction()
5089 if (!sev_es_guest(vcpu->kvm)) in svm_vcpu_deliver_sipi_vector()
5105 int type = kvm->arch.vm_type; in svm_vm_init()
5109 kvm->arch.has_protected_state = in svm_vm_init()
5111 to_kvm_sev_info(kvm)->need_init = true; in svm_vm_init()
5113 kvm->arch.has_private_mem = (type == KVM_X86_SNP_VM); in svm_vm_init()
5114 kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; in svm_vm_init()
5118 kvm->arch.pause_in_guest = true; in svm_vm_init()
5283 * The default MMIO mask is a single bit (excluding the present bit),
5286 * memory encryption is enabled.
5293 /* If there is no memory encryption support, use existing mask */ in svm_adjust_mmio_mask()
5297 /* If memory encryption is not enabled, use existing mask */ in svm_adjust_mmio_mask()
5305 /* Increment the mask bit if it is the same as the encryption bit */ in svm_adjust_mmio_mask()
5310 * If the mask bit location is below 52, then some bits above the in svm_adjust_mmio_mask()
5316 * If the mask bit location is 52 (or above), then clear the mask. in svm_adjust_mmio_mask()
5338 * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush. in svm_set_cpu_caps()
5397 /* Don't advertise Bus Lock Detect to guest if SVM support is absent */ in svm_set_cpu_caps()
5410 * NX is required for shadow paging and for NPT if the NX huge pages in svm_hardware_setup()
5411 * mitigation is enabled. in svm_hardware_setup()
5415 return -EOPNOTSUPP; in svm_hardware_setup()
5422 return -ENOMEM; in svm_hardware_setup()
5466 * KVM's MMU doesn't support using 2-level paging for itself, and thus in svm_hardware_setup()
5467 * NPT isn't supported if the host is using 2-level paging since host in svm_hardware_setup()
5468 * CR4 is unchanged on VMRUN. in svm_hardware_setup()
5548 pr_info("PMU virtualization is disabled\n"); in svm_hardware_setup()
5553 * It seems that on AMD processors PTE's accessed bit is in svm_hardware_setup()
5555 * This is not expected behaviour and our tests fail because in svm_hardware_setup()
5557 * A workaround here is to disable support for in svm_hardware_setup()
5558 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. in svm_hardware_setup()
5559 * In this case userspace can know if there is support using in svm_hardware_setup()
5594 return -EOPNOTSUPP; in svm_init()
5601 * Common KVM initialization _must_ come last, after this, /dev/kvm is in svm_init()