Lines Matching +full:supervisor +full:- +full:mode +full:- +full:visible
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Kernel-based Virtual Machine driver for Linux
5 * This module enables machines with Intel VT-x extensions to run virtual
31 #include <linux/entry-kvm.h>
49 #include <asm/spec-ctrl.h>
134 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */
186 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
202 /* Default doubles per-vcpu window every exit. */
206 /* Default resets per-vcpu window every exit to ple_window. */
214 /* Default is SYSTEM mode, 1 for host-guest mode */
288 return -ENOMEM; in vmx_setup_l1d_flush()
327 return -EINVAL; in vmentry_l1d_flush_parse()
370 if (!vmx->disable_fb_clear) in vmx_disable_fb_clear()
377 vmx->msr_ia32_mcu_opt_ctrl = msr; in vmx_disable_fb_clear()
382 if (!vmx->disable_fb_clear) in vmx_enable_fb_clear()
385 vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; in vmx_enable_fb_clear()
386 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); in vmx_enable_fb_clear()
396 * CPU buffers is unnecessary, at the cost of MSR accesses on VM-Entry in vmx_update_fb_clear_dis()
397 * and VM-Exit. in vmx_update_fb_clear_dis()
399 vmx->disable_fb_clear = !cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF) && in vmx_update_fb_clear_dis()
409 if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || in vmx_update_fb_clear_dis()
410 ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && in vmx_update_fb_clear_dis()
411 (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && in vmx_update_fb_clear_dis()
412 (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && in vmx_update_fb_clear_dis()
413 (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && in vmx_update_fb_clear_dis()
414 (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) in vmx_update_fb_clear_dis()
415 vmx->disable_fb_clear = false; in vmx_update_fb_clear_dis()
485 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
522 vmx->segment_cache.bitmask = 0; in vmx_segment_cache_clear()
539 return -ENOMEM; in hv_enable_l2_tlb_flush()
541 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; in hv_enable_l2_tlb_flush()
543 evmcs->partition_assist_page = partition_assist_page; in hv_enable_l2_tlb_flush()
544 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; in hv_enable_l2_tlb_flush()
545 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; in hv_enable_l2_tlb_flush()
574 pr_info("Using Hyper-V Enlightened VMCS\n"); in hv_init_evmcs()
604 * Reset everything to support using non-enlightened VMCS access later in hv_reset_evmcs()
607 vp_ap->nested_control.features.directhypercall = 0; in hv_reset_evmcs()
608 vp_ap->current_nested_vmcs = 0; in hv_reset_evmcs()
609 vp_ap->enlighten_vmentry = 0; in hv_reset_evmcs()
618 * Comment's format: document - errata name - stepping - processor name.
623 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */
625 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */
626 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
627 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
629 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
631 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */
632 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */
634 * 320767.pdf - AAP86 - B1 -
635 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
638 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
640 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
642 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
644 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
645 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
646 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
648 /* Xeon E3-1220 V2 */
678 return -ENOENT; in possible_passthrough_msr_slot()
706 r = possible_passthrough_msr_slot(msr) != -ENOENT; in is_valid_passthrough_msr()
719 return &vmx->guest_uret_msrs[i]; in vmx_find_uret_msr()
726 unsigned int slot = msr - vmx->guest_uret_msrs; in vmx_set_guest_uret_msr()
729 if (msr->load_into_hardware) { in vmx_set_guest_uret_msr()
731 ret = kvm_set_user_return_msr(slot, data, msr->mask); in vmx_set_guest_uret_msr()
735 msr->data = data; in vmx_set_guest_uret_msr()
742 * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
743 * atomically track post-VMXON state, e.g. this may be called in NMI context.
744 * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
745 * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
746 * magically in RM, VM86, compat mode, or at CPL>0.
759 return -EIO; in kvm_cpu_vmxoff()
780 vmcs_clear(v->vmcs); in vmx_emergency_disable()
790 if (loaded_vmcs->cpu != cpu) in __loaded_vmcs_clear()
792 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) in __loaded_vmcs_clear()
795 vmcs_clear(loaded_vmcs->vmcs); in __loaded_vmcs_clear()
796 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) in __loaded_vmcs_clear()
797 vmcs_clear(loaded_vmcs->shadow_vmcs); in __loaded_vmcs_clear()
799 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); in __loaded_vmcs_clear()
803 * current percpu list, complete before setting loaded_vmcs->cpu to in __loaded_vmcs_clear()
804 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first in __loaded_vmcs_clear()
810 loaded_vmcs->cpu = -1; in __loaded_vmcs_clear()
811 loaded_vmcs->launched = 0; in __loaded_vmcs_clear()
816 int cpu = loaded_vmcs->cpu; in loaded_vmcs_clear()
818 if (cpu != -1) in loaded_vmcs_clear()
829 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { in vmx_segment_cache_test_set()
830 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); in vmx_segment_cache_test_set()
831 vmx->segment_cache.bitmask = 0; in vmx_segment_cache_test_set()
833 ret = vmx->segment_cache.bitmask & mask; in vmx_segment_cache_test_set()
834 vmx->segment_cache.bitmask |= mask; in vmx_segment_cache_test_set()
840 u16 *p = &vmx->segment_cache.seg[seg].selector; in vmx_read_guest_seg_selector()
849 ulong *p = &vmx->segment_cache.seg[seg].base; in vmx_read_guest_seg_base()
858 u32 *p = &vmx->segment_cache.seg[seg].limit; in vmx_read_guest_seg_limit()
867 u32 *p = &vmx->segment_cache.seg[seg].ar; in vmx_read_guest_seg_ar()
888 if ((vcpu->guest_debug & in vmx_update_exception_bitmap()
892 if (to_vmx(vcpu)->rmode.vm86_active) in vmx_update_exception_bitmap()
903 eb |= get_vmcs12(vcpu)->exception_bitmap; in vmx_update_exception_bitmap()
912 * non-reserved faults. For vmcs02, however, PFEC_MASK in vmx_update_exception_bitmap()
927 if (vcpu->arch.xfd_no_write_intercept) in vmx_update_exception_bitmap()
941 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); in msr_write_intercepted()
948 if (vmx->loaded_vmcs->launched) in __vmx_vcpu_run_flags()
954 * it after vmexit and store it in vmx->spec_ctrl. in __vmx_vcpu_run_flags()
973 for (i = 0; i < m->nr; ++i) { in vmx_find_loadstore_msr_slot()
974 if (m->val[i].index == msr) in vmx_find_loadstore_msr_slot()
977 return -ENOENT; in vmx_find_loadstore_msr_slot()
983 struct msr_autoload *m = &vmx->msr_autoload; in clear_atomic_switch_msr()
1003 i = vmx_find_loadstore_msr_slot(&m->guest, msr); in clear_atomic_switch_msr()
1006 --m->guest.nr; in clear_atomic_switch_msr()
1007 m->guest.val[i] = m->guest.val[m->guest.nr]; in clear_atomic_switch_msr()
1008 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); in clear_atomic_switch_msr()
1011 i = vmx_find_loadstore_msr_slot(&m->host, msr); in clear_atomic_switch_msr()
1015 --m->host.nr; in clear_atomic_switch_msr()
1016 m->host.val[i] = m->host.val[m->host.nr]; in clear_atomic_switch_msr()
1017 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); in clear_atomic_switch_msr()
1036 struct msr_autoload *m = &vmx->msr_autoload; in add_atomic_switch_msr()
1070 i = vmx_find_loadstore_msr_slot(&m->guest, msr); in add_atomic_switch_msr()
1072 j = vmx_find_loadstore_msr_slot(&m->host, msr); in add_atomic_switch_msr()
1074 if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) || in add_atomic_switch_msr()
1075 (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) { in add_atomic_switch_msr()
1081 i = m->guest.nr++; in add_atomic_switch_msr()
1082 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); in add_atomic_switch_msr()
1084 m->guest.val[i].index = msr; in add_atomic_switch_msr()
1085 m->guest.val[i].value = guest_val; in add_atomic_switch_msr()
1091 j = m->host.nr++; in add_atomic_switch_msr()
1092 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); in add_atomic_switch_msr()
1094 m->host.val[j].index = msr; in add_atomic_switch_msr()
1095 m->host.val[j].value = host_val; in add_atomic_switch_msr()
1100 u64 guest_efer = vmx->vcpu.arch.efer; in update_transition_efer()
1109 * LMA and LME handled by hardware; SCE meaningless outside long mode. in update_transition_efer()
1114 /* SCE is meaningful only in long mode on Intel */ in update_transition_efer()
1125 (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) { in update_transition_efer()
1145 vmx->guest_uret_msrs[i].data = guest_efer; in update_transition_efer()
1146 vmx->guest_uret_msrs[i].mask = ~ignore_bits; in update_transition_efer()
1153 * On 32-bit kernels, VM exits still load the FS and GS bases from the
1183 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); in pt_can_write_msr()
1188 /* The base must be 128-byte aligned and a legal physical address. */ in pt_output_base_valid()
1196 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); in pt_load_msr()
1197 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); in pt_load_msr()
1198 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); in pt_load_msr()
1199 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); in pt_load_msr()
1201 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); in pt_load_msr()
1202 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); in pt_load_msr()
1210 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); in pt_save_msr()
1211 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); in pt_save_msr()
1212 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); in pt_save_msr()
1213 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); in pt_save_msr()
1215 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); in pt_save_msr()
1216 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); in pt_save_msr()
1229 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); in pt_guest_enter()
1230 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { in pt_guest_enter()
1232 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); in pt_guest_enter()
1233 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); in pt_guest_enter()
1242 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { in pt_guest_exit()
1243 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); in pt_guest_exit()
1244 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); in pt_guest_exit()
1249 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. in pt_guest_exit()
1251 if (vmx->pt_desc.host.ctl) in pt_guest_exit()
1252 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); in pt_guest_exit()
1258 if (unlikely(fs_sel != host->fs_sel)) { in vmx_set_host_fs_gs()
1263 host->fs_sel = fs_sel; in vmx_set_host_fs_gs()
1265 if (unlikely(gs_sel != host->gs_sel)) { in vmx_set_host_fs_gs()
1270 host->gs_sel = gs_sel; in vmx_set_host_fs_gs()
1272 if (unlikely(fs_base != host->fs_base)) { in vmx_set_host_fs_gs()
1274 host->fs_base = fs_base; in vmx_set_host_fs_gs()
1276 if (unlikely(gs_base != host->gs_base)) { in vmx_set_host_fs_gs()
1278 host->gs_base = gs_base; in vmx_set_host_fs_gs()
1293 vmx->req_immediate_exit = false; in vmx_prepare_switch_to_guest()
1298 * to/from long-mode by setting MSR_EFER.LMA. in vmx_prepare_switch_to_guest()
1300 if (!vmx->guest_uret_msrs_loaded) { in vmx_prepare_switch_to_guest()
1301 vmx->guest_uret_msrs_loaded = true; in vmx_prepare_switch_to_guest()
1303 if (!vmx->guest_uret_msrs[i].load_into_hardware) in vmx_prepare_switch_to_guest()
1307 vmx->guest_uret_msrs[i].data, in vmx_prepare_switch_to_guest()
1308 vmx->guest_uret_msrs[i].mask); in vmx_prepare_switch_to_guest()
1312 if (vmx->nested.need_vmcs12_to_shadow_sync) in vmx_prepare_switch_to_guest()
1315 if (vmx->guest_state_loaded) in vmx_prepare_switch_to_guest()
1318 host_state = &vmx->loaded_vmcs->host_state; in vmx_prepare_switch_to_guest()
1324 host_state->ldt_sel = kvm_read_ldt(); in vmx_prepare_switch_to_guest()
1327 savesegment(ds, host_state->ds_sel); in vmx_prepare_switch_to_guest()
1328 savesegment(es, host_state->es_sel); in vmx_prepare_switch_to_guest()
1331 if (likely(is_64bit_mm(current->mm))) { in vmx_prepare_switch_to_guest()
1333 fs_sel = current->thread.fsindex; in vmx_prepare_switch_to_guest()
1334 gs_sel = current->thread.gsindex; in vmx_prepare_switch_to_guest()
1335 fs_base = current->thread.fsbase; in vmx_prepare_switch_to_guest()
1336 vmx->msr_host_kernel_gs_base = current->thread.gsbase; in vmx_prepare_switch_to_guest()
1341 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); in vmx_prepare_switch_to_guest()
1344 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); in vmx_prepare_switch_to_guest()
1353 vmx->guest_state_loaded = true; in vmx_prepare_switch_to_guest()
1360 if (!vmx->guest_state_loaded) in vmx_prepare_switch_to_host()
1363 host_state = &vmx->loaded_vmcs->host_state; in vmx_prepare_switch_to_host()
1365 ++vmx->vcpu.stat.host_state_reload; in vmx_prepare_switch_to_host()
1368 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); in vmx_prepare_switch_to_host()
1370 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { in vmx_prepare_switch_to_host()
1371 kvm_load_ldt(host_state->ldt_sel); in vmx_prepare_switch_to_host()
1373 load_gs_index(host_state->gs_sel); in vmx_prepare_switch_to_host()
1375 loadsegment(gs, host_state->gs_sel); in vmx_prepare_switch_to_host()
1378 if (host_state->fs_sel & 7) in vmx_prepare_switch_to_host()
1379 loadsegment(fs, host_state->fs_sel); in vmx_prepare_switch_to_host()
1381 if (unlikely(host_state->ds_sel | host_state->es_sel)) { in vmx_prepare_switch_to_host()
1382 loadsegment(ds, host_state->ds_sel); in vmx_prepare_switch_to_host()
1383 loadsegment(es, host_state->es_sel); in vmx_prepare_switch_to_host()
1388 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); in vmx_prepare_switch_to_host()
1391 vmx->guest_state_loaded = false; in vmx_prepare_switch_to_host()
1392 vmx->guest_uret_msrs_loaded = false; in vmx_prepare_switch_to_host()
1399 if (vmx->guest_state_loaded) in vmx_read_guest_kernel_gs_base()
1400 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); in vmx_read_guest_kernel_gs_base()
1402 return vmx->msr_guest_kernel_gs_base; in vmx_read_guest_kernel_gs_base()
1408 if (vmx->guest_state_loaded) in vmx_write_guest_kernel_gs_base()
1411 vmx->msr_guest_kernel_gs_base = data; in vmx_write_guest_kernel_gs_base()
1419 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; in vmx_vcpu_load_vmcs()
1423 loaded_vmcs_clear(vmx->loaded_vmcs); in vmx_vcpu_load_vmcs()
1427 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to in vmx_vcpu_load_vmcs()
1434 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, in vmx_vcpu_load_vmcs()
1440 if (prev != vmx->loaded_vmcs->vmcs) { in vmx_vcpu_load_vmcs()
1441 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; in vmx_vcpu_load_vmcs()
1442 vmcs_load(vmx->loaded_vmcs->vmcs); in vmx_vcpu_load_vmcs()
1448 * performs IBPB on nested VM-Exit (a single nested transition in vmx_vcpu_load_vmcs()
1451 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) in vmx_vcpu_load_vmcs()
1465 * Linux uses per-cpu TSS and GDT, so set these when switching in vmx_vcpu_load_vmcs()
1469 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); in vmx_vcpu_load_vmcs()
1478 vmx->loaded_vmcs->cpu = cpu; in vmx_vcpu_load_vmcs()
1494 vmx->host_debugctlmsr = get_debugctlmsr(); in vmx_vcpu_load()
1517 if (vmx->rmode.vm86_active) { in vmx_get_rflags()
1519 save_rflags = vmx->rmode.save_rflags; in vmx_get_rflags()
1522 vmx->rflags = rflags; in vmx_get_rflags()
1524 return vmx->rflags; in vmx_get_rflags()
1539 vmx->rflags = rflags; in vmx_set_rflags()
1545 vmx->rflags = rflags; in vmx_set_rflags()
1546 if (vmx->rmode.vm86_active) { in vmx_set_rflags()
1547 vmx->rmode.save_rflags = rflags; in vmx_set_rflags()
1552 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) in vmx_set_rflags()
1553 vmx->emulation_required = vmx_emulation_required(vcpu); in vmx_set_rflags()
1599 if (data & vmx->pt_desc.ctl_bitmask) in vmx_rtit_ctl_check()
1606 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && in vmx_rtit_ctl_check()
1607 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN)) in vmx_rtit_ctl_check()
1617 !intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_rtit_ctl_check()
1625 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); in vmx_rtit_ctl_check()
1626 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && in vmx_rtit_ctl_check()
1630 value = intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_rtit_ctl_check()
1632 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && in vmx_rtit_ctl_check()
1636 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); in vmx_rtit_ctl_check()
1637 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && in vmx_rtit_ctl_check()
1647 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) in vmx_rtit_ctl_check()
1650 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) in vmx_rtit_ctl_check()
1653 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) in vmx_rtit_ctl_check()
1656 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) in vmx_rtit_ctl_check()
1672 if (to_vmx(vcpu)->exit_reason.enclave_mode) { in vmx_check_emulate_instruction()
1681 union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason; in skip_emulated_instruction()
1690 * (namely Hyper-V) don't set it due to it being undefined behavior, in skip_emulated_instruction()
1701 * the RIP that actually triggered the VM-Exit. But, because in skip_emulated_instruction()
1702 * most instructions that cause VM-Exit will #UD in an enclave, in skip_emulated_instruction()
1703 * most instruction-based VM-Exits simply do not occur. in skip_emulated_instruction()
1710 * WARN if KVM tries to skip a non-zero length instruction on in skip_emulated_instruction()
1711 * a VM-Exit from an enclave. in skip_emulated_instruction()
1717 "skipping instruction after SGX enclave VM-Exit"); in skip_emulated_instruction()
1723 * We need to mask out the high 32 bits of RIP if not in 64-bit in skip_emulated_instruction()
1724 * mode, but just finding out that we are in 64-bit mode is in skip_emulated_instruction()
1744 * Recognizes a pending MTF VM-exit and records the nested state for later
1756 * Per the SDM, MTF takes priority over debug-trap exceptions besides in vmx_update_emulated_instruction()
1757 * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps in vmx_update_emulated_instruction()
1759 * intercepted #DB deliberately avoids single-step #DB and MTF updates in vmx_update_emulated_instruction()
1762 * any #DB exception pending delivery must be a debug-trap of lower in vmx_update_emulated_instruction()
1767 (!vcpu->arch.exception.pending || in vmx_update_emulated_instruction()
1768 vcpu->arch.exception.vector == DB_VECTOR) && in vmx_update_emulated_instruction()
1769 (!vcpu->arch.exception_vmexit.pending || in vmx_update_emulated_instruction()
1770 vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { in vmx_update_emulated_instruction()
1771 vmx->nested.mtf_pending = true; in vmx_update_emulated_instruction()
1774 vmx->nested.mtf_pending = false; in vmx_update_emulated_instruction()
1792 if (kvm_hlt_in_guest(vcpu->kvm) && in vmx_clear_hlt()
1799 struct kvm_queued_exception *ex = &vcpu->arch.exception; in vmx_inject_exception()
1800 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; in vmx_inject_exception()
1805 if (ex->has_error_code) { in vmx_inject_exception()
1812 * ABI lets userspace shove in arbitrary 32-bit values. Drop in vmx_inject_exception()
1813 * the upper bits to avoid VM-Fail, losing information that in vmx_inject_exception()
1816 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); in vmx_inject_exception()
1820 if (vmx->rmode.vm86_active) { in vmx_inject_exception()
1822 if (kvm_exception_is_soft(ex->vector)) in vmx_inject_exception()
1823 inc_eip = vcpu->arch.event_exit_inst_len; in vmx_inject_exception()
1824 kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); in vmx_inject_exception()
1828 WARN_ON_ONCE(vmx->emulation_required); in vmx_inject_exception()
1830 if (kvm_exception_is_soft(ex->vector)) { in vmx_inject_exception()
1832 vmx->vcpu.arch.event_exit_inst_len); in vmx_inject_exception()
1851 uret_msr->load_into_hardware = load_into_hardware; in vmx_setup_uret_msr()
1866 * The SYSCALL MSRs are only needed on long mode guests, and only in vmx_setup_uret_msrs()
1869 load_syscall_msrs = is_long_mode(&vmx->vcpu) && in vmx_setup_uret_msrs()
1870 (vmx->vcpu.arch.efer & EFER_SCE); in vmx_setup_uret_msrs()
1879 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || in vmx_setup_uret_msrs()
1880 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDPID)); in vmx_setup_uret_msrs()
1892 * next VM-Enter. in vmx_setup_uret_msrs()
1894 vmx->guest_uret_msrs_loaded = false; in vmx_setup_uret_msrs()
1902 return vmcs12->tsc_offset; in vmx_get_l2_tsc_offset()
1913 return vmcs12->tsc_multiplier; in vmx_get_l2_tsc_multiplier()
1920 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); in vmx_write_tsc_offset()
1925 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); in vmx_write_tsc_multiplier()
1951 WARN_ON_ONCE(vmx->msr_ia32_feature_control_valid_bits & in is_vmx_feature_control_msr_valid()
1954 if (!msr->host_initiated && in is_vmx_feature_control_msr_valid()
1955 (vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED)) in is_vmx_feature_control_msr_valid()
1958 if (msr->host_initiated) in is_vmx_feature_control_msr_valid()
1961 valid_bits = vmx->msr_ia32_feature_control_valid_bits; in is_vmx_feature_control_msr_valid()
1963 return !(msr->data & ~valid_bits); in is_vmx_feature_control_msr_valid()
1968 switch (msr->index) { in vmx_get_msr_feature()
1972 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data); in vmx_get_msr_feature()
1979 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'.
1980 * Returns 0 on success, non-0 otherwise.
1989 switch (msr_info->index) { in vmx_get_msr()
1992 msr_info->data = vmcs_readl(GUEST_FS_BASE); in vmx_get_msr()
1995 msr_info->data = vmcs_readl(GUEST_GS_BASE); in vmx_get_msr()
1998 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); in vmx_get_msr()
2004 if (!msr_info->host_initiated && in vmx_get_msr()
2005 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) in vmx_get_msr()
2009 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) in vmx_get_msr()
2012 msr_info->data = vmx->msr_ia32_umwait_control; in vmx_get_msr()
2015 if (!msr_info->host_initiated && in vmx_get_msr()
2019 msr_info->data = to_vmx(vcpu)->spec_ctrl; in vmx_get_msr()
2022 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); in vmx_get_msr()
2025 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); in vmx_get_msr()
2028 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); in vmx_get_msr()
2032 (!msr_info->host_initiated && in vmx_get_msr()
2035 msr_info->data = vmcs_read64(GUEST_BNDCFGS); in vmx_get_msr()
2038 if (!msr_info->host_initiated && in vmx_get_msr()
2039 !(vmx->msr_ia32_feature_control & in vmx_get_msr()
2042 msr_info->data = vcpu->arch.mcg_ext_ctl; in vmx_get_msr()
2045 msr_info->data = vmx->msr_ia32_feature_control; in vmx_get_msr()
2048 if (!msr_info->host_initiated && in vmx_get_msr()
2051 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash in vmx_get_msr()
2052 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; in vmx_get_msr()
2057 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, in vmx_get_msr()
2058 &msr_info->data)) in vmx_get_msr()
2063 * instead of just ignoring the features, different Hyper-V in vmx_get_msr()
2068 if (!msr_info->host_initiated && guest_cpuid_has_evmcs(vcpu)) in vmx_get_msr()
2069 nested_evmcs_filter_control_msr(vcpu, msr_info->index, in vmx_get_msr()
2070 &msr_info->data); in vmx_get_msr()
2076 msr_info->data = vmx->pt_desc.guest.ctl; in vmx_get_msr()
2081 msr_info->data = vmx->pt_desc.guest.status; in vmx_get_msr()
2085 !intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_get_msr()
2088 msr_info->data = vmx->pt_desc.guest.cr3_match; in vmx_get_msr()
2092 (!intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_get_msr()
2094 !intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_get_msr()
2097 msr_info->data = vmx->pt_desc.guest.output_base; in vmx_get_msr()
2101 (!intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_get_msr()
2103 !intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_get_msr()
2106 msr_info->data = vmx->pt_desc.guest.output_mask; in vmx_get_msr()
2109 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; in vmx_get_msr()
2111 (index >= 2 * vmx->pt_desc.num_address_ranges)) in vmx_get_msr()
2114 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; in vmx_get_msr()
2116 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; in vmx_get_msr()
2119 msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL); in vmx_get_msr()
2123 msr = vmx_find_uret_msr(vmx, msr_info->index); in vmx_get_msr()
2125 msr_info->data = msr->data; in vmx_get_msr()
2161 * Returns 0 on success, non-0 otherwise.
2169 u32 msr_index = msr_info->index; in vmx_set_msr()
2170 u64 data = msr_info->data; in vmx_set_msr()
2192 * Always intercepting WRMSR could incur non-negligible in vmx_set_msr()
2195 * upon the first write with a non-zero value (indicating in vmx_set_msr()
2203 vcpu->arch.xfd_no_write_intercept = true; in vmx_set_msr()
2210 get_vmcs12(vcpu)->guest_sysenter_cs = data; in vmx_set_msr()
2216 get_vmcs12(vcpu)->guest_sysenter_eip = data; in vmx_set_msr()
2223 get_vmcs12(vcpu)->guest_sysenter_esp = data; in vmx_set_msr()
2230 invalid = data & ~vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); in vmx_set_msr()
2240 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & in vmx_set_msr()
2242 get_vmcs12(vcpu)->guest_ia32_debugctl = data; in vmx_set_msr()
2245 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && in vmx_set_msr()
2252 (!msr_info->host_initiated && in vmx_set_msr()
2260 ((vmx->nested.msrs.entry_ctls_high & VM_ENTRY_LOAD_BNDCFGS) || in vmx_set_msr()
2261 (vmx->nested.msrs.exit_ctls_high & VM_EXIT_CLEAR_BNDCFGS))) in vmx_set_msr()
2262 get_vmcs12(vcpu)->guest_bndcfgs = data; in vmx_set_msr()
2267 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) in vmx_set_msr()
2270 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ in vmx_set_msr()
2274 vmx->msr_ia32_umwait_control = data; in vmx_set_msr()
2277 if (!msr_info->host_initiated && in vmx_set_msr()
2284 vmx->spec_ctrl = data; in vmx_set_msr()
2289 * For non-nested: in vmx_set_msr()
2290 * When it's written (to non-zero) for the first time, pass in vmx_set_msr()
2305 if (!msr_info->host_initiated && in vmx_set_msr()
2306 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) in vmx_set_msr()
2317 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) in vmx_set_msr()
2318 get_vmcs12(vcpu)->guest_ia32_pat = data; in vmx_set_msr()
2324 if ((!msr_info->host_initiated && in vmx_set_msr()
2325 !(to_vmx(vcpu)->msr_ia32_feature_control & in vmx_set_msr()
2329 vcpu->arch.mcg_ext_ctl = data; in vmx_set_msr()
2335 vmx->msr_ia32_feature_control = data; in vmx_set_msr()
2336 if (msr_info->host_initiated && data == 0) in vmx_set_msr()
2354 if (!msr_info->host_initiated && in vmx_set_msr()
2356 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && in vmx_set_msr()
2357 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) in vmx_set_msr()
2359 vmx->msr_ia32_sgxlepubkeyhash in vmx_set_msr()
2360 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; in vmx_set_msr()
2363 if (!msr_info->host_initiated) in vmx_set_msr()
2364 return 1; /* they are read-only */ in vmx_set_msr()
2371 vmx->nested.vmxon) in vmx_set_msr()
2374 vmx->pt_desc.guest.ctl = data; in vmx_set_msr()
2382 vmx->pt_desc.guest.status = data; in vmx_set_msr()
2387 if (!intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_set_msr()
2390 vmx->pt_desc.guest.cr3_match = data; in vmx_set_msr()
2395 if (!intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_set_msr()
2397 !intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_set_msr()
2402 vmx->pt_desc.guest.output_base = data; in vmx_set_msr()
2407 if (!intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_set_msr()
2409 !intel_pt_validate_cap(vmx->pt_desc.caps, in vmx_set_msr()
2412 vmx->pt_desc.guest.output_mask = data; in vmx_set_msr()
2417 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; in vmx_set_msr()
2418 if (index >= 2 * vmx->pt_desc.num_address_ranges) in vmx_set_msr()
2423 vmx->pt_desc.guest.addr_b[index / 2] = data; in vmx_set_msr()
2425 vmx->pt_desc.guest.addr_a[index / 2] = data; in vmx_set_msr()
2428 if (data && !vcpu_to_pmu(vcpu)->version) in vmx_set_msr()
2475 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); in vmx_cache_reg()
2478 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); in vmx_cache_reg()
2485 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; in vmx_cache_reg()
2487 vcpu->arch.cr0 &= ~guest_owned_bits; in vmx_cache_reg()
2488 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; in vmx_cache_reg()
2496 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); in vmx_cache_reg()
2499 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; in vmx_cache_reg()
2501 vcpu->arch.cr4 &= ~guest_owned_bits; in vmx_cache_reg()
2502 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; in vmx_cache_reg()
2505 KVM_BUG_ON(1, vcpu->kvm); in vmx_cache_reg()
2557 return -EIO; in adjust_vmx_controls()
2607 return -EIO; in setup_vmcs_config()
2613 return -EIO; in setup_vmcs_config()
2628 &vmx_cap->ept, &vmx_cap->vpid); in setup_vmcs_config()
2631 vmx_cap->ept) { in setup_vmcs_config()
2633 "1-setting enable EPT VM-execution control\n"); in setup_vmcs_config()
2636 return -EIO; in setup_vmcs_config()
2638 vmx_cap->ept = 0; in setup_vmcs_config()
2641 vmx_cap->vpid) { in setup_vmcs_config()
2643 "1-setting enable VPID VM-execution control\n"); in setup_vmcs_config()
2646 return -EIO; in setup_vmcs_config()
2648 vmx_cap->vpid = 0; in setup_vmcs_config()
2663 return -EIO; in setup_vmcs_config()
2669 return -EIO; in setup_vmcs_config()
2681 return -EIO; in setup_vmcs_config()
2690 pr_warn_once("Inconsistent VM-Entry/VM-Exit pair, entry = %x, exit = %x\n", in setup_vmcs_config()
2694 return -EIO; in setup_vmcs_config()
2702 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ in setup_vmcs_config()
2704 return -EIO; in setup_vmcs_config()
2707 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ in setup_vmcs_config()
2709 return -EIO; in setup_vmcs_config()
2712 /* Require Write-Back (WB) memory type for VMCS accesses. */ in setup_vmcs_config()
2714 return -EIO; in setup_vmcs_config()
2718 vmcs_conf->size = vmx_msr_high & 0x1fff; in setup_vmcs_config()
2719 vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; in setup_vmcs_config()
2721 vmcs_conf->revision_id = vmx_msr_low; in setup_vmcs_config()
2723 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; in setup_vmcs_config()
2724 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; in setup_vmcs_config()
2725 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; in setup_vmcs_config()
2726 vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; in setup_vmcs_config()
2727 vmcs_conf->vmexit_ctrl = _vmexit_control; in setup_vmcs_config()
2728 vmcs_conf->vmentry_ctrl = _vmentry_control; in setup_vmcs_config()
2729 vmcs_conf->misc = misc_msr; in setup_vmcs_config()
2775 return -EIO; in vmx_check_processor_compat()
2779 return -EIO; in vmx_check_processor_compat()
2785 return -EIO; in vmx_check_processor_compat()
2807 return -EFAULT; in kvm_cpu_vmxon()
2817 return -EBUSY; in vmx_hardware_enable()
2820 * This can happen if we hot-added a CPU but failed to allocate in vmx_hardware_enable()
2824 return -EFAULT; in vmx_hardware_enable()
2876 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; in alloc_vmcs_cpu()
2878 vmcs->hdr.revision_id = vmcs_config.revision_id; in alloc_vmcs_cpu()
2881 vmcs->hdr.shadow_vmcs = 1; in alloc_vmcs_cpu()
2895 if (!loaded_vmcs->vmcs) in free_loaded_vmcs()
2898 free_vmcs(loaded_vmcs->vmcs); in free_loaded_vmcs()
2899 loaded_vmcs->vmcs = NULL; in free_loaded_vmcs()
2900 if (loaded_vmcs->msr_bitmap) in free_loaded_vmcs()
2901 free_page((unsigned long)loaded_vmcs->msr_bitmap); in free_loaded_vmcs()
2902 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); in free_loaded_vmcs()
2907 loaded_vmcs->vmcs = alloc_vmcs(false); in alloc_loaded_vmcs()
2908 if (!loaded_vmcs->vmcs) in alloc_loaded_vmcs()
2909 return -ENOMEM; in alloc_loaded_vmcs()
2911 vmcs_clear(loaded_vmcs->vmcs); in alloc_loaded_vmcs()
2913 loaded_vmcs->shadow_vmcs = NULL; in alloc_loaded_vmcs()
2914 loaded_vmcs->hv_timer_soft_disabled = false; in alloc_loaded_vmcs()
2915 loaded_vmcs->cpu = -1; in alloc_loaded_vmcs()
2916 loaded_vmcs->launched = 0; in alloc_loaded_vmcs()
2919 loaded_vmcs->msr_bitmap = (unsigned long *) in alloc_loaded_vmcs()
2921 if (!loaded_vmcs->msr_bitmap) in alloc_loaded_vmcs()
2923 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); in alloc_loaded_vmcs()
2926 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); in alloc_loaded_vmcs()
2927 memset(&loaded_vmcs->controls_shadow, 0, in alloc_loaded_vmcs()
2934 return -ENOMEM; in alloc_loaded_vmcs()
2957 return -ENOMEM; in alloc_kvm_area()
2962 * vmcs->revision_id to KVM_EVMCS_VERSION instead of in alloc_kvm_area()
2971 vmcs->hdr.revision_id = vmcs_config.revision_id; in alloc_kvm_area()
2985 * is in the middle of the transition from real mode to in fix_pmode_seg()
2986 * protected mode it is safe to assume that RPL 0 is a good in fix_pmode_seg()
2990 save->selector &= ~SEGMENT_RPL_MASK; in fix_pmode_seg()
2991 save->dpl = save->selector & SEGMENT_RPL_MASK; in fix_pmode_seg()
2992 save->s = 1; in fix_pmode_seg()
3003 * Update real mode segment cache. It may be not up-to-date if segment in enter_pmode()
3004 * register was written while vcpu was in a guest mode. in enter_pmode()
3006 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); in enter_pmode()
3007 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); in enter_pmode()
3008 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); in enter_pmode()
3009 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); in enter_pmode()
3010 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); in enter_pmode()
3011 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); in enter_pmode()
3013 vmx->rmode.vm86_active = 0; in enter_pmode()
3015 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); in enter_pmode()
3019 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; in enter_pmode()
3027 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); in enter_pmode()
3028 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); in enter_pmode()
3029 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); in enter_pmode()
3030 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); in enter_pmode()
3031 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); in enter_pmode()
3032 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); in enter_pmode()
3056 if (save->base & 0xf) in fix_rmode_seg()
3058 "when entering protected mode (seg=%d)", seg); in fix_rmode_seg()
3061 vmcs_write16(sf->selector, var.selector); in fix_rmode_seg()
3062 vmcs_writel(sf->base, var.base); in fix_rmode_seg()
3063 vmcs_write32(sf->limit, var.limit); in fix_rmode_seg()
3064 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); in fix_rmode_seg()
3071 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); in enter_rmode()
3074 * KVM should never use VM86 to virtualize Real Mode when L2 is active, in enter_rmode()
3076 * if unrestricted guest is disabled, VM-Enter (from L1) with CR0.PG=0 in enter_rmode()
3077 * should VM-Fail and KVM should reject userspace attempts to stuff in enter_rmode()
3082 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); in enter_rmode()
3083 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); in enter_rmode()
3084 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); in enter_rmode()
3085 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); in enter_rmode()
3086 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); in enter_rmode()
3087 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); in enter_rmode()
3088 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); in enter_rmode()
3090 vmx->rmode.vm86_active = 1; in enter_rmode()
3094 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); in enter_rmode()
3095 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); in enter_rmode()
3099 vmx->rmode.save_rflags = flags; in enter_rmode()
3107 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); in enter_rmode()
3108 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); in enter_rmode()
3109 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); in enter_rmode()
3110 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); in enter_rmode()
3111 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); in enter_rmode()
3112 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); in enter_rmode()
3123 vcpu->arch.efer = efer; in vmx_set_efer()
3130 if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) in vmx_set_efer()
3148 pr_debug_ratelimited("%s: tss fixup for long mode. \n", in enter_lmode()
3154 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); in enter_lmode()
3159 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); in exit_lmode()
3170 * the CPU is not required to invalidate guest-physical mappings on in vmx_flush_tlb_all()
3171 * VM-Entry, even if VPID is disabled. Guest-physical mappings are in vmx_flush_tlb_all()
3173 * (INVVPID also isn't required to invalidate guest-physical mappings). in vmx_flush_tlb_all()
3181 vpid_sync_vcpu_single(vmx->vpid); in vmx_flush_tlb_all()
3182 vpid_sync_vcpu_single(vmx->nested.vpid02); in vmx_flush_tlb_all()
3191 return to_vmx(vcpu)->vpid; in vmx_get_current_vpid()
3196 struct kvm_mmu *mmu = vcpu->arch.mmu; in vmx_flush_tlb_current()
3197 u64 root_hpa = mmu->root.hpa; in vmx_flush_tlb_current()
3205 mmu->root_role.level)); in vmx_flush_tlb_current()
3223 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are in vmx_flush_tlb_guest()
3224 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is in vmx_flush_tlb_guest()
3225 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), in vmx_flush_tlb_guest()
3233 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in vmx_ept_load_pdptrs()
3239 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); in vmx_ept_load_pdptrs()
3240 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); in vmx_ept_load_pdptrs()
3241 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); in vmx_ept_load_pdptrs()
3242 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); in vmx_ept_load_pdptrs()
3248 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in ept_save_pdptrs()
3253 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); in ept_save_pdptrs()
3254 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); in ept_save_pdptrs()
3255 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); in ept_save_pdptrs()
3256 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); in ept_save_pdptrs()
3269 if (to_vmx(vcpu)->nested.vmxon) in vmx_is_valid_cr0()
3291 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) in vmx_set_cr0()
3294 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) in vmx_set_cr0()
3300 vcpu->arch.cr0 = cr0; in vmx_set_cr0()
3304 if (vcpu->arch.efer & EFER_LME) { in vmx_set_cr0()
3314 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If in vmx_set_cr0()
3331 * e.g. after nested VM-Enter. in vmx_set_cr0()
3344 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; in vmx_set_cr0()
3348 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ in vmx_set_cr0()
3353 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but in vmx_set_cr0()
3354 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. in vmx_set_cr0()
3360 /* depends on vcpu->arch.cr0 to be set to a new value */ in vmx_set_cr0()
3361 vmx->emulation_required = vmx_emulation_required(vcpu); in vmx_set_cr0()
3388 struct kvm *kvm = vcpu->kvm; in vmx_load_mmu_pgd()
3400 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; in vmx_load_mmu_pgd()
3402 guest_cr3 = vcpu->arch.cr3; in vmx_load_mmu_pgd()
3403 else /* vmcs.GUEST_CR3 is already up-to-date. */ in vmx_load_mmu_pgd()
3426 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) in vmx_is_valid_cr4()
3440 * is in force while we are in guest mode. Do not let guests control in vmx_set_cr4()
3446 else if (vmx->rmode.vm86_active) in vmx_set_cr4()
3461 vcpu->arch.cr4 = cr4; in vmx_set_cr4()
3475 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in in vmx_set_cr4()
3477 * to be manually disabled when guest switches to non-paging in vmx_set_cr4()
3478 * mode. in vmx_set_cr4()
3501 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { in vmx_get_segment()
3502 *var = vmx->rmode.segs[seg]; in vmx_get_segment()
3504 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) in vmx_get_segment()
3506 var->base = vmx_read_guest_seg_base(vmx, seg); in vmx_get_segment()
3507 var->selector = vmx_read_guest_seg_selector(vmx, seg); in vmx_get_segment()
3510 var->base = vmx_read_guest_seg_base(vmx, seg); in vmx_get_segment()
3511 var->limit = vmx_read_guest_seg_limit(vmx, seg); in vmx_get_segment()
3512 var->selector = vmx_read_guest_seg_selector(vmx, seg); in vmx_get_segment()
3514 var->unusable = (ar >> 16) & 1; in vmx_get_segment()
3515 var->type = ar & 15; in vmx_get_segment()
3516 var->s = (ar >> 4) & 1; in vmx_get_segment()
3517 var->dpl = (ar >> 5) & 3; in vmx_get_segment()
3525 var->present = !var->unusable; in vmx_get_segment()
3526 var->avl = (ar >> 12) & 1; in vmx_get_segment()
3527 var->l = (ar >> 13) & 1; in vmx_get_segment()
3528 var->db = (ar >> 14) & 1; in vmx_get_segment()
3529 var->g = (ar >> 15) & 1; in vmx_get_segment()
3536 if (to_vmx(vcpu)->rmode.vm86_active) { in vmx_get_segment_base()
3547 if (unlikely(vmx->rmode.vm86_active)) in vmx_get_cpl()
3559 ar = var->type & 15; in vmx_segment_access_rights()
3560 ar |= (var->s & 1) << 4; in vmx_segment_access_rights()
3561 ar |= (var->dpl & 3) << 5; in vmx_segment_access_rights()
3562 ar |= (var->present & 1) << 7; in vmx_segment_access_rights()
3563 ar |= (var->avl & 1) << 12; in vmx_segment_access_rights()
3564 ar |= (var->l & 1) << 13; in vmx_segment_access_rights()
3565 ar |= (var->db & 1) << 14; in vmx_segment_access_rights()
3566 ar |= (var->g & 1) << 15; in vmx_segment_access_rights()
3567 ar |= (var->unusable || !var->present) << 16; in vmx_segment_access_rights()
3579 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { in __vmx_set_segment()
3580 vmx->rmode.segs[seg] = *var; in __vmx_set_segment()
3582 vmcs_write16(sf->selector, var->selector); in __vmx_set_segment()
3583 else if (var->s) in __vmx_set_segment()
3584 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); in __vmx_set_segment()
3588 vmcs_writel(sf->base, var->base); in __vmx_set_segment()
3589 vmcs_write32(sf->limit, var->limit); in __vmx_set_segment()
3590 vmcs_write16(sf->selector, var->selector); in __vmx_set_segment()
3598 * state vmexit when "unrestricted guest" mode is turned on. in __vmx_set_segment()
3604 var->type |= 0x1; /* Accessed */ in __vmx_set_segment()
3606 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); in __vmx_set_segment()
3613 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); in vmx_set_segment()
3626 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); in vmx_get_idt()
3627 dt->address = vmcs_readl(GUEST_IDTR_BASE); in vmx_get_idt()
3632 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); in vmx_set_idt()
3633 vmcs_writel(GUEST_IDTR_BASE, dt->address); in vmx_set_idt()
3638 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); in vmx_get_gdt()
3639 dt->address = vmcs_readl(GUEST_GDTR_BASE); in vmx_get_gdt()
3644 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); in vmx_set_gdt()
3645 vmcs_writel(GUEST_GDTR_BASE, dt->address); in vmx_set_gdt()
3754 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ in tr_valid()
3798 /* real mode guest state checks */ in __vmx_guest_state_valid()
3813 /* protected mode guest state checks */ in __vmx_guest_state_valid()
3834 * - Add checks on RIP in __vmx_guest_state_valid()
3835 * - Add checks on RFLAGS in __vmx_guest_state_valid()
3849 return -EFAULT; in init_rmode_tss()
3854 return -EFAULT; in init_rmode_tss()
3857 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) in init_rmode_tss()
3858 return -EFAULT; in init_rmode_tss()
3870 /* Protect kvm_vmx->ept_identity_pagetable_done. */ in init_rmode_identity_map()
3871 mutex_lock(&kvm->slots_lock); in init_rmode_identity_map()
3873 if (likely(kvm_vmx->ept_identity_pagetable_done)) in init_rmode_identity_map()
3876 if (!kvm_vmx->ept_identity_map_addr) in init_rmode_identity_map()
3877 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; in init_rmode_identity_map()
3881 kvm_vmx->ept_identity_map_addr, in init_rmode_identity_map()
3888 /* Set up identity-mapping pagetable for EPT in real mode */ in init_rmode_identity_map()
3893 r = -EFAULT; in init_rmode_identity_map()
3897 kvm_vmx->ept_identity_pagetable_done = true; in init_rmode_identity_map()
3900 mutex_unlock(&kvm->slots_lock); in init_rmode_identity_map()
3909 vmcs_write16(sf->selector, 0); in seg_setup()
3910 vmcs_writel(sf->base, 0); in seg_setup()
3911 vmcs_write32(sf->limit, 0xffff); in seg_setup()
3916 vmcs_write32(sf->ar_bytes, ar); in seg_setup()
3947 * When KVM is a nested hypervisor on top of Hyper-V and uses in vmx_msr_bitmap_l01_changed()
3952 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; in vmx_msr_bitmap_l01_changed()
3954 if (evmcs->hv_enlightenments_control.msr_bitmap) in vmx_msr_bitmap_l01_changed()
3955 evmcs->hv_clean_fields &= in vmx_msr_bitmap_l01_changed()
3959 vmx->nested.force_msr_bitmap_recalc = true; in vmx_msr_bitmap_l01_changed()
3965 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; in vmx_disable_intercept_for_msr()
3979 if (idx != -ENOENT) { in vmx_disable_intercept_for_msr()
3981 clear_bit(idx, vmx->shadow_msr_intercept.read); in vmx_disable_intercept_for_msr()
3983 clear_bit(idx, vmx->shadow_msr_intercept.write); in vmx_disable_intercept_for_msr()
4009 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; in vmx_enable_intercept_for_msr()
4023 if (idx != -ENOENT) { in vmx_enable_intercept_for_msr()
4025 set_bit(idx, vmx->shadow_msr_intercept.read); in vmx_enable_intercept_for_msr()
4027 set_bit(idx, vmx->shadow_msr_intercept.write); in vmx_enable_intercept_for_msr()
4041 * x2APIC indices for 64-bit accesses into the RDMSR and WRMSR halves in vmx_update_msr_bitmap_x2apic()
4048 u64 *msr_bitmap = (u64 *)vmx->vmcs01.msr_bitmap; in vmx_update_msr_bitmap_x2apic()
4049 u8 mode; in vmx_update_msr_bitmap_x2apic() local
4057 mode = MSR_BITMAP_MODE_X2APIC; in vmx_update_msr_bitmap_x2apic()
4059 mode |= MSR_BITMAP_MODE_X2APIC_APICV; in vmx_update_msr_bitmap_x2apic()
4061 mode = 0; in vmx_update_msr_bitmap_x2apic()
4064 if (mode == vmx->x2apic_msr_bitmap_mode) in vmx_update_msr_bitmap_x2apic()
4067 vmx->x2apic_msr_bitmap_mode = mode; in vmx_update_msr_bitmap_x2apic()
4070 * Reset the bitmap for MSRs 0x800 - 0x83f. Leave AMD's uber-extended in vmx_update_msr_bitmap_x2apic()
4074 * mode, only the current timer count needs on-demand emulation by KVM. in vmx_update_msr_bitmap_x2apic()
4076 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) in vmx_update_msr_bitmap_x2apic()
4077 msr_bitmap[read_idx] = ~kvm_lapic_readable_reg_mask(vcpu->arch.apic); in vmx_update_msr_bitmap_x2apic()
4087 !(mode & MSR_BITMAP_MODE_X2APIC)); in vmx_update_msr_bitmap_x2apic()
4089 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { in vmx_update_msr_bitmap_x2apic()
4101 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); in pt_update_intercept_for_msr()
4108 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { in pt_update_intercept_for_msr()
4123 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn)) in vmx_guest_apic_has_interrupt()
4128 vapic_page = vmx->nested.virtual_apic_map.hva; in vmx_guest_apic_has_interrupt()
4150 if (!test_bit(i, vmx->shadow_msr_intercept.read)) in vmx_msr_filter_changed()
4153 if (!test_bit(i, vmx->shadow_msr_intercept.write)) in vmx_msr_filter_changed()
4166 if (vcpu->mode == IN_GUEST_MODE) { in kvm_vcpu_trigger_posted_interrupt()
4171 * event is being sent from a fastpath VM-Exit handler, in in kvm_vcpu_trigger_posted_interrupt()
4173 * re-entering the guest. in kvm_vcpu_trigger_posted_interrupt()
4178 * Case 1: vCPU stays in non-root mode. Sending a notification in kvm_vcpu_trigger_posted_interrupt()
4181 * Case 2: vCPU exits to root mode and is still runnable. The in kvm_vcpu_trigger_posted_interrupt()
4182 * PIR will be synced to the vIRR before re-entering the guest. in kvm_vcpu_trigger_posted_interrupt()
4186 * Case 3: vCPU exits to root mode and is blocked. vcpu_block() in kvm_vcpu_trigger_posted_interrupt()
4194 __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); in kvm_vcpu_trigger_posted_interrupt()
4201 * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest(). in kvm_vcpu_trigger_posted_interrupt()
4212 vector == vmx->nested.posted_intr_nv) { in vmx_deliver_nested_posted_interrupt()
4217 vmx->nested.pi_pending = true; in vmx_deliver_nested_posted_interrupt()
4221 * This pairs with the smp_mb_*() after setting vcpu->mode in in vmx_deliver_nested_posted_interrupt()
4224 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as in vmx_deliver_nested_posted_interrupt()
4226 * done before making the request is visible when the request in vmx_deliver_nested_posted_interrupt()
4227 * is visible, it doesn't ensure ordering between the store to in vmx_deliver_nested_posted_interrupt()
4228 * vcpu->requests and the load from vcpu->mode. in vmx_deliver_nested_posted_interrupt()
4236 return -1; in vmx_deliver_nested_posted_interrupt()
4240 * 1. If target vcpu is running(non-root mode), send posted interrupt
4242 * 2. If target vcpu isn't running(root mode), kick it to pick up the
4254 /* Note, this is called iff the local APIC is in-kernel. */ in vmx_deliver_posted_interrupt()
4255 if (!vcpu->arch.apic->apicv_active) in vmx_deliver_posted_interrupt()
4256 return -1; in vmx_deliver_posted_interrupt()
4258 if (pi_test_and_set_pir(vector, &vmx->pi_desc)) in vmx_deliver_posted_interrupt()
4262 if (pi_test_and_set_on(&vmx->pi_desc)) in vmx_deliver_posted_interrupt()
4267 * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is in vmx_deliver_posted_interrupt()
4269 * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE. in vmx_deliver_posted_interrupt()
4278 struct kvm_vcpu *vcpu = apic->vcpu; in vmx_deliver_interrupt()
4285 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, in vmx_deliver_interrupt()
4291 * Set up the vmcs's constant host-state fields, i.e., host-state fields that
4293 * Note that host-state that does change is set elsewhere. E.g., host-state
4312 vmx->loaded_vmcs->host_state.cr3 = cr3; in vmx_set_constant_host_state()
4317 vmx->loaded_vmcs->host_state.cr4 = cr4; in vmx_set_constant_host_state()
4343 * SYSENTER is used for 32-bit system calls on either 32-bit or in vmx_set_constant_host_state()
4344 * 64-bit kernels. It is always zero If neither is allowed, otherwise in vmx_set_constant_host_state()
4345 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may in vmx_set_constant_host_state()
4365 struct kvm_vcpu *vcpu = &vmx->vcpu; in set_cr4_guest_host_mask()
4367 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & in set_cr4_guest_host_mask()
4368 ~vcpu->arch.cr4_guest_rsvd_bits; in set_cr4_guest_host_mask()
4370 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; in set_cr4_guest_host_mask()
4371 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; in set_cr4_guest_host_mask()
4373 if (is_guest_mode(&vmx->vcpu)) in set_cr4_guest_host_mask()
4374 vcpu->arch.cr4_guest_owned_bits &= in set_cr4_guest_host_mask()
4375 ~get_vmcs12(vcpu)->cr4_guest_host_mask; in set_cr4_guest_host_mask()
4376 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); in set_cr4_guest_host_mask()
4383 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) in vmx_pin_based_exec_ctrl()
4403 * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. in vmx_vmentry_ctrl()
4443 vmx->nested.update_vmcs01_apicv_status = true; in vmx_refresh_apicv_exec_ctrl()
4483 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) in vmx_exec_control()
4486 if (!cpu_need_tpr_shadow(&vmx->vcpu)) in vmx_exec_control()
4502 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) in vmx_exec_control()
4505 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) in vmx_exec_control()
4518 if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) in vmx_tertiary_exec_control()
4534 * If the control is for an opt-in feature, clear the control if the in vmx_adjust_secondary_exec_control()
4536 * control is opt-out, i.e. an exiting control, clear the control if in vmx_adjust_secondary_exec_control()
4557 vmx->nested.msrs.secondary_ctls_high |= control; in vmx_adjust_secondary_exec_control()
4559 vmx->nested.msrs.secondary_ctls_high &= ~control; in vmx_adjust_secondary_exec_control()
4570 struct kvm_vcpu *__vcpu = &(vmx)->vcpu; \
4583 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */
4592 struct kvm_vcpu *vcpu = &vmx->vcpu; in vmx_secondary_exec_control()
4600 if (vmx->vpid == 0) in vmx_secondary_exec_control()
4608 if (kvm_pause_in_guest(vmx->vcpu.kvm)) in vmx_secondary_exec_control()
4637 if (!enable_pml || !atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) in vmx_secondary_exec_control()
4668 if (!vcpu->kvm->arch.bus_lock_detection_enabled) in vmx_secondary_exec_control()
4671 if (!kvm_notify_vmexit_enabled(vcpu->kvm)) in vmx_secondary_exec_control()
4679 return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table)); in vmx_get_pid_table_order()
4690 if (kvm_vmx->pid_table) in vmx_alloc_ipiv_pid_table()
4696 return -ENOMEM; in vmx_alloc_ipiv_pid_table()
4698 kvm_vmx->pid_table = (void *)page_address(pages); in vmx_alloc_ipiv_pid_table()
4711 struct kvm *kvm = vmx->vcpu.kvm; in init_vmcs()
4718 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); in init_vmcs()
4733 if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { in init_vmcs()
4742 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc))); in init_vmcs()
4745 if (vmx_can_use_ipiv(&vmx->vcpu)) { in init_vmcs()
4746 vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); in init_vmcs()
4747 vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); in init_vmcs()
4752 vmx->ple_window = ple_window; in init_vmcs()
4753 vmx->ple_window_dirty = true; in init_vmcs()
4757 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); in init_vmcs()
4774 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); in init_vmcs()
4776 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); in init_vmcs()
4779 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); in init_vmcs()
4786 vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); in init_vmcs()
4787 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); in init_vmcs()
4791 if (vmx->vpid != 0) in init_vmcs()
4792 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); in init_vmcs()
4798 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); in init_vmcs()
4799 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); in init_vmcs()
4802 vmx_write_encls_bitmap(&vmx->vcpu, NULL); in init_vmcs()
4805 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); in init_vmcs()
4807 vmx->pt_desc.guest.output_mask = 0x7F; in init_vmcs()
4818 if (cpu_need_tpr_shadow(&vmx->vcpu)) in init_vmcs()
4820 __pa(vmx->vcpu.arch.apic->regs)); in init_vmcs()
4834 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); in __vmx_vcpu_reset()
4838 vmx->nested.posted_intr_nv = -1; in __vmx_vcpu_reset()
4839 vmx->nested.vmxon_ptr = INVALID_GPA; in __vmx_vcpu_reset()
4840 vmx->nested.current_vmptr = INVALID_GPA; in __vmx_vcpu_reset()
4843 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; in __vmx_vcpu_reset()
4846 vcpu->arch.microcode_version = 0x100000000ULL; in __vmx_vcpu_reset()
4847 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; in __vmx_vcpu_reset()
4853 vmx->pi_desc.nv = POSTED_INTR_VECTOR; in __vmx_vcpu_reset()
4854 vmx->pi_desc.sn = 1; in __vmx_vcpu_reset()
4864 vmx->rmode.vm86_active = 0; in vmx_vcpu_reset()
4865 vmx->spec_ctrl = 0; in vmx_vcpu_reset()
4867 vmx->msr_ia32_umwait_control = 0; in vmx_vcpu_reset()
4869 vmx->hv_deadline_tsc = -1; in vmx_vcpu_reset()
4911 vpid_sync_context(vmx->vpid); in vmx_vcpu_reset()
4936 int irq = vcpu->arch.interrupt.nr; in vmx_inject_irq()
4938 trace_kvm_inj_virq(irq, vcpu->arch.interrupt.soft, reinjected); in vmx_inject_irq()
4940 ++vcpu->stat.irq_injections; in vmx_inject_irq()
4941 if (vmx->rmode.vm86_active) { in vmx_inject_irq()
4943 if (vcpu->arch.interrupt.soft) in vmx_inject_irq()
4944 inc_eip = vcpu->arch.event_exit_inst_len; in vmx_inject_irq()
4949 if (vcpu->arch.interrupt.soft) { in vmx_inject_irq()
4952 vmx->vcpu.arch.event_exit_inst_len); in vmx_inject_irq()
4966 * Tracking the NMI-blocked state in software is built upon in vmx_inject_nmi()
4968 * well-behaving guests: They have to keep IRQs disabled at in vmx_inject_nmi()
4973 vmx->loaded_vmcs->soft_vnmi_blocked = 1; in vmx_inject_nmi()
4974 vmx->loaded_vmcs->vnmi_blocked_time = 0; in vmx_inject_nmi()
4977 ++vcpu->stat.nmi_injections; in vmx_inject_nmi()
4978 vmx->loaded_vmcs->nmi_known_unmasked = false; in vmx_inject_nmi()
4980 if (vmx->rmode.vm86_active) { in vmx_inject_nmi()
4997 return vmx->loaded_vmcs->soft_vnmi_blocked; in vmx_get_nmi_mask()
4998 if (vmx->loaded_vmcs->nmi_known_unmasked) in vmx_get_nmi_mask()
5001 vmx->loaded_vmcs->nmi_known_unmasked = !masked; in vmx_get_nmi_mask()
5010 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { in vmx_set_nmi_mask()
5011 vmx->loaded_vmcs->soft_vnmi_blocked = masked; in vmx_set_nmi_mask()
5012 vmx->loaded_vmcs->vnmi_blocked_time = 0; in vmx_set_nmi_mask()
5015 vmx->loaded_vmcs->nmi_known_unmasked = !masked; in vmx_set_nmi_mask()
5030 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) in vmx_nmi_blocked()
5040 if (to_vmx(vcpu)->nested.nested_run_pending) in vmx_nmi_allowed()
5041 return -EBUSY; in vmx_nmi_allowed()
5043 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ in vmx_nmi_allowed()
5045 return -EBUSY; in vmx_nmi_allowed()
5062 if (to_vmx(vcpu)->nested.nested_run_pending) in vmx_interrupt_allowed()
5063 return -EBUSY; in vmx_interrupt_allowed()
5066 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, in vmx_interrupt_allowed()
5070 return -EBUSY; in vmx_interrupt_allowed()
5082 mutex_lock(&kvm->slots_lock); in vmx_set_tss_addr()
5085 mutex_unlock(&kvm->slots_lock); in vmx_set_tss_addr()
5090 to_kvm_vmx(kvm)->tss_addr = addr; in vmx_set_tss_addr()
5097 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; in vmx_set_identity_map_addr()
5107 * from user space while in guest debugging mode. in rmode_exception()
5109 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = in rmode_exception()
5111 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) in rmode_exception()
5115 return !(vcpu->guest_debug & in rmode_exception()
5135 * Cause the #SS fault with 0 error code in VM86 mode. in handle_rmode_exception()
5139 if (vcpu->arch.halt_request) { in handle_rmode_exception()
5140 vcpu->arch.halt_request = 0; in handle_rmode_exception()
5149 * Forward all other exceptions that are valid in real mode. in handle_rmode_exception()
5150 * FIXME: Breaks guest debugging in real mode, needs to be fixed with in handle_rmode_exception()
5170 * - Guest CPL == 3 (user mode)
5171 * - Guest has #AC detection enabled in CR0
5172 * - Guest EFLAGS has AC bit set
5186 struct kvm_run *kvm_run = vcpu->run; in handle_exception_nmi()
5191 vect_info = vmx->idt_vectoring_info; in handle_exception_nmi()
5196 * vmx_vcpu_run() if a #MC occurs on VM-Entry. NMIs are handled by in handle_exception_nmi()
5220 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { in handle_exception_nmi()
5225 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero in handle_exception_nmi()
5242 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in handle_exception_nmi()
5243 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; in handle_exception_nmi()
5244 vcpu->run->internal.ndata = 4; in handle_exception_nmi()
5245 vcpu->run->internal.data[0] = vect_info; in handle_exception_nmi()
5246 vcpu->run->internal.data[1] = intr_info; in handle_exception_nmi()
5247 vcpu->run->internal.data[2] = error_code; in handle_exception_nmi()
5248 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; in handle_exception_nmi()
5254 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { in handle_exception_nmi()
5268 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) in handle_exception_nmi()
5274 if (!(vcpu->guest_debug & in handle_exception_nmi()
5278 * instruction. ICEBP generates a trap-like #DB, but in handle_exception_nmi()
5280 * is an instruction intercept, i.e. the VM-Exit occurs in handle_exception_nmi()
5282 * avoid single-step #DB and MTF updates, as ICEBP is in handle_exception_nmi()
5287 * if single-step is enabled in RFLAGS and STI or MOVSS in handle_exception_nmi()
5289 * on VM-Exit due to #DB interception. VM-Entry has a in handle_exception_nmi()
5290 * consistency check that a single-step #DB is pending in handle_exception_nmi()
5294 * delay when activating single-step breakpoints must in handle_exception_nmi()
5296 * as appropriate for all other VM-Exits types. in handle_exception_nmi()
5309 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; in handle_exception_nmi()
5310 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); in handle_exception_nmi()
5315 * user space while in guest debugging mode. Reading it for in handle_exception_nmi()
5318 vmx->vcpu.arch.event_exit_inst_len = in handle_exception_nmi()
5320 kvm_run->exit_reason = KVM_EXIT_DEBUG; in handle_exception_nmi()
5321 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); in handle_exception_nmi()
5322 kvm_run->debug.arch.exception = ex_no; in handle_exception_nmi()
5331 * Handle split lock. Depending on detection mode this will in handle_exception_nmi()
5339 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; in handle_exception_nmi()
5340 kvm_run->ex.exception = ex_no; in handle_exception_nmi()
5341 kvm_run->ex.error_code = error_code; in handle_exception_nmi()
5349 ++vcpu->stat.irq_exits; in handle_external_interrupt()
5355 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; in handle_triple_fault()
5356 vcpu->mmio_needed = 0; in handle_triple_fault()
5369 ++vcpu->stat.io_exits; in handle_io()
5392 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
5404 * hardware. It consists of the L2-owned bits from the new in handle_set_cr0()
5405 * value combined with the L1-owned bits from L1's guest_cr0. in handle_set_cr0()
5407 val = (val & ~vmcs12->cr0_guest_host_mask) | in handle_set_cr0()
5408 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); in handle_set_cr0()
5426 val = (val & ~vmcs12->cr4_guest_host_mask) | in handle_set_cr4()
5427 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); in handle_set_cr4()
5486 * KVM_GUESTDBG_SINGLESTEP-triggered in handle_cr()
5489 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; in handle_cr()
5495 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); in handle_cr()
5496 return -EIO; in handle_cr()
5522 vcpu->run->exit_reason = 0; in handle_cr()
5547 * As the vm-exit takes precedence over the debug trap, we in handle_dr()
5551 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { in handle_dr()
5552 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; in handle_dr()
5553 vcpu->run->debug.arch.dr7 = dr7; in handle_dr()
5554 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); in handle_dr()
5555 vcpu->run->debug.arch.exception = DB_VECTOR; in handle_dr()
5556 vcpu->run->exit_reason = KVM_EXIT_DEBUG; in handle_dr()
5564 if (vcpu->guest_debug == 0) { in handle_dr()
5572 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; in handle_dr()
5593 get_debugreg(vcpu->arch.db[0], 0); in vmx_sync_dirty_debug_regs()
5594 get_debugreg(vcpu->arch.db[1], 1); in vmx_sync_dirty_debug_regs()
5595 get_debugreg(vcpu->arch.db[2], 2); in vmx_sync_dirty_debug_regs()
5596 get_debugreg(vcpu->arch.db[3], 3); in vmx_sync_dirty_debug_regs()
5597 get_debugreg(vcpu->arch.dr6, 6); in vmx_sync_dirty_debug_regs()
5598 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); in vmx_sync_dirty_debug_regs()
5600 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; in vmx_sync_dirty_debug_regs()
5627 ++vcpu->stat.irq_window_exits; in handle_interrupt_window()
5649 * not cared. So make a short-circuit here by avoiding in handle_apic_access()
5666 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ in handle_apic_eoi_induced()
5676 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and in handle_apic_write()
5679 * the vAPIC page for the correct 16-byte chunk. KVM needs only to in handle_apic_write()
5697 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); in handle_task_switch()
5698 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); in handle_task_switch()
5699 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); in handle_task_switch()
5707 vcpu->arch.nmi_injected = false; in handle_task_switch()
5715 if (vmx->idt_vectoring_info & in handle_task_switch()
5741 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, in handle_task_switch()
5759 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && in handle_ept_violation()
5783 vcpu->arch.exit_qualification = exit_qualification; in handle_ept_violation()
5790 * would also use advanced VM-exit information for EPT violations to in handle_ept_violation()
5822 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) in handle_nmi_window()
5823 return -EIO; in handle_nmi_window()
5826 ++vcpu->stat.nmi_window_exits; in handle_nmi_window()
5836 return vmx->emulation_required && !vmx->rmode.vm86_active && in vmx_emulation_required_with_pending_exception()
5837 (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); in vmx_emulation_required_with_pending_exception()
5849 while (vmx->emulation_required && count-- != 0) { in handle_invalid_guest_state()
5851 return handle_interrupt_window(&vmx->vcpu); in handle_invalid_guest_state()
5864 if (vcpu->arch.halt_request) { in handle_invalid_guest_state()
5865 vcpu->arch.halt_request = 0; in handle_invalid_guest_state()
5894 unsigned int old = vmx->ple_window; in grow_ple_window()
5896 vmx->ple_window = __grow_ple_window(old, ple_window, in grow_ple_window()
5900 if (vmx->ple_window != old) { in grow_ple_window()
5901 vmx->ple_window_dirty = true; in grow_ple_window()
5902 trace_kvm_ple_window_update(vcpu->vcpu_id, in grow_ple_window()
5903 vmx->ple_window, old); in grow_ple_window()
5910 unsigned int old = vmx->ple_window; in shrink_ple_window()
5912 vmx->ple_window = __shrink_ple_window(old, ple_window, in shrink_ple_window()
5916 if (vmx->ple_window != old) { in shrink_ple_window()
5917 vmx->ple_window_dirty = true; in shrink_ple_window()
5918 trace_kvm_ple_window_update(vcpu->vcpu_id, in shrink_ple_window()
5919 vmx->ple_window, old); in shrink_ple_window()
5924 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
5925 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
5929 if (!kvm_pause_in_guest(vcpu->kvm)) in handle_pause()
5933 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" in handle_pause()
5934 * VM-execution control is ignored if CPL > 0. OTOH, KVM in handle_pause()
5982 trace_kvm_pml_full(vcpu->vcpu_id); in handle_pml_full()
5990 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && in handle_pml_full()
6007 if (!vmx->req_immediate_exit && in handle_fastpath_preemption_timer()
6008 !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) { in handle_fastpath_preemption_timer()
6049 * VM-Exits. Unconditionally set the flag here and leave the handling to in handle_bus_lock_vmexit()
6052 to_vmx(vcpu)->exit_reason.bus_lock_detected = true; in handle_bus_lock_vmexit()
6061 ++vcpu->stat.notify_window_exits; in handle_notify()
6071 if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || in handle_notify()
6073 vcpu->run->exit_reason = KVM_EXIT_NOTIFY; in handle_notify()
6074 vcpu->run->notify.flags = context_invalid ? in handle_notify()
6151 *reason = vmx->exit_reason.full; in vmx_get_exit_info()
6153 if (!(vmx->exit_reason.failed_vmentry)) { in vmx_get_exit_info()
6154 *info2 = vmx->idt_vectoring_info; in vmx_get_exit_info()
6169 if (vmx->pml_pg) { in vmx_destroy_pml_buffer()
6170 __free_page(vmx->pml_pg); in vmx_destroy_pml_buffer()
6171 vmx->pml_pg = NULL; in vmx_destroy_pml_buffer()
6184 if (pml_idx == (PML_ENTITY_NUM - 1)) in vmx_flush_pml_buffer()
6193 pml_buf = page_address(vmx->pml_pg); in vmx_flush_pml_buffer()
6198 WARN_ON(gpa & (PAGE_SIZE - 1)); in vmx_flush_pml_buffer()
6203 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); in vmx_flush_pml_buffer()
6210 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), in vmx_dump_sel()
6211 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), in vmx_dump_sel()
6212 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); in vmx_dump_sel()
6219 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); in vmx_dump_dtsel()
6228 for (i = 0, e = m->val; i < m->nr; ++i, ++e) in vmx_dump_msrs()
6229 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); in vmx_dump_msrs()
6262 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", in dump_vmcs()
6263 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); in dump_vmcs()
6294 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); in dump_vmcs()
6299 vmx->msr_autoload.guest.val[efer_slot].value); in dump_vmcs()
6302 vcpu->arch.efer | (EFER_LMA | EFER_LME)); in dump_vmcs()
6305 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); in dump_vmcs()
6324 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); in dump_vmcs()
6326 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest); in dump_vmcs()
6357 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); in dump_vmcs()
6392 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); in dump_vmcs()
6393 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); in dump_vmcs()
6414 union vmx_exit_reason exit_reason = vmx->exit_reason; in __vmx_handle_exit()
6415 u32 vectoring_info = vmx->idt_vectoring_info; in __vmx_handle_exit()
6422 * mode as if vcpus is in root mode, the PML buffer must has been in __vmx_handle_exit()
6430 * KVM should never reach this point with a pending nested VM-Enter. in __vmx_handle_exit()
6431 * More specifically, short-circuiting VM-Entry to emulate L2 due to in __vmx_handle_exit()
6433 * allowed a nested VM-Enter with an invalid vmcs12. More below. in __vmx_handle_exit()
6435 if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm)) in __vmx_handle_exit()
6436 return -EIO; in __vmx_handle_exit()
6451 * address-translation-based dirty tracking (e.g. EPT write in __vmx_handle_exit()
6461 * operation, nested VM-Enter rejects any attempt to enter L2 in __vmx_handle_exit()
6470 if (vmx->emulation_required) { in __vmx_handle_exit()
6480 if (vmx->emulation_required) in __vmx_handle_exit()
6485 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; in __vmx_handle_exit()
6486 vcpu->run->fail_entry.hardware_entry_failure_reason in __vmx_handle_exit()
6488 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; in __vmx_handle_exit()
6492 if (unlikely(vmx->fail)) { in __vmx_handle_exit()
6494 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; in __vmx_handle_exit()
6495 vcpu->run->fail_entry.hardware_entry_failure_reason in __vmx_handle_exit()
6497 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; in __vmx_handle_exit()
6505 * The vm-exit can be triggered again after return to guest that in __vmx_handle_exit()
6517 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in __vmx_handle_exit()
6518 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; in __vmx_handle_exit()
6519 vcpu->run->internal.data[0] = vectoring_info; in __vmx_handle_exit()
6520 vcpu->run->internal.data[1] = exit_reason.full; in __vmx_handle_exit()
6521 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification; in __vmx_handle_exit()
6523 vcpu->run->internal.data[ndata++] = in __vmx_handle_exit()
6526 vcpu->run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu; in __vmx_handle_exit()
6527 vcpu->run->internal.ndata = ndata; in __vmx_handle_exit()
6532 vmx->loaded_vmcs->soft_vnmi_blocked)) { in __vmx_handle_exit()
6534 vmx->loaded_vmcs->soft_vnmi_blocked = 0; in __vmx_handle_exit()
6535 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && in __vmx_handle_exit()
6536 vcpu->arch.nmi_pending) { in __vmx_handle_exit()
6539 * NMI-blocked window if the guest runs with IRQs in __vmx_handle_exit()
6543 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " in __vmx_handle_exit()
6545 __func__, vcpu->vcpu_id); in __vmx_handle_exit()
6546 vmx->loaded_vmcs->soft_vnmi_blocked = 0; in __vmx_handle_exit()
6581 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in __vmx_handle_exit()
6582 vcpu->run->internal.suberror = in __vmx_handle_exit()
6584 vcpu->run->internal.ndata = 2; in __vmx_handle_exit()
6585 vcpu->run->internal.data[0] = exit_reason.full; in __vmx_handle_exit()
6586 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; in __vmx_handle_exit()
6598 if (to_vmx(vcpu)->exit_reason.bus_lock_detected) { in vmx_handle_exit()
6600 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; in vmx_handle_exit()
6602 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; in vmx_handle_exit()
6623 * This code is only executed when the flush mode is 'cond' or in vmx_l1d_flush()
6630 * Clear the per-vcpu flush bit, it gets set again in vmx_l1d_flush()
6634 flush_l1d = vcpu->arch.l1tf_flush_l1d; in vmx_l1d_flush()
6635 vcpu->arch.l1tf_flush_l1d = false; in vmx_l1d_flush()
6638 * Clear the per-cpu flush bit, it gets set again from in vmx_l1d_flush()
6648 vcpu->stat.l1d_flush++; in vmx_l1d_flush()
6687 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; in vmx_update_cr8_intercept()
6689 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; in vmx_update_cr8_intercept()
6708 vmx->nested.change_vmcs01_virtual_apic_mode = true; in vmx_set_virtual_apic_mode()
6731 * the guest may have inserted a non-APIC mapping into in vmx_set_virtual_apic_mode()
6751 struct kvm *kvm = vcpu->kvm; in vmx_set_apic_access_page_addr()
6759 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; in vmx_set_apic_access_page_addr()
6774 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) in vmx_set_apic_access_page_addr()
6783 mmu_seq = kvm->mmu_invalidate_seq; in vmx_set_apic_access_page_addr()
6788 * controls the APIC-access page memslot, and only deletes the memslot in vmx_set_apic_access_page_addr()
6795 read_lock(&vcpu->kvm->mmu_lock); in vmx_set_apic_access_page_addr()
6798 read_unlock(&vcpu->kvm->mmu_lock); in vmx_set_apic_access_page_addr()
6803 read_unlock(&vcpu->kvm->mmu_lock); in vmx_set_apic_access_page_addr()
6822 if (max_isr == -1) in vmx_hwapic_isr_update()
6839 if (vector == -1) in vmx_set_rvi()
6855 * vmcs12 virtual-interrupt-delivery enabled. in vmx_hwapic_irr_update()
6857 * intercepts external-interrupts and in that case in vmx_hwapic_irr_update()
6871 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) in vmx_sync_pir_to_irr()
6872 return -EIO; in vmx_sync_pir_to_irr()
6874 if (pi_test_on(&vmx->pi_desc)) { in vmx_sync_pir_to_irr()
6875 pi_clear_on(&vmx->pi_desc); in vmx_sync_pir_to_irr()
6882 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr); in vmx_sync_pir_to_irr()
6895 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected in vmx_sync_pir_to_irr()
6901 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. in vmx_sync_pir_to_irr()
6926 pi_clear_on(&vmx->pi_desc); in vmx_apicv_pre_state_restore()
6927 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir)); in vmx_apicv_pre_state_restore()
6945 * only when xfd contains a non-zero value. in handle_nm_fault_irqoff()
6949 if (vcpu->arch.guest_fpu.fpstate->xfd) in handle_nm_fault_irqoff()
6950 rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); in handle_nm_fault_irqoff()
6955 u32 intr_info = vmx_get_intr_info(&vmx->vcpu); in handle_exception_irqoff()
6959 vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); in handle_exception_irqoff()
6962 handle_nm_fault_irqoff(&vmx->vcpu); in handle_exception_irqoff()
6974 if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, in handle_external_interrupt_irqoff()
6975 "unexpected VM-Exit interrupt info: 0x%x", intr_info)) in handle_external_interrupt_irqoff()
6982 vcpu->arch.at_instruction_boundary = true; in handle_external_interrupt_irqoff()
6989 if (vmx->emulation_required) in vmx_handle_exit_irqoff()
6992 if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) in vmx_handle_exit_irqoff()
6994 else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI) in vmx_handle_exit_irqoff()
7010 * real mode. in vmx_has_emulated_msr()
7031 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; in vmx_recover_nmi_blocking()
7034 if (vmx->loaded_vmcs->nmi_known_unmasked) in vmx_recover_nmi_blocking()
7037 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); in vmx_recover_nmi_blocking()
7042 * Re-set bit "block by NMI" before VM entry if vmexit caused by in vmx_recover_nmi_blocking()
7046 * If the VM exit sets the valid bit in the IDT-vectoring in vmx_recover_nmi_blocking()
7055 vmx->loaded_vmcs->nmi_known_unmasked = in vmx_recover_nmi_blocking()
7058 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) in vmx_recover_nmi_blocking()
7059 vmx->loaded_vmcs->vnmi_blocked_time += in vmx_recover_nmi_blocking()
7061 vmx->loaded_vmcs->entry_time)); in vmx_recover_nmi_blocking()
7075 vcpu->arch.nmi_injected = false; in __vmx_complete_interrupts()
7089 vcpu->arch.nmi_injected = true; in __vmx_complete_interrupts()
7098 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); in __vmx_complete_interrupts()
7108 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); in __vmx_complete_interrupts()
7120 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, in vmx_complete_interrupts()
7139 struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu); in atomic_switch_perf_msrs()
7141 pmu->host_cross_mapped_mask = 0; in atomic_switch_perf_msrs()
7142 if (pmu->pebs_enable & pmu->global_ctrl) in atomic_switch_perf_msrs()
7164 if (vmx->req_immediate_exit) { in vmx_update_hv_timer()
7166 vmx->loaded_vmcs->hv_timer_soft_disabled = false; in vmx_update_hv_timer()
7167 } else if (vmx->hv_deadline_tsc != -1) { in vmx_update_hv_timer()
7169 if (vmx->hv_deadline_tsc > tscl) in vmx_update_hv_timer()
7170 /* set_hv_timer ensures the delta fits in 32-bits */ in vmx_update_hv_timer()
7171 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> in vmx_update_hv_timer()
7177 vmx->loaded_vmcs->hv_timer_soft_disabled = false; in vmx_update_hv_timer()
7178 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { in vmx_update_hv_timer()
7179 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); in vmx_update_hv_timer()
7180 vmx->loaded_vmcs->hv_timer_soft_disabled = true; in vmx_update_hv_timer()
7186 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { in vmx_update_host_rsp()
7187 vmx->loaded_vmcs->host_state.rsp = host_rsp; in vmx_update_host_rsp()
7201 vmx->spec_ctrl = __rdmsr(MSR_IA32_SPEC_CTRL); in vmx_spec_ctrl_restore_host()
7207 * transitioning from a less privileged predictor mode, regardless of in vmx_spec_ctrl_restore_host()
7211 vmx->spec_ctrl != hostval) in vmx_spec_ctrl_restore_host()
7219 switch (to_vmx(vcpu)->exit_reason.basic) { in vmx_exit_handlers_fastpath()
7245 kvm_arch_has_assigned_device(vcpu->kvm)) in vmx_vcpu_enter_exit()
7250 if (vcpu->arch.cr2 != native_read_cr2()) in vmx_vcpu_enter_exit()
7251 native_write_cr2(vcpu->arch.cr2); in vmx_vcpu_enter_exit()
7253 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, in vmx_vcpu_enter_exit()
7256 vcpu->arch.cr2 = native_read_cr2(); in vmx_vcpu_enter_exit()
7257 vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; in vmx_vcpu_enter_exit()
7259 vmx->idt_vectoring_info = 0; in vmx_vcpu_enter_exit()
7263 if (unlikely(vmx->fail)) { in vmx_vcpu_enter_exit()
7264 vmx->exit_reason.full = 0xdead; in vmx_vcpu_enter_exit()
7268 vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON); in vmx_vcpu_enter_exit()
7269 if (likely(!vmx->exit_reason.failed_vmentry)) in vmx_vcpu_enter_exit()
7270 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); in vmx_vcpu_enter_exit()
7272 if ((u16)vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI && in vmx_vcpu_enter_exit()
7290 vmx->loaded_vmcs->soft_vnmi_blocked)) in vmx_vcpu_run()
7291 vmx->loaded_vmcs->entry_time = ktime_get(); in vmx_vcpu_run()
7296 * consistency check VM-Exit due to invalid guest state and bail. in vmx_vcpu_run()
7298 if (unlikely(vmx->emulation_required)) { in vmx_vcpu_run()
7299 vmx->fail = 0; in vmx_vcpu_run()
7301 vmx->exit_reason.full = EXIT_REASON_INVALID_STATE; in vmx_vcpu_run()
7302 vmx->exit_reason.failed_vmentry = 1; in vmx_vcpu_run()
7304 vmx->exit_qualification = ENTRY_FAIL_DEFAULT; in vmx_vcpu_run()
7306 vmx->exit_intr_info = 0; in vmx_vcpu_run()
7312 if (vmx->ple_window_dirty) { in vmx_vcpu_run()
7313 vmx->ple_window_dirty = false; in vmx_vcpu_run()
7314 vmcs_write32(PLE_WINDOW, vmx->ple_window); in vmx_vcpu_run()
7321 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); in vmx_vcpu_run()
7324 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); in vmx_vcpu_run()
7326 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); in vmx_vcpu_run()
7327 vcpu->arch.regs_dirty = 0; in vmx_vcpu_run()
7331 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time in vmx_vcpu_run()
7332 * it switches back to the current->mm, which can occur in KVM context in vmx_vcpu_run()
7334 * toggles a static key while handling a VM-Exit. in vmx_vcpu_run()
7337 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { in vmx_vcpu_run()
7339 vmx->loaded_vmcs->host_state.cr3 = cr3; in vmx_vcpu_run()
7343 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { in vmx_vcpu_run()
7345 vmx->loaded_vmcs->host_state.cr4 = cr4; in vmx_vcpu_run()
7349 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) in vmx_vcpu_run()
7350 set_debugreg(vcpu->arch.dr6, 6); in vmx_vcpu_run()
7352 /* When single-stepping over STI and MOV SS, we must clear the in vmx_vcpu_run()
7357 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) in vmx_vcpu_run()
7378 current_evmcs->hv_clean_fields |= in vmx_vcpu_run()
7381 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); in vmx_vcpu_run()
7385 if (vmx->host_debugctlmsr) in vmx_vcpu_run()
7386 update_debugctlmsr(vmx->host_debugctlmsr); in vmx_vcpu_run()
7410 if (vmx->nested.nested_run_pending && in vmx_vcpu_run()
7411 !vmx->exit_reason.failed_vmentry) in vmx_vcpu_run()
7412 ++vcpu->stat.nested_run; in vmx_vcpu_run()
7414 vmx->nested.nested_run_pending = 0; in vmx_vcpu_run()
7417 if (unlikely(vmx->fail)) in vmx_vcpu_run()
7420 if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) in vmx_vcpu_run()
7425 if (unlikely(vmx->exit_reason.failed_vmentry)) in vmx_vcpu_run()
7428 vmx->loaded_vmcs->launched = 1; in vmx_vcpu_run()
7445 free_vpid(vmx->vpid); in vmx_vcpu_free()
7447 free_loaded_vmcs(vmx->loaded_vmcs); in vmx_vcpu_free()
7459 INIT_LIST_HEAD(&vmx->pi_wakeup_list); in vmx_vcpu_create()
7461 err = -ENOMEM; in vmx_vcpu_create()
7463 vmx->vpid = allocate_vpid(); in vmx_vcpu_create()
7472 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); in vmx_vcpu_create()
7473 if (!vmx->pml_pg) in vmx_vcpu_create()
7478 vmx->guest_uret_msrs[i].mask = -1ull; in vmx_vcpu_create()
7487 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; in vmx_vcpu_create()
7490 err = alloc_loaded_vmcs(&vmx->vmcs01); in vmx_vcpu_create()
7495 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a in vmx_vcpu_create()
7496 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the in vmx_vcpu_create()
7502 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; in vmx_vcpu_create()
7504 evmcs->hv_enlightenments_control.msr_bitmap = 1; in vmx_vcpu_create()
7508 bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS); in vmx_vcpu_create()
7509 bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); in vmx_vcpu_create()
7520 if (kvm_cstate_in_guest(vcpu->kvm)) { in vmx_vcpu_create()
7527 vmx->loaded_vmcs = &vmx->vmcs01; in vmx_vcpu_create()
7530 err = kvm_alloc_apic_access_page(vcpu->kvm); in vmx_vcpu_create()
7536 err = init_rmode_identity_map(vcpu->kvm); in vmx_vcpu_create()
7542 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], in vmx_vcpu_create()
7543 __pa(&vmx->pi_desc) | PID_TABLE_ENTRY_VALID); in vmx_vcpu_create()
7548 free_loaded_vmcs(vmx->loaded_vmcs); in vmx_vcpu_create()
7552 free_vpid(vmx->vpid); in vmx_vcpu_create()
7556 …nt and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/ad…
7557 …tion disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/ad…
7562 kvm->arch.pause_in_guest = true; in vmx_vm_init()
7601 * When there is no need to deal with noncoherent DMA (e.g., no VT-d in vmx_get_mt_mask()
7602 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The in vmx_get_mt_mask()
7613 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) in vmx_get_mt_mask()
7617 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) in vmx_get_mt_mask()
7648 * (indicating "allowed-1") if they are supported in the guest's CPUID.
7655 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; in nested_vmx_cr_fixed1_bits_update()
7656 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; in nested_vmx_cr_fixed1_bits_update()
7659 if (entry && (entry->_reg & (_cpuid_mask))) \ in nested_vmx_cr_fixed1_bits_update()
7660 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ in nested_vmx_cr_fixed1_bits_update()
7703 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; in update_intel_pt_cfg()
7704 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; in update_intel_pt_cfg()
7705 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; in update_intel_pt_cfg()
7706 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; in update_intel_pt_cfg()
7710 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, in update_intel_pt_cfg()
7714 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | in update_intel_pt_cfg()
7722 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) in update_intel_pt_cfg()
7723 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; in update_intel_pt_cfg()
7729 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) in update_intel_pt_cfg()
7730 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | in update_intel_pt_cfg()
7736 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) in update_intel_pt_cfg()
7737 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | in update_intel_pt_cfg()
7741 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) in update_intel_pt_cfg()
7742 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | in update_intel_pt_cfg()
7746 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) in update_intel_pt_cfg()
7747 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; in update_intel_pt_cfg()
7750 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) in update_intel_pt_cfg()
7751 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; in update_intel_pt_cfg()
7754 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) in update_intel_pt_cfg()
7755 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; in update_intel_pt_cfg()
7758 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) in update_intel_pt_cfg()
7759 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); in update_intel_pt_cfg()
7785 vmx->msr_ia32_feature_control_valid_bits |= in vmx_vcpu_after_set_cpuid()
7789 vmx->msr_ia32_feature_control_valid_bits &= in vmx_vcpu_after_set_cpuid()
7825 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; in vmx_vcpu_after_set_cpuid()
7827 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; in vmx_vcpu_after_set_cpuid()
7830 vmx->msr_ia32_feature_control_valid_bits |= in vmx_vcpu_after_set_cpuid()
7833 vmx->msr_ia32_feature_control_valid_bits &= in vmx_vcpu_after_set_cpuid()
7918 to_vmx(vcpu)->req_immediate_exit = true; in vmx_request_immediate_exit()
7929 if (info->intercept == x86_intercept_in || in vmx_check_intercept_io()
7930 info->intercept == x86_intercept_ins) { in vmx_check_intercept_io()
7931 port = info->src_val; in vmx_check_intercept_io()
7932 size = info->dst_bytes; in vmx_check_intercept_io()
7934 port = info->dst_val; in vmx_check_intercept_io()
7935 size = info->src_bytes; in vmx_check_intercept_io()
7939 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction in vmx_check_intercept_io()
7940 * VM-exits depend on the 'unconditional IO exiting' VM-execution in vmx_check_intercept_io()
7943 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. in vmx_check_intercept_io()
7962 switch (info->intercept) { in vmx_check_intercept()
7970 exception->vector = UD_VECTOR; in vmx_check_intercept()
7971 exception->error_code_valid = false; in vmx_check_intercept()
7998 * PAUSE is a single-byte NOP with a REPE prefix, i.e. collides in vmx_check_intercept()
8001 * PAUSE-loop-exiting, software can't expect a given PAUSE to in vmx_check_intercept()
8005 if ((info->rep_prefix != REPE_PREFIX) || in vmx_check_intercept()
8024 u64 low = a << shift, high = a >> (64 - shift); in u64_shl_div_u64()
8043 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; in vmx_set_hv_timer()
8048 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; in vmx_set_hv_timer()
8050 ktimer->timer_advance_ns); in vmx_set_hv_timer()
8053 delta_tsc -= lapic_timer_advance_cycles; in vmx_set_hv_timer()
8058 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio && in vmx_set_hv_timer()
8061 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) in vmx_set_hv_timer()
8062 return -ERANGE; in vmx_set_hv_timer()
8071 return -ERANGE; in vmx_set_hv_timer()
8073 vmx->hv_deadline_tsc = tscl + delta_tsc; in vmx_set_hv_timer()
8080 to_vmx(vcpu)->hv_deadline_tsc = -1; in vmx_cancel_hv_timer()
8086 if (!kvm_pause_in_guest(vcpu->kvm)) in vmx_sched_in()
8098 vmx->nested.update_vmcs01_cpu_dirty_logging = true; in vmx_update_cpu_dirty_logging()
8107 if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) in vmx_update_cpu_dirty_logging()
8115 if (vcpu->arch.mcg_cap & MCG_LMCE_P) in vmx_setup_mce()
8116 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= in vmx_setup_mce()
8119 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= in vmx_setup_mce()
8127 if (to_vmx(vcpu)->nested.nested_run_pending) in vmx_smi_allowed()
8128 return -EBUSY; in vmx_smi_allowed()
8138 * SMI and RSM. Using the common VM-Exit + VM-Enter routines is wrong in vmx_enter_smm()
8140 * E.g. most MSRs are left untouched, but many are modified by VM-Exit in vmx_enter_smm()
8141 * and VM-Enter, and thus L2's values may be corrupted on SMI+RSM. in vmx_enter_smm()
8143 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); in vmx_enter_smm()
8144 if (vmx->nested.smm.guest_mode) in vmx_enter_smm()
8145 nested_vmx_vmexit(vcpu, -1, 0, 0); in vmx_enter_smm()
8147 vmx->nested.smm.vmxon = vmx->nested.vmxon; in vmx_enter_smm()
8148 vmx->nested.vmxon = false; in vmx_enter_smm()
8158 if (vmx->nested.smm.vmxon) { in vmx_leave_smm()
8159 vmx->nested.vmxon = true; in vmx_leave_smm()
8160 vmx->nested.smm.vmxon = false; in vmx_leave_smm()
8163 if (vmx->nested.smm.guest_mode) { in vmx_leave_smm()
8168 vmx->nested.nested_run_pending = 1; in vmx_leave_smm()
8169 vmx->nested.smm.guest_mode = false; in vmx_leave_smm()
8182 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); in vmx_apic_init_signal_blocked()
8188 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; in vmx_migrate_timers()
8220 free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); in vmx_vm_destroy()
8227 * and when the masking occurs relative to the canonicality check isn't visible
8243 * or a supervisor address. in vmx_get_untagged_addr()
8260 * Untag the address by sign-extending the lam_bit, but NOT to bit 63. in vmx_get_untagged_addr()
8262 * doesn't change a user access to a supervisor access, and vice versa. in vmx_get_untagged_addr()
8421 (unsigned long *)&vcpu->arch.pmu.global_status); in vmx_handle_intel_pt_intr()
8429 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm in vmx_setup_user_return_msrs()
8430 * will emulate SYSCALL in legacy mode if the vendor string in guest in vmx_setup_user_return_msrs()
8459 * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to in vmx_setup_me_spte_mask()
8467 kvm_get_shadow_phys_bits() - 1); in vmx_setup_me_spte_mask()
8490 return -EIO; in hardware_setup()
8521 return -EOPNOTSUPP; in hardware_setup()
8601 * and EPT A/D bit features are enabled -- PML depends on them to work. in hardware_setup()
8641 return -EINVAL; in hardware_setup()
8711 return -EOPNOTSUPP; in vmx_init()
8728 * mitigation mode. in vmx_init()