Lines Matching full:nested
13 #include "nested.h"
184 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; in nested_vmx_failValid()
197 if (vmx->nested.current_vmptr == INVALID_GPA && in nested_vmx_fail()
208 pr_debug_ratelimited("nested vmx abort, indicator %d\n", indicator); in nested_vmx_abort()
225 vmx->nested.need_vmcs12_to_shadow_sync = false; in vmx_disable_shadow_vmcs()
235 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); in nested_release_evmcs()
236 vmx->nested.hv_evmcs = NULL; in nested_release_evmcs()
239 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; in nested_release_evmcs()
242 hv_vcpu->nested.pa_page_gpa = INVALID_GPA; in nested_release_evmcs()
243 hv_vcpu->nested.vm_id = 0; in nested_release_evmcs()
244 hv_vcpu->nested.vp_id = 0; in nested_release_evmcs()
261 * vmx->nested.hv_evmcs but this shouldn't be a problem. in nested_evmcs_handle_vmclear()
267 if (nested_vmx_evmcs(vmx) && vmptr == vmx->nested.hv_evmcs_vmptr) in nested_evmcs_handle_vmclear()
321 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
331 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) in free_nested()
336 vmx->nested.vmxon = false; in free_nested()
337 vmx->nested.smm.vmxon = false; in free_nested()
338 vmx->nested.vmxon_ptr = INVALID_GPA; in free_nested()
339 free_vpid(vmx->nested.vpid02); in free_nested()
340 vmx->nested.posted_intr_nv = -1; in free_nested()
341 vmx->nested.current_vmptr = INVALID_GPA; in free_nested()
348 kfree(vmx->nested.cached_vmcs12); in free_nested()
349 vmx->nested.cached_vmcs12 = NULL; in free_nested()
350 kfree(vmx->nested.cached_shadow_vmcs12); in free_nested()
351 vmx->nested.cached_shadow_vmcs12 = NULL; in free_nested()
357 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); in free_nested()
358 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); in free_nested()
359 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); in free_nested()
360 vmx->nested.pi_desc = NULL; in free_nested()
366 free_loaded_vmcs(&vmx->nested.vmcs02); in free_nested()
416 if (vmx->nested.pml_full) { in nested_ept_inject_page_fault()
418 vmx->nested.pml_full = false; in nested_ept_inject_page_fault()
421 * It should be impossible to trigger a nested PML Full VM-Exit in nested_ept_inject_page_fault()
466 bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; in nested_ept_new_eptp()
467 int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); in nested_ept_new_eptp()
626 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; in nested_vmx_prepare_msr_bitmap()
627 struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; in nested_vmx_prepare_msr_bitmap()
637 * - Nested hypervisor (L1) is attempting to launch the same L2 as in nested_vmx_prepare_msr_bitmap()
639 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature in nested_vmx_prepare_msr_bitmap()
642 if (!vmx->nested.force_msr_bitmap_recalc) { in nested_vmx_prepare_msr_bitmap()
717 kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); in nested_vmx_prepare_msr_bitmap()
719 vmx->nested.force_msr_bitmap_recalc = false; in nested_vmx_prepare_msr_bitmap()
728 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; in nested_cache_shadow_vmcs12()
747 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; in nested_flush_cached_shadow_vmcs12()
763 * In nested virtualization, check if L1 has set
946 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, in nested_vmx_max_atomic_switch_msrs()
947 vmx->nested.msrs.misc_high); in nested_vmx_max_atomic_switch_msrs()
953 * Load guest's/host's msr at nested entry/exit.
1133 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are
1180 * while L2 entries are tagged with vmx->nested.vpid02).
1187 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); in nested_has_guest_tlb_tag()
1209 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. in nested_vmx_transition_tlb_flush()
1231 if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { in nested_vmx_transition_tlb_flush()
1232 vmx->nested.last_vpid = vmcs12->virtual_processor_id; in nested_vmx_transition_tlb_flush()
1265 u64 vmx_basic = vmcs_config.nested.basic; in vmx_restore_vmx_basic()
1292 vmx->nested.msrs.basic = data; in vmx_restore_vmx_basic()
1331 vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp); in vmx_restore_control_msr()
1343 vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp); in vmx_restore_control_msr()
1363 u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low, in vmx_restore_vmx_misc()
1364 vmcs_config.nested.misc_high); in vmx_restore_vmx_misc()
1376 if ((vmx->nested.msrs.pinbased_ctls_high & in vmx_restore_vmx_misc()
1391 vmx->nested.msrs.misc_low = data; in vmx_restore_vmx_misc()
1392 vmx->nested.msrs.misc_high = data >> 32; in vmx_restore_vmx_misc()
1399 u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps, in vmx_restore_vmx_ept_vpid_cap()
1400 vmcs_config.nested.vpid_caps); in vmx_restore_vmx_ept_vpid_cap()
1406 vmx->nested.msrs.ept_caps = data; in vmx_restore_vmx_ept_vpid_cap()
1407 vmx->nested.msrs.vpid_caps = data >> 32; in vmx_restore_vmx_ept_vpid_cap()
1425 const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index); in vmx_restore_fixed0_msr()
1434 *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data; in vmx_restore_fixed0_msr()
1451 if (vmx->nested.vmxon) in vmx_set_vmx_msr()
1492 vmx->nested.msrs.vmcs_enum = data; in vmx_set_vmx_msr()
1495 if (data & ~vmcs_config.nested.vmfunc_controls) in vmx_set_vmx_msr()
1497 vmx->nested.msrs.vmfunc_controls = data; in vmx_set_vmx_msr()
1657 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; in copy_enlightened_to_vmcs12()
1667 hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; in copy_enlightened_to_vmcs12()
1668 hv_vcpu->nested.vm_id = evmcs->hv_vm_id; in copy_enlightened_to_vmcs12()
1669 hv_vcpu->nested.vp_id = evmcs->hv_vp_id; in copy_enlightened_to_vmcs12()
1905 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; in copy_vmcs12_to_enlightened()
2081 * This is an equivalent of the nested hypervisor executing the vmptrld
2101 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { in nested_vmx_handle_enlightened_vmptrld()
2102 vmx->nested.current_vmptr = INVALID_GPA; in nested_vmx_handle_enlightened_vmptrld()
2107 &vmx->nested.hv_evmcs_map)) in nested_vmx_handle_enlightened_vmptrld()
2110 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; in nested_vmx_handle_enlightened_vmptrld()
2134 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && in nested_vmx_handle_enlightened_vmptrld()
2135 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { in nested_vmx_handle_enlightened_vmptrld()
2140 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; in nested_vmx_handle_enlightened_vmptrld()
2162 vmx->nested.hv_evmcs->hv_clean_fields &= in nested_vmx_handle_enlightened_vmptrld()
2165 vmx->nested.force_msr_bitmap_recalc = true; in nested_vmx_handle_enlightened_vmptrld()
2183 vmx->nested.need_vmcs12_to_shadow_sync = false; in nested_sync_vmcs12_to_shadow()
2189 container_of(timer, struct vcpu_vmx, nested.preemption_timer); in vmx_preemption_timer_fn()
2191 vmx->nested.preemption_timer_expired = true; in vmx_preemption_timer_fn()
2206 if (!vmx->nested.has_preemption_timer_deadline) { in vmx_calc_preemption_timer_value()
2207 vmx->nested.preemption_timer_deadline = in vmx_calc_preemption_timer_value()
2209 vmx->nested.has_preemption_timer_deadline = true; in vmx_calc_preemption_timer_value()
2211 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; in vmx_calc_preemption_timer_value()
2224 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); in vmx_start_preemption_timer()
2234 hrtimer_start(&vmx->nested.preemption_timer, in vmx_start_preemption_timer()
2241 if (vmx->nested.nested_run_pending && in nested_vmx_calc_efer()
2260 if (vmx->nested.vmcs02_initialized) in prepare_vmcs02_constant_state()
2262 vmx->nested.vmcs02_initialized = true; in prepare_vmcs02_constant_state()
2284 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); in prepare_vmcs02_constant_state()
2333 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) in prepare_vmcs02_early_rare()
2334 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); in prepare_vmcs02_early_rare()
2346 if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) in prepare_vmcs02_early()
2357 vmx->nested.pi_pending = false; in prepare_vmcs02_early()
2359 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; in prepare_vmcs02_early()
2361 vmx->nested.posted_intr_nv = -1; in prepare_vmcs02_early()
2375 vmx->nested.l1_tpr_threshold = -1; in prepare_vmcs02_early()
2491 if (vmx->nested.nested_run_pending) { in prepare_vmcs02_early()
2573 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && in prepare_vmcs02_rare()
2627 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
2645 if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) { in prepare_vmcs02()
2647 vmx->nested.dirty_vmcs12 = false; in prepare_vmcs02()
2653 if (vmx->nested.nested_run_pending && in prepare_vmcs02()
2659 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.pre_vmenter_debugctl); in prepare_vmcs02()
2661 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || in prepare_vmcs02()
2663 vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs); in prepare_vmcs02()
2674 if (vmx->nested.nested_run_pending && in prepare_vmcs02()
2721 * loading nested state after migration, it is possible to in prepare_vmcs02()
2737 * on nested VM-Exit, which can occur without actually running L2 and in prepare_vmcs02()
2797 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) in nested_vmx_check_eptp()
2801 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) in nested_vmx_check_eptp()
2811 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) in nested_vmx_check_eptp()
2815 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) in nested_vmx_check_eptp()
2828 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) in nested_vmx_check_eptp()
2844 vmx->nested.msrs.pinbased_ctls_low, in nested_check_vm_execution_controls()
2845 vmx->nested.msrs.pinbased_ctls_high)) || in nested_check_vm_execution_controls()
2847 vmx->nested.msrs.procbased_ctls_low, in nested_check_vm_execution_controls()
2848 vmx->nested.msrs.procbased_ctls_high))) in nested_check_vm_execution_controls()
2853 vmx->nested.msrs.secondary_ctls_low, in nested_check_vm_execution_controls()
2854 vmx->nested.msrs.secondary_ctls_high))) in nested_check_vm_execution_controls()
2881 ~vmx->nested.msrs.vmfunc_controls)) in nested_check_vm_execution_controls()
2903 vmx->nested.msrs.exit_ctls_low, in nested_check_vm_exit_controls()
2904 vmx->nested.msrs.exit_ctls_high)) || in nested_check_vm_exit_controls()
2920 vmx->nested.msrs.entry_ctls_low, in nested_check_vm_entry_controls()
2921 vmx->nested.msrs.entry_ctls_high))) in nested_check_vm_entry_controls()
3086 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; in nested_vmx_check_vmcs_link_ptr()
3171 if (to_vmx(vcpu)->nested.nested_run_pending && in nested_vmx_check_guest_state()
3280 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { in nested_get_evmcs_page()
3292 vmx->nested.need_vmcs12_to_shadow_sync = true; in nested_get_evmcs_page()
3309 * the guest CR3 might be restored prior to setting the nested in nested_get_vmcs12_pages()
3318 map = &vmx->nested.apic_access_page_map; in nested_get_vmcs12_pages()
3334 map = &vmx->nested.virtual_apic_map; in nested_get_vmcs12_pages()
3360 map = &vmx->nested.pi_desc_map; in nested_get_vmcs12_pages()
3363 vmx->nested.pi_desc = in nested_get_vmcs12_pages()
3375 vmx->nested.pi_desc = NULL; in nested_get_vmcs12_pages()
3423 if (WARN_ON_ONCE(vmx->nested.pml_full)) in nested_vmx_write_pml_buffer()
3427 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is in nested_vmx_write_pml_buffer()
3435 vmx->nested.pml_full = true; in nested_vmx_write_pml_buffer()
3460 if (!to_vmx(vcpu)->nested.vmxon) { in nested_vmx_check_permission()
3508 vmx->nested.current_vmptr, in nested_vmx_enter_non_root_mode()
3526 if (!vmx->nested.nested_run_pending || in nested_vmx_enter_non_root_mode()
3528 vmx->nested.pre_vmenter_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); in nested_vmx_enter_non_root_mode()
3530 (!vmx->nested.nested_run_pending || in nested_vmx_enter_non_root_mode()
3532 vmx->nested.pre_vmenter_bndcfgs = vmcs_read64(GUEST_BNDCFGS); in nested_vmx_enter_non_root_mode()
3536 * nested early checks are disabled. In the event of a "late" VM-Fail, in nested_vmx_enter_non_root_mode()
3542 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested in nested_vmx_enter_non_root_mode()
3545 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as in nested_vmx_enter_non_root_mode()
3553 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); in nested_vmx_enter_non_root_mode()
3618 vmx->nested.preemption_timer_expired = false; in nested_vmx_enter_non_root_mode()
3651 vmx->nested.need_vmcs12_to_shadow_sync = true; in nested_vmx_enter_non_root_mode()
3656 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
3657 * for running an L2 nested guest.
3682 vmx->nested.current_vmptr == INVALID_GPA)) in nested_vmx_run()
3707 * The nested entry process starts with enforcing various prerequisites in nested_vmx_run()
3735 * the nested entry. in nested_vmx_run()
3737 vmx->nested.nested_run_pending = 1; in nested_vmx_run()
3738 vmx->nested.has_preemption_timer_deadline = false; in nested_vmx_run()
3745 kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) { in nested_vmx_run()
3746 vmx->nested.pi_pending = true; in nested_vmx_run()
3748 kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv); in nested_vmx_run()
3751 /* Hide L1D cache contents from the nested guest. */ in nested_vmx_run()
3777 vmx->nested.nested_run_pending = 0; in nested_vmx_run()
3782 vmx->nested.nested_run_pending = 0; in nested_vmx_run()
3792 vmx->nested.nested_run_pending = 0; in nested_vmx_run()
3802 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
3933 if (!vmx->nested.pi_pending) in vmx_complete_nested_posted_interrupt()
3936 if (!vmx->nested.pi_desc) in vmx_complete_nested_posted_interrupt()
3939 vmx->nested.pi_pending = false; in vmx_complete_nested_posted_interrupt()
3941 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) in vmx_complete_nested_posted_interrupt()
3944 max_irr = pi_find_highest_vector(vmx->nested.pi_desc); in vmx_complete_nested_posted_interrupt()
3946 vapic_page = vmx->nested.virtual_apic_map.hva; in vmx_complete_nested_posted_interrupt()
3950 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, in vmx_complete_nested_posted_interrupt()
3997 * hardware and avoid inducing failure on nested VM-Entry if L1 in nested_vmx_inject_exception_vmexit()
4073 to_vmx(vcpu)->nested.preemption_timer_expired; in nested_vmx_preemption_timer_pending()
4079 void *vapic = vmx->nested.virtual_apic_map.hva; in vmx_has_nested_events()
4083 vmx->nested.mtf_pending) in vmx_has_nested_events()
4108 if (vmx->nested.pi_pending && vmx->nested.pi_desc && in vmx_has_nested_events()
4109 pi_test_on(vmx->nested.pi_desc)) { in vmx_has_nested_events()
4110 max_irr = pi_find_highest_vector(vmx->nested.pi_desc); in vmx_has_nested_events()
4206 * Only a pending nested run blocks a pending exception. If there is a in vmx_check_nested_events()
4210 bool block_nested_exceptions = vmx->nested.nested_run_pending; in vmx_check_nested_events()
4230 vmx->nested.mtf_pending = false; in vmx_check_nested_events()
4274 if (vmx->nested.mtf_pending) { in vmx_check_nested_events()
4359 if (irq == vmx->nested.posted_intr_nv) { in vmx_check_nested_events()
4360 vmx->nested.pi_pending = true; in vmx_check_nested_events()
4384 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); in vmx_get_preemption_timer_value()
4484 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; in sync_vmcs02_to_vmcs12_rare()
4493 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) in copy_vmcs02_to_vmcs12_rare()
4500 vmx->loaded_vmcs = &vmx->nested.vmcs02; in copy_vmcs02_to_vmcs12_rare()
4506 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02); in copy_vmcs02_to_vmcs12_rare()
4523 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = in sync_vmcs02_to_vmcs12()
4548 !vmx->nested.nested_run_pending) in sync_vmcs02_to_vmcs12()
4553 * In some cases (usually, nested EPT), L2 is allowed to change its in sync_vmcs02_to_vmcs12()
4587 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
4645 * A part of what we need to when the nested L2 guest exits and we want to
4648 * This function is to be called not only on normal nested exit, but also on
4649 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
4816 * nested VMENTER (not worth adding a variable in nested_vmx). in nested_vmx_restore_host_state()
4854 * of VMFail), leaving the nested VM's MSRs in the software model in nested_vmx_restore_host_state()
4857 * MSR that was (prematurely) loaded from the nested VMEntry load in nested_vmx_restore_host_state()
4907 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
4918 vmx->nested.mtf_pending = false; in nested_vmx_vmexit()
4921 WARN_ON_ONCE(vmx->nested.nested_run_pending); in nested_vmx_vmexit()
4949 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); in nested_vmx_vmexit()
5017 if (vmx->nested.l1_tpr_threshold != -1) in nested_vmx_vmexit()
5018 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); in nested_vmx_vmexit()
5020 if (vmx->nested.change_vmcs01_virtual_apic_mode) { in nested_vmx_vmexit()
5021 vmx->nested.change_vmcs01_virtual_apic_mode = false; in nested_vmx_vmexit()
5025 if (vmx->nested.update_vmcs01_cpu_dirty_logging) { in nested_vmx_vmexit()
5026 vmx->nested.update_vmcs01_cpu_dirty_logging = false; in nested_vmx_vmexit()
5031 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); in nested_vmx_vmexit()
5032 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); in nested_vmx_vmexit()
5033 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); in nested_vmx_vmexit()
5034 vmx->nested.pi_desc = NULL; in nested_vmx_vmexit()
5036 if (vmx->nested.reload_vmcs01_apic_access_page) { in nested_vmx_vmexit()
5037 vmx->nested.reload_vmcs01_apic_access_page = false; in nested_vmx_vmexit()
5041 if (vmx->nested.update_vmcs01_apicv_status) { in nested_vmx_vmexit()
5042 vmx->nested.update_vmcs01_apicv_status = false; in nested_vmx_vmexit()
5048 vmx->nested.need_vmcs12_to_shadow_sync = true; in nested_vmx_vmexit()
5259 * when L1 executes VMXOFF or the vCPU is forced out of nested in alloc_shadow_vmcs()
5280 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); in enter_vmx_operation()
5284 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); in enter_vmx_operation()
5285 if (!vmx->nested.cached_vmcs12) in enter_vmx_operation()
5288 vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; in enter_vmx_operation()
5289 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); in enter_vmx_operation()
5290 if (!vmx->nested.cached_shadow_vmcs12) in enter_vmx_operation()
5296 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, in enter_vmx_operation()
5298 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; in enter_vmx_operation()
5300 vmx->nested.vpid02 = allocate_vpid(); in enter_vmx_operation()
5302 vmx->nested.vmcs02_initialized = false; in enter_vmx_operation()
5303 vmx->nested.vmxon = true; in enter_vmx_operation()
5313 kfree(vmx->nested.cached_shadow_vmcs12); in enter_vmx_operation()
5316 kfree(vmx->nested.cached_vmcs12); in enter_vmx_operation()
5319 free_loaded_vmcs(&vmx->nested.vmcs02); in enter_vmx_operation()
5371 if (vmx->nested.vmxon) in handle_vmxon()
5399 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; in handle_vmxon()
5409 vmx->nested.vmxon_ptr = vmptr; in handle_vmxon()
5421 if (vmx->nested.current_vmptr == INVALID_GPA) in nested_release_vmcs12()
5432 vmx->nested.posted_intr_nv = -1; in nested_release_vmcs12()
5436 vmx->nested.current_vmptr >> PAGE_SHIFT, in nested_release_vmcs12()
5437 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); in nested_release_vmcs12()
5441 vmx->nested.current_vmptr = INVALID_GPA; in nested_release_vmcs12()
5475 if (vmptr == vmx->nested.vmxon_ptr) in handle_vmclear()
5479 if (vmptr == vmx->nested.current_vmptr) in handle_vmclear()
5538 if (vmx->nested.current_vmptr == INVALID_GPA || in handle_vmread()
5648 if (vmx->nested.current_vmptr == INVALID_GPA || in handle_vmwrite()
5720 vmx->nested.dirty_vmcs12 = true; in handle_vmwrite()
5728 vmx->nested.current_vmptr = vmptr; in set_current_vmptr()
5733 vmx->nested.need_vmcs12_to_shadow_sync = true; in set_current_vmptr()
5735 vmx->nested.dirty_vmcs12 = true; in set_current_vmptr()
5736 vmx->nested.force_msr_bitmap_recalc = true; in set_current_vmptr()
5755 if (vmptr == vmx->nested.vmxon_ptr) in handle_vmptrld()
5762 if (vmx->nested.current_vmptr != vmptr) { in handle_vmptrld()
5763 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; in handle_vmptrld()
5797 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, in handle_vmptrld()
5814 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; in handle_vmptrst()
5851 if (!(vmx->nested.msrs.secondary_ctls_high & in handle_invept()
5853 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { in handle_invept()
5865 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; in handle_invept()
5881 * Nested EPT roots are always held through guest_mmu, in handle_invept()
5932 if (!(vmx->nested.msrs.secondary_ctls_high & in handle_invvpid()
5934 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { in handle_invvpid()
5946 types = (vmx->nested.msrs.vpid_caps & in handle_invvpid()
6058 * VMFUNC for nested VMs, but not for L1. in handle_vmfunc()
6092 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode in handle_vmfunc()
6362 * L0 always deals with the EPT violation. If nested EPT is in nested_vmx_l0_wants_exit()
6363 * used, and the nested mmu code discovers that the address is in nested_vmx_l0_wants_exit()
6544 WARN_ON_ONCE(vmx->nested.nested_run_pending); in nested_vmx_reflect_vmexit()
6547 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM in nested_vmx_reflect_vmexit()
6614 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { in vmx_get_nested_state()
6615 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; in vmx_get_nested_state()
6616 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; in vmx_get_nested_state()
6631 if (vmx->nested.smm.vmxon) in vmx_get_nested_state()
6634 if (vmx->nested.smm.guest_mode) in vmx_get_nested_state()
6640 if (vmx->nested.nested_run_pending) in vmx_get_nested_state()
6643 if (vmx->nested.mtf_pending) in vmx_get_nested_state()
6647 vmx->nested.has_preemption_timer_deadline) { in vmx_get_nested_state()
6651 vmx->nested.preemption_timer_deadline; in vmx_get_nested_state()
6677 if (!vmx->nested.need_vmcs12_to_shadow_sync) { in vmx_get_nested_state()
6715 to_vmx(vcpu)->nested.nested_run_pending = 0; in vmx_leave_nested()
6789 !vmx->nested.enlightened_vmcs_enabled)) in vmx_set_nested_state()
6797 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; in vmx_set_nested_state()
6827 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; in vmx_set_nested_state()
6835 vmx->nested.smm.vmxon = true; in vmx_set_nested_state()
6836 vmx->nested.vmxon = false; in vmx_set_nested_state()
6839 vmx->nested.smm.guest_mode = true; in vmx_set_nested_state()
6852 vmx->nested.nested_run_pending = in vmx_set_nested_state()
6855 vmx->nested.mtf_pending = in vmx_set_nested_state()
6880 vmx->nested.has_preemption_timer_deadline = false; in vmx_set_nested_state()
6882 vmx->nested.has_preemption_timer_deadline = true; in vmx_set_nested_state()
6883 vmx->nested.preemption_timer_deadline = in vmx_set_nested_state()
6892 vmx->nested.dirty_vmcs12 = true; in vmx_set_nested_state()
6893 vmx->nested.force_msr_bitmap_recalc = true; in vmx_set_nested_state()
6898 if (vmx->nested.mtf_pending) in vmx_set_nested_state()
6904 vmx->nested.nested_run_pending = 0; in vmx_set_nested_state()
7076 /* nested EPT: emulate EPT also to L1 */ in nested_vmx_setup_secondary_ctls()
7179 * returned for the various VMX controls MSRs when nested VMX is enabled.
7181 * valid during nested entry from L1 to L2.
7189 struct nested_vmx_msrs *msrs = &vmcs_conf->nested; in nested_vmx_setup_ctls_msrs()