64 #include "vmcs.h"
458 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr)  in vmclear_error()  argument
461 			vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR));  in vmclear_error()
464 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr)  in vmptrld_error()  argument
467 			vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR));  in vmptrld_error()
482 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
483 DEFINE_PER_CPU(struct vmcs *, current_vmcs);
485  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
541 	evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs;  in hv_enable_l2_tlb_flush()
558 	 * Enlightened VMCS usage should be recommended and the host needs  in hv_init_evmcs()
574 			pr_info("Using Hyper-V Enlightened VMCS\n");  in hv_init_evmcs()
604 	 * Reset everything to support using non-enlightened VMCS access later  in hv_reset_evmcs()
780 		vmcs_clear(v->vmcs);  in vmx_emergency_disable()
792 	if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)  in __loaded_vmcs_clear()
795 	vmcs_clear(loaded_vmcs->vmcs);  in __loaded_vmcs_clear()
1154  * VMCS rather than the segment table.  KVM uses this helper to figure
1155  * out the current bases to poke them into the VMCS before entry.
1226 	 * GUEST_IA32_RTIT_CTL is already set in the VMCS.  in pt_guest_enter()
1420 	struct vmcs *prev;  in vmx_vcpu_load_vmcs()
1440 	if (prev != vmx->loaded_vmcs->vmcs) {  in vmx_vcpu_load_vmcs()
1441 		per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;  in vmx_vcpu_load_vmcs()
1442 		vmcs_load(vmx->loaded_vmcs->vmcs);  in vmx_vcpu_load_vmcs()
1446 		 * the active VMCS within a vCPU, unless IBRS is advertised to  in vmx_vcpu_load_vmcs()
1449 		 * may switch the active VMCS multiple times).  in vmx_vcpu_load_vmcs()
1451 		if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev))  in vmx_vcpu_load_vmcs()
1686 	 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on  in skip_emulated_instruction()
1687 	 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be  in skip_emulated_instruction()
1700 		 * vmcs.GUEST_RIP points at the exit point of the enclave, not  in skip_emulated_instruction()
1787 	 * Ensure that we clear the HLT state in the VMCS.  We don't need to  in vmx_clear_hlt()
1808 		 * bits, and the VMCS field being 32 bits, Intel CPUs and thus  in vmx_inject_exception()
2062 		 * Enlightened VMCS v1 doesn't have certain VMCS fields but  in vmx_get_msr()
2702 	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */  in setup_vmcs_config()
2712 	/* Require Write-Back (WB) memory type for VMCS accesses. */  in setup_vmcs_config()
2778 		pr_err("Failed to setup VMCS config on CPU %d\n", cpu);  in vmx_check_processor_compat()
2784 		pr_err("Inconsistent VMCS config on CPU %d\n", cpu);  in vmx_check_processor_compat()
2862 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)  in alloc_vmcs_cpu()
2866 	struct vmcs *vmcs;  in alloc_vmcs_cpu()  local
2871 	vmcs = page_address(pages);  in alloc_vmcs_cpu()
2872 	memset(vmcs, 0, vmcs_config.size);  in alloc_vmcs_cpu()
2874 	/* KVM supports Enlightened VMCS v1 only */  in alloc_vmcs_cpu()
2876 		vmcs->hdr.revision_id = KVM_EVMCS_VERSION;  in alloc_vmcs_cpu()
2878 		vmcs->hdr.revision_id = vmcs_config.revision_id;  in alloc_vmcs_cpu()
2881 		vmcs->hdr.shadow_vmcs = 1;  in alloc_vmcs_cpu()
2882 	return vmcs;  in alloc_vmcs_cpu()
2885 void free_vmcs(struct vmcs *vmcs)  in free_vmcs()  argument
2887 	free_page((unsigned long)vmcs);  in free_vmcs()
2891  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2895 	if (!loaded_vmcs->vmcs)  in free_loaded_vmcs()
2898 	free_vmcs(loaded_vmcs->vmcs);  in free_loaded_vmcs()
2899 	loaded_vmcs->vmcs = NULL;  in free_loaded_vmcs()
2907 	loaded_vmcs->vmcs = alloc_vmcs(false);  in alloc_loaded_vmcs()
2908 	if (!loaded_vmcs->vmcs)  in alloc_loaded_vmcs()
2911 	vmcs_clear(loaded_vmcs->vmcs);  in alloc_loaded_vmcs()
2952 		struct vmcs *vmcs;  in alloc_kvm_area()  local
2954 		vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);  in alloc_kvm_area()
2955 		if (!vmcs) {  in alloc_kvm_area()
2962 		 * vmcs->revision_id to KVM_EVMCS_VERSION instead of  in alloc_kvm_area()
2971 			vmcs->hdr.revision_id = vmcs_config.revision_id;  in alloc_kvm_area()
2973 		per_cpu(vmxarea, cpu) = vmcs;  in alloc_kvm_area()
3316 		 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks  in vmx_set_cr0()
3403 		else /* vmcs.GUEST_CR3 is already up-to-date. */  in vmx_load_mmu_pgd()
3952 		struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs;  in vmx_msr_bitmap_l01_changed()
4291  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
4307 	 * Save the most likely value for this task's CR3 in the VMCS.  in vmx_set_constant_host_state()
4314 	/* Save the most likely value for this task's CR4 in the VMCS. */  in vmx_set_constant_host_state()
5286 			 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS  in handle_exception_nmi()
6262 	pr_err("VMCS %p, last attempted VM-entry on CPU %d\n",  in dump_vmcs()
6263 	       vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu);  in dump_vmcs()
6706 	/* Postpone execution until vmcs01 is the current VMCS. */  in vmx_set_virtual_apic_mode()
6757 	/* Defer reload until vmcs01 is the current VMCS. */  in vmx_set_apic_access_page_addr()
7330 	 * Refresh vmcs.HOST_CR3 if necessary.  This must be done immediately  in vmx_vcpu_run()
7502 		struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs;  in vmx_vcpu_create()
8544 	 * using the APIC_ACCESS_ADDR VMCS field.  in hardware_setup()