67 #include "vmcs.h"
468 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr)  in vmclear_error()  argument
471 			vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR));  in vmclear_error()
474 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr)  in vmptrld_error()  argument
477 			vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR));  in vmptrld_error()
491 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
492 DEFINE_PER_CPU(struct vmcs *, current_vmcs);
494  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
544 	evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs;  in hv_enable_l2_tlb_flush()
561 	 * Enlightened VMCS usage should be recommended and the host needs  in hv_init_evmcs()
577 			pr_info("Using Hyper-V Enlightened VMCS\n");  in hv_init_evmcs()
606 	 * Reset everything to support using non-enlightened VMCS access later  in hv_reset_evmcs()
773 		vmcs_clear(v->vmcs);  in vmx_emergency_disable_virtualization_cpu()
785 	if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)  in __loaded_vmcs_clear()
788 	vmcs_clear(loaded_vmcs->vmcs);  in __loaded_vmcs_clear()
1153  * VMCS rather than the segment table.  KVM uses this helper to figure
1154  * out the current bases to poke them into the VMCS before entry.
1225 	 * GUEST_IA32_RTIT_CTL is already set in the VMCS.  in pt_guest_enter()
1449 	struct vmcs *prev;  in vmx_vcpu_load_vmcs()
1469 	if (prev != vmx->loaded_vmcs->vmcs) {  in vmx_vcpu_load_vmcs()
1470 		per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;  in vmx_vcpu_load_vmcs()
1471 		vmcs_load(vmx->loaded_vmcs->vmcs);  in vmx_vcpu_load_vmcs()
1475 		 * the active VMCS within a vCPU, unless IBRS is advertised to  in vmx_vcpu_load_vmcs()
1478 		 * may switch the active VMCS multiple times).  in vmx_vcpu_load_vmcs()
1481 		    (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)))  in vmx_vcpu_load_vmcs()
1722 	 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on  in skip_emulated_instruction()
1723 	 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be  in skip_emulated_instruction()
1736 		 * vmcs.GUEST_RIP points at the exit point of the enclave, not  in skip_emulated_instruction()
1823 	 * Ensure that we clear the HLT state in the VMCS.  We don't need to  in vmx_clear_hlt()
1844 		 * bits, and the VMCS field being 32 bits, Intel CPUs and thus  in vmx_inject_exception()
2098 		 * Enlightened VMCS v1 doesn't have certain VMCS fields but  in vmx_get_msr()
2751 	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */  in setup_vmcs_config()
2758 	 * VMCS fields for 64-bit kernels, and per the SDM, "This bit is always  in setup_vmcs_config()
2765 	/* Require Write-Back (WB) memory type for VMCS accesses. */  in setup_vmcs_config()
2827 		pr_err("Failed to setup VMCS config on CPU %d\n", cpu);  in vmx_check_processor_compat()
2833 		pr_err("Inconsistent VMCS config on CPU %d\n", cpu);  in vmx_check_processor_compat()
2908 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)  in alloc_vmcs_cpu()
2912 	struct vmcs *vmcs;  in alloc_vmcs_cpu()  local
2917 	vmcs = page_address(pages);  in alloc_vmcs_cpu()
2918 	memset(vmcs, 0, vmx_basic_vmcs_size(vmcs_config.basic));  in alloc_vmcs_cpu()
2920 	/* KVM supports Enlightened VMCS v1 only */  in alloc_vmcs_cpu()
2922 		vmcs->hdr.revision_id = KVM_EVMCS_VERSION;  in alloc_vmcs_cpu()
2924 		vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic);  in alloc_vmcs_cpu()
2927 		vmcs->hdr.shadow_vmcs = 1;  in alloc_vmcs_cpu()
2928 	return vmcs;  in alloc_vmcs_cpu()
2931 void free_vmcs(struct vmcs *vmcs)  in free_vmcs()  argument
2933 	free_page((unsigned long)vmcs);  in free_vmcs()
2937  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2941 	if (!loaded_vmcs->vmcs)  in free_loaded_vmcs()
2944 	free_vmcs(loaded_vmcs->vmcs);  in free_loaded_vmcs()
2945 	loaded_vmcs->vmcs = NULL;  in free_loaded_vmcs()
2953 	loaded_vmcs->vmcs = alloc_vmcs(false);  in alloc_loaded_vmcs()
2954 	if (!loaded_vmcs->vmcs)  in alloc_loaded_vmcs()
2957 	vmcs_clear(loaded_vmcs->vmcs);  in alloc_loaded_vmcs()
2998 		struct vmcs *vmcs;  in alloc_kvm_area()  local
3000 		vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);  in alloc_kvm_area()
3001 		if (!vmcs) {  in alloc_kvm_area()
3008 		 * vmcs->revision_id to KVM_EVMCS_VERSION instead of  in alloc_kvm_area()
3017 			vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic);  in alloc_kvm_area()
3019 		per_cpu(vmxarea, cpu) = vmcs;  in alloc_kvm_area()
3362 		 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks  in vmx_set_cr0()
3448 		else /* vmcs.GUEST_CR3 is already up-to-date. */  in vmx_load_mmu_pgd()
4009 		struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs;  in vmx_msr_bitmap_l01_changed()
4334  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
4350 	 * Save the most likely value for this task's CR3 in the VMCS.  in vmx_set_constant_host_state()
4357 	/* Save the most likely value for this task's CR4 in the VMCS. */  in vmx_set_constant_host_state()
5349 			 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS  in handle_exception_nmi()
6358 	pr_err("VMCS %p, last attempted VM-entry on CPU %d\n",  in dump_vmcs()
6359 	       vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu);  in dump_vmcs()
6506 		 * If KVM is dumping the VMCS, then something has gone wrong  in dump_vmcs()
6507 		 * already.  Derefencing an address from the VMCS, which could  in dump_vmcs()
6803 	/* Postpone execution until vmcs01 is the current VMCS. */  in vmx_set_virtual_apic_mode()
6856 	/* Defer reload until vmcs01 is the current VMCS. */  in vmx_set_apic_access_page_addr()
7453 	 * Refresh vmcs.HOST_CR3 if necessary.  This must be done immediately  in vmx_vcpu_run()
7621 		struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs;  in vmx_vcpu_create()
8555 	 * using the APIC_ACCESS_ADDR VMCS field.  in vmx_hardware_setup()