103 	((struct kvm_vcpu *)(ctxt)->vcpu)
125 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
126 static void process_nmi(struct kvm_vcpu *vcpu);
127 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
128 static void store_regs(struct kvm_vcpu *vcpu);
129 static int sync_regs(struct kvm_vcpu *vcpu);
130 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
132 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
133 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
272 	STATS_DESC_COUNTER(VCPU, pf_taken),
273 	STATS_DESC_COUNTER(VCPU, pf_fixed),
274 	STATS_DESC_COUNTER(VCPU, pf_emulate),
275 	STATS_DESC_COUNTER(VCPU, pf_spurious),
276 	STATS_DESC_COUNTER(VCPU, pf_fast),
277 	STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
278 	STATS_DESC_COUNTER(VCPU, pf_guest),
279 	STATS_DESC_COUNTER(VCPU, tlb_flush),
280 	STATS_DESC_COUNTER(VCPU, invlpg),
281 	STATS_DESC_COUNTER(VCPU, exits),
282 	STATS_DESC_COUNTER(VCPU, io_exits),
283 	STATS_DESC_COUNTER(VCPU, mmio_exits),
284 	STATS_DESC_COUNTER(VCPU, signal_exits),
285 	STATS_DESC_COUNTER(VCPU, irq_window_exits),
286 	STATS_DESC_COUNTER(VCPU, nmi_window_exits),
287 	STATS_DESC_COUNTER(VCPU, l1d_flush),
288 	STATS_DESC_COUNTER(VCPU, halt_exits),
289 	STATS_DESC_COUNTER(VCPU, request_irq_exits),
290 	STATS_DESC_COUNTER(VCPU, irq_exits),
291 	STATS_DESC_COUNTER(VCPU, host_state_reload),
292 	STATS_DESC_COUNTER(VCPU, fpu_reload),
293 	STATS_DESC_COUNTER(VCPU, insn_emulation),
294 	STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
295 	STATS_DESC_COUNTER(VCPU, hypercalls),
296 	STATS_DESC_COUNTER(VCPU, irq_injections),
297 	STATS_DESC_COUNTER(VCPU, nmi_injections),
298 	STATS_DESC_COUNTER(VCPU, req_event),
299 	STATS_DESC_COUNTER(VCPU, nested_run),
300 	STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
301 	STATS_DESC_COUNTER(VCPU, directed_yield_successful),
302 	STATS_DESC_COUNTER(VCPU, preemption_reported),
303 	STATS_DESC_COUNTER(VCPU, preemption_other),
304 	STATS_DESC_IBOOLEAN(VCPU, guest_mode),
305 	STATS_DESC_COUNTER(VCPU, notify_window_exits),
355 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)  in kvm_async_pf_hash_reset()  argument
359 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_async_pf_hash_reset()
474 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)  in kvm_get_apic_base()  argument
476 	return vcpu->arch.apic_base;  in kvm_get_apic_base()
479 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)  in kvm_get_apic_mode()  argument
481 	return kvm_apic_mode(kvm_get_apic_base(vcpu));  in kvm_get_apic_mode()
485 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_apic_base()  argument
487 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);  in kvm_set_apic_base()
489 	u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |  in kvm_set_apic_base()
490 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);  in kvm_set_apic_base()
501 	kvm_lapic_set_base(vcpu, msr_info->data);  in kvm_set_apic_base()
502 	kvm_recalculate_apic_map(vcpu->kvm);  in kvm_set_apic_base()
573 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,  in kvm_deliver_exception_payload()  argument
586 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;  in kvm_deliver_exception_payload()
603 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
604 		vcpu->arch.dr6 |= ex->payload;  in kvm_deliver_exception_payload()
605 		vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
613 		vcpu->arch.dr6 &= ~BIT(12);  in kvm_deliver_exception_payload()
616 		vcpu->arch.cr2 = ex->payload;  in kvm_deliver_exception_payload()
625 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,  in kvm_queue_exception_vmexit()  argument
629 	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;  in kvm_queue_exception_vmexit()
640 /* Forcibly leave the nested mode in cases like a vCPU reset */
641 static void kvm_leave_nested(struct kvm_vcpu *vcpu)  in kvm_leave_nested()  argument
643 	kvm_x86_ops.nested_ops->leave_nested(vcpu);  in kvm_leave_nested()
646 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,  in kvm_multiple_exception()  argument
653 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_multiple_exception()
662 	if (!reinject && is_guest_mode(vcpu) &&  in kvm_multiple_exception()
663 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {  in kvm_multiple_exception()
664 		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,  in kvm_multiple_exception()
669 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {  in kvm_multiple_exception()
679 			WARN_ON_ONCE(kvm_is_exception_pending(vcpu));  in kvm_multiple_exception()
680 			vcpu->arch.exception.injected = true;  in kvm_multiple_exception()
690 			vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
691 			vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
693 		vcpu->arch.exception.has_error_code = has_error;  in kvm_multiple_exception()
694 		vcpu->arch.exception.vector = nr;  in kvm_multiple_exception()
695 		vcpu->arch.exception.error_code = error_code;  in kvm_multiple_exception()
696 		vcpu->arch.exception.has_payload = has_payload;  in kvm_multiple_exception()
697 		vcpu->arch.exception.payload = payload;  in kvm_multiple_exception()
698 		if (!is_guest_mode(vcpu))  in kvm_multiple_exception()
699 			kvm_deliver_exception_payload(vcpu,  in kvm_multiple_exception()
700 						      &vcpu->arch.exception);  in kvm_multiple_exception()
705 	prev_nr = vcpu->arch.exception.vector;  in kvm_multiple_exception()
708 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_multiple_exception()
719 		vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
720 		vcpu->arch.exception.pending = false;  in kvm_multiple_exception()
722 		kvm_queue_exception_e(vcpu, DF_VECTOR, 0);  in kvm_multiple_exception()
731 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_queue_exception()  argument
733 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);  in kvm_queue_exception()
737 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_requeue_exception()  argument
739 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);  in kvm_requeue_exception()
743 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_p()  argument
746 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);  in kvm_queue_exception_p()
750 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_e_p()  argument
753 	kvm_multiple_exception(vcpu, nr, true, error_code,  in kvm_queue_exception_e_p()
757 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)  in kvm_complete_insn_gp()  argument
760 		kvm_inject_gp(vcpu, 0);  in kvm_complete_insn_gp()
762 		return kvm_skip_emulated_instruction(vcpu);  in kvm_complete_insn_gp()
768 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)  in complete_emulated_insn_gp()  argument
771 		kvm_inject_gp(vcpu, 0);  in complete_emulated_insn_gp()
775 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |  in complete_emulated_insn_gp()
779 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)  in kvm_inject_page_fault()  argument
781 	++vcpu->stat.pf_guest;  in kvm_inject_page_fault()
787 	if (is_guest_mode(vcpu) && fault->async_page_fault)  in kvm_inject_page_fault()
788 		kvm_queue_exception_vmexit(vcpu, PF_VECTOR,  in kvm_inject_page_fault()
792 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,  in kvm_inject_page_fault()
796 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,  in kvm_inject_emulated_page_fault()  argument
802 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :  in kvm_inject_emulated_page_fault()
803 					       vcpu->arch.walk_mmu;  in kvm_inject_emulated_page_fault()
811 		kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,  in kvm_inject_emulated_page_fault()
814 	fault_mmu->inject_page_fault(vcpu, fault);  in kvm_inject_emulated_page_fault()
818 void kvm_inject_nmi(struct kvm_vcpu *vcpu)  in kvm_inject_nmi()  argument
820 	atomic_inc(&vcpu->arch.nmi_queued);  in kvm_inject_nmi()
821 	kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_inject_nmi()
824 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_queue_exception_e()  argument
826 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);  in kvm_queue_exception_e()
830 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_requeue_exception_e()  argument
832 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);  in kvm_requeue_exception_e()
840 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)  in kvm_require_cpl()  argument
842 	if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)  in kvm_require_cpl()
844 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in kvm_require_cpl()
848 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)  in kvm_require_dr()  argument
850 	if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))  in kvm_require_dr()
853 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_require_dr()
858 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)  in pdptr_rsvd_bits()  argument
860 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);  in pdptr_rsvd_bits()
866 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)  in load_pdptrs()  argument
868 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in load_pdptrs()
879 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),  in load_pdptrs()
885 	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,  in load_pdptrs()
892 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {  in load_pdptrs()
902 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);  in load_pdptrs()
905 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in load_pdptrs()
906 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);  in load_pdptrs()
907 	vcpu->arch.pdptrs_from_userspace = false;  in load_pdptrs()
913 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_is_valid_cr0()  argument
926 	return static_call(kvm_x86_is_valid_cr0)(vcpu, cr0);  in kvm_is_valid_cr0()
929 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)  in kvm_post_set_cr0()  argument
944 			kvm_init_mmu(vcpu);  in kvm_post_set_cr0()
950 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_post_set_cr0()
951 		kvm_async_pf_hash_reset(vcpu);  in kvm_post_set_cr0()
958 			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr0()
962 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr0()
965 	    kvm_mmu_honors_guest_mtrrs(vcpu->kvm) &&  in kvm_post_set_cr0()
966 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))  in kvm_post_set_cr0()
967 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);  in kvm_post_set_cr0()
971 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_set_cr0()  argument
973 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_set_cr0()
975 	if (!kvm_is_valid_cr0(vcpu, cr0))  in kvm_set_cr0()
984 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&  in kvm_set_cr0()
988 		if (!is_pae(vcpu))  in kvm_set_cr0()
990 		static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in kvm_set_cr0()
995 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&  in kvm_set_cr0()
996 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&  in kvm_set_cr0()
997 	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr0()
1001 	    (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))  in kvm_set_cr0()
1004 	static_call(kvm_x86_set_cr0)(vcpu, cr0);  in kvm_set_cr0()
1006 	kvm_post_set_cr0(vcpu, old_cr0, cr0);  in kvm_set_cr0()
1012 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)  in kvm_lmsw()  argument
1014 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));  in kvm_lmsw()
1018 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_guest_xsave_state()  argument
1020 	if (vcpu->arch.guest_state_protected)  in kvm_load_guest_xsave_state()
1023 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_guest_xsave_state()
1025 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_guest_xsave_state()
1026 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);  in kvm_load_guest_xsave_state()
1028 		if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_guest_xsave_state()
1029 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_guest_xsave_state()
1030 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);  in kvm_load_guest_xsave_state()
1034 	    vcpu->arch.pkru != vcpu->arch.host_pkru &&  in kvm_load_guest_xsave_state()
1035 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_guest_xsave_state()
1036 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))  in kvm_load_guest_xsave_state()
1037 		write_pkru(vcpu->arch.pkru);  in kvm_load_guest_xsave_state()
1041 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_host_xsave_state()  argument
1043 	if (vcpu->arch.guest_state_protected)  in kvm_load_host_xsave_state()
1047 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_host_xsave_state()
1048 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {  in kvm_load_host_xsave_state()
1049 		vcpu->arch.pkru = rdpkru();  in kvm_load_host_xsave_state()
1050 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_host_xsave_state()
1051 			write_pkru(vcpu->arch.host_pkru);  in kvm_load_host_xsave_state()
1054 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_host_xsave_state()
1056 		if (vcpu->arch.xcr0 != host_xcr0)  in kvm_load_host_xsave_state()
1059 		if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_host_xsave_state()
1060 		    vcpu->arch.ia32_xss != host_xss)  in kvm_load_host_xsave_state()
1068 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)  in kvm_guest_supported_xfd()  argument
1070 	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;  in kvm_guest_supported_xfd()
1074 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)  in __kvm_set_xcr()  argument
1077 	u64 old_xcr0 = vcpu->arch.xcr0;  in __kvm_set_xcr()
1093 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;  in __kvm_set_xcr()
1112 	vcpu->arch.xcr0 = xcr0;  in __kvm_set_xcr()
1115 		kvm_update_cpuid_runtime(vcpu);  in __kvm_set_xcr()
1119 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)  in kvm_emulate_xsetbv()  argument
1122 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||  in kvm_emulate_xsetbv()
1123 	    __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {  in kvm_emulate_xsetbv()
1124 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_xsetbv()
1128 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_xsetbv()
1132 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in __kvm_is_valid_cr4()  argument
1137 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)  in __kvm_is_valid_cr4()
1144 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_is_valid_cr4()  argument
1146 	return __kvm_is_valid_cr4(vcpu, cr4) &&  in kvm_is_valid_cr4()
1147 	       static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);  in kvm_is_valid_cr4()
1150 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)  in kvm_post_set_cr4()  argument
1153 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr4()
1165 		kvm_mmu_unload(vcpu);  in kvm_post_set_cr4()
1177 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr4()
1187 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_post_set_cr4()
1192 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_set_cr4()  argument
1194 	unsigned long old_cr4 = kvm_read_cr4(vcpu);  in kvm_set_cr4()
1196 	if (!kvm_is_valid_cr4(vcpu, cr4))  in kvm_set_cr4()
1199 	if (is_long_mode(vcpu)) {  in kvm_set_cr4()
1204 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)  in kvm_set_cr4()
1206 		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr4()
1211 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))  in kvm_set_cr4()
1215 	static_call(kvm_x86_set_cr4)(vcpu, cr4);  in kvm_set_cr4()
1217 	kvm_post_set_cr4(vcpu, old_cr4, cr4);  in kvm_set_cr4()
1223 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)  in kvm_invalidate_pcid()  argument
1225 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_invalidate_pcid()
1237 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_invalidate_pcid()
1246 	if (kvm_get_active_pcid(vcpu) == pcid) {  in kvm_invalidate_pcid()
1247 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);  in kvm_invalidate_pcid()
1248 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_invalidate_pcid()
1256 	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))  in kvm_invalidate_pcid()
1260 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)  in kvm_invalidate_pcid()
1263 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);  in kvm_invalidate_pcid()
1266 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)  in kvm_set_cr3()  argument
1271 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {  in kvm_set_cr3()
1279 	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))  in kvm_set_cr3()
1285 	 * the current vCPU mode is accurate.  in kvm_set_cr3()
1287 	if (!kvm_vcpu_is_legal_cr3(vcpu, cr3))  in kvm_set_cr3()
1290 	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))  in kvm_set_cr3()
1293 	if (cr3 != kvm_read_cr3(vcpu))  in kvm_set_cr3()
1294 		kvm_mmu_new_pgd(vcpu, cr3);  in kvm_set_cr3()
1296 	vcpu->arch.cr3 = cr3;  in kvm_set_cr3()
1297 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_set_cr3()
1309 		kvm_invalidate_pcid(vcpu, pcid);  in kvm_set_cr3()
1315 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)  in kvm_set_cr8()  argument
1319 	if (lapic_in_kernel(vcpu))  in kvm_set_cr8()
1320 		kvm_lapic_set_tpr(vcpu, cr8);  in kvm_set_cr8()
1322 		vcpu->arch.cr8 = cr8;  in kvm_set_cr8()
1327 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)  in kvm_get_cr8()  argument
1329 	if (lapic_in_kernel(vcpu))  in kvm_get_cr8()
1330 		return kvm_lapic_get_cr8(vcpu);  in kvm_get_cr8()
1332 		return vcpu->arch.cr8;  in kvm_get_cr8()
1336 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)  in kvm_update_dr0123()  argument
1340 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {  in kvm_update_dr0123()
1342 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_update_dr0123()
1346 void kvm_update_dr7(struct kvm_vcpu *vcpu)  in kvm_update_dr7()  argument
1350 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)  in kvm_update_dr7()
1351 		dr7 = vcpu->arch.guest_debug_dr7;  in kvm_update_dr7()
1353 		dr7 = vcpu->arch.dr7;  in kvm_update_dr7()
1354 	static_call(kvm_x86_set_dr7)(vcpu, dr7);  in kvm_update_dr7()
1355 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1357 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1361 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)  in kvm_dr6_fixed()  argument
1365 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))  in kvm_dr6_fixed()
1368 	if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))  in kvm_dr6_fixed()
1373 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)  in kvm_set_dr()  argument
1375 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_set_dr()
1379 		vcpu->arch.db[array_index_nospec(dr, size)] = val;  in kvm_set_dr()
1380 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))  in kvm_set_dr()
1381 			vcpu->arch.eff_db[dr] = val;  in kvm_set_dr()
1387 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);  in kvm_set_dr()
1393 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;  in kvm_set_dr()
1394 		kvm_update_dr7(vcpu);  in kvm_set_dr()
1402 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)  in kvm_get_dr()  argument
1404 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_get_dr()
1408 		*val = vcpu->arch.db[array_index_nospec(dr, size)];  in kvm_get_dr()
1412 		*val = vcpu->arch.dr6;  in kvm_get_dr()
1416 		*val = vcpu->arch.dr7;  in kvm_get_dr()
1422 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)  in kvm_emulate_rdpmc()  argument
1424 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdpmc()
1427 	if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {  in kvm_emulate_rdpmc()
1428 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_rdpmc()
1432 	kvm_rax_write(vcpu, (u32)data);  in kvm_emulate_rdpmc()
1433 	kvm_rdx_write(vcpu, data >> 32);  in kvm_emulate_rdpmc()
1434 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_rdpmc()
1593  * patch, are immutable once the vCPU model is defined.
1702 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr_feature()  argument
1720 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in __kvm_valid_efer()  argument
1722 	if (efer & EFER_AUTOIBRS && !guest_cpuid_has(vcpu, X86_FEATURE_AUTOIBRS))  in __kvm_valid_efer()
1725 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))  in __kvm_valid_efer()
1728 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))  in __kvm_valid_efer()
1732 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))  in __kvm_valid_efer()
1735 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))  in __kvm_valid_efer()
1741 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in kvm_valid_efer()  argument
1746 	return __kvm_valid_efer(vcpu, efer);  in kvm_valid_efer()
1750 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_efer()  argument
1752 	u64 old_efer = vcpu->arch.efer;  in set_efer()
1760 		if (!__kvm_valid_efer(vcpu, efer))  in set_efer()
1763 		if (is_paging(vcpu) &&  in set_efer()
1764 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))  in set_efer()
1769 	efer |= vcpu->arch.efer & EFER_LMA;  in set_efer()
1771 	r = static_call(kvm_x86_set_efer)(vcpu, efer);  in set_efer()
1778 		kvm_mmu_reset_context(vcpu);  in set_efer()
1782 		kvm_hv_xsaves_xsavec_maybe_warn(vcpu);  in set_efer()
1793 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)  in kvm_msr_allowed()  argument
1797 	struct kvm *kvm = vcpu->kvm;  in kvm_msr_allowed()
1842 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,  in __kvm_set_msr()  argument
1853 		if (is_noncanonical_address(data, vcpu))  in __kvm_set_msr()
1870 		data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));  in __kvm_set_msr()
1877 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_set_msr()
1878 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))  in __kvm_set_msr()
1890 		if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)  in __kvm_set_msr()
1901 	return static_call(kvm_x86_set_msr)(vcpu, &msr);  in __kvm_set_msr()
1904 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_set_msr_ignored_check()  argument
1907 	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);  in kvm_set_msr_ignored_check()
1922 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in __kvm_get_msr()  argument
1934 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_get_msr()
1935 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))  in __kvm_get_msr()
1943 	ret = static_call(kvm_x86_get_msr)(vcpu, &msr);  in __kvm_get_msr()
1949 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_get_msr_ignored_check()  argument
1952 	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);  in kvm_get_msr_ignored_check()
1964 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr_with_filter()  argument
1966 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))  in kvm_get_msr_with_filter()
1968 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr_with_filter()
1971 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr_with_filter()  argument
1973 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))  in kvm_set_msr_with_filter()
1975 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr_with_filter()
1978 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr()  argument
1980 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr()
1984 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr()  argument
1986 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr()
1990 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)  in complete_userspace_rdmsr()  argument
1992 	if (!vcpu->run->msr.error) {  in complete_userspace_rdmsr()
1993 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);  in complete_userspace_rdmsr()
1994 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);  in complete_userspace_rdmsr()
1998 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)  in complete_emulated_msr_access()  argument
2000 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);  in complete_emulated_msr_access()
2003 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)  in complete_emulated_rdmsr()  argument
2005 	complete_userspace_rdmsr(vcpu);  in complete_emulated_rdmsr()
2006 	return complete_emulated_msr_access(vcpu);  in complete_emulated_rdmsr()
2009 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)  in complete_fast_msr_access()  argument
2011 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);  in complete_fast_msr_access()
2014 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)  in complete_fast_rdmsr()  argument
2016 	complete_userspace_rdmsr(vcpu);  in complete_fast_rdmsr()
2017 	return complete_fast_msr_access(vcpu);  in complete_fast_rdmsr()
2032 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,  in kvm_msr_user_space()  argument
2034 			      int (*completion)(struct kvm_vcpu *vcpu),  in kvm_msr_user_space()  argument
2040 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))  in kvm_msr_user_space()
2043 	vcpu->run->exit_reason = exit_reason;  in kvm_msr_user_space()
2044 	vcpu->run->msr.error = 0;  in kvm_msr_user_space()
2045 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));  in kvm_msr_user_space()
2046 	vcpu->run->msr.reason = msr_reason;  in kvm_msr_user_space()
2047 	vcpu->run->msr.index = index;  in kvm_msr_user_space()
2048 	vcpu->run->msr.data = data;  in kvm_msr_user_space()
2049 	vcpu->arch.complete_userspace_io = completion;  in kvm_msr_user_space()
2054 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_rdmsr()  argument
2056 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdmsr()
2060 	r = kvm_get_msr_with_filter(vcpu, ecx, &data);  in kvm_emulate_rdmsr()
2065 		kvm_rax_write(vcpu, data & -1u);  in kvm_emulate_rdmsr()
2066 		kvm_rdx_write(vcpu, (data >> 32) & -1u);  in kvm_emulate_rdmsr()
2069 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,  in kvm_emulate_rdmsr()
2075 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);  in kvm_emulate_rdmsr()
2079 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_wrmsr()  argument
2081 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_wrmsr()
2082 	u64 data = kvm_read_edx_eax(vcpu);  in kvm_emulate_wrmsr()
2085 	r = kvm_set_msr_with_filter(vcpu, ecx, data);  in kvm_emulate_wrmsr()
2091 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,  in kvm_emulate_wrmsr()
2100 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);  in kvm_emulate_wrmsr()
2104 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)  in kvm_emulate_as_nop()  argument
2106 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_as_nop()
2109 int kvm_emulate_invd(struct kvm_vcpu *vcpu)  in kvm_emulate_invd()  argument
2112 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_invd()
2116 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)  in kvm_handle_invalid_op()  argument
2118 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_handle_invalid_op()
2124 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)  in kvm_emulate_monitor_mwait()  argument
2126 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&  in kvm_emulate_monitor_mwait()
2127 	    !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))  in kvm_emulate_monitor_mwait()
2128 		return kvm_handle_invalid_op(vcpu);  in kvm_emulate_monitor_mwait()
2131 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_monitor_mwait()
2133 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)  in kvm_emulate_mwait()  argument
2135 	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");  in kvm_emulate_mwait()
2139 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)  in kvm_emulate_monitor()  argument
2141 	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");  in kvm_emulate_monitor()
2145 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)  in kvm_vcpu_exit_request()  argument
2148 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||  in kvm_vcpu_exit_request()
2159 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_x2apic_icr_irqoff()  argument
2161 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))  in handle_fastpath_set_x2apic_icr_irqoff()
2168 		return kvm_x2apic_icr_write(vcpu->arch.apic, data);  in handle_fastpath_set_x2apic_icr_irqoff()
2173 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_tscdeadline()  argument
2175 	if (!kvm_can_use_hv_timer(vcpu))  in handle_fastpath_set_tscdeadline()
2178 	kvm_set_lapic_tscdeadline_msr(vcpu, data);  in handle_fastpath_set_tscdeadline()
2182 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)  in handle_fastpath_set_msr_irqoff()  argument
2184 	u32 msr = kvm_rcx_read(vcpu);  in handle_fastpath_set_msr_irqoff()
2188 	kvm_vcpu_srcu_read_lock(vcpu);  in handle_fastpath_set_msr_irqoff()
2192 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2193 		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
2194 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
2199 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2200 		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {  in handle_fastpath_set_msr_irqoff()
2201 			kvm_skip_emulated_instruction(vcpu);  in handle_fastpath_set_msr_irqoff()
2212 	kvm_vcpu_srcu_read_unlock(vcpu);  in handle_fastpath_set_msr_irqoff()
2221 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr()  argument
2223 	return kvm_get_msr_ignored_check(vcpu, index, data, true);  in do_get_msr()
2226 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_set_msr()  argument
2232 	 * not support modifying the guest vCPU model on the fly, e.g. changing  in do_set_msr()
2237 	if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index)) {  in do_set_msr()
2238 		if (do_get_msr(vcpu, index, &val) || *data != val)  in do_set_msr()
2244 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);  in do_set_msr()
2354 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,  in kvm_write_system_time()  argument
2357 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_write_system_time()
2359 	if (vcpu->vcpu_id == 0 && !host_initiated) {  in kvm_write_system_time()
2361 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2366 	vcpu->arch.time = system_time;  in kvm_write_system_time()
2367 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2371 		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,  in kvm_write_system_time()
2374 		kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvm_write_system_time()
2427 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2429 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)  in set_tsc_khz()  argument
2435 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in set_tsc_khz()
2442 			vcpu->arch.tsc_catchup = 1;  in set_tsc_khz()
2443 			vcpu->arch.tsc_always_catchup = 1;  in set_tsc_khz()
2461 	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);  in set_tsc_khz()
2465 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)  in kvm_set_tsc_khz()  argument
2473 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in kvm_set_tsc_khz()
2479 			   &vcpu->arch.virtual_tsc_shift,  in kvm_set_tsc_khz()
2480 			   &vcpu->arch.virtual_tsc_mult);  in kvm_set_tsc_khz()
2481 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;  in kvm_set_tsc_khz()
2496 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);  in kvm_set_tsc_khz()
2499 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)  in compute_guest_tsc()  argument
2501 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,  in compute_guest_tsc()
2502 				      vcpu->arch.virtual_tsc_mult,  in compute_guest_tsc()
2503 				      vcpu->arch.virtual_tsc_shift);  in compute_guest_tsc()
2504 	tsc += vcpu->arch.this_tsc_write;  in compute_guest_tsc()
2515 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)  in kvm_track_tsc_matching()  argument
2518 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_track_tsc_matching()
2524 	 * vCPUs doesn't include the reference vCPU, hence "+1".  in kvm_track_tsc_matching()
2527 				 atomic_read(&vcpu->kvm->online_vcpus)) &&  in kvm_track_tsc_matching()
2538 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_track_tsc_matching()
2540 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,  in kvm_track_tsc_matching()
2541 			    atomic_read(&vcpu->kvm->online_vcpus),  in kvm_track_tsc_matching()
2571 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)  in kvm_compute_l1_tsc_offset()  argument
2575 	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);  in kvm_compute_l1_tsc_offset()
2580 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)  in kvm_read_l1_tsc()  argument
2582 	return vcpu->arch.l1_tsc_offset +  in kvm_read_l1_tsc()
2583 		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);  in kvm_read_l1_tsc()
2612 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)  in kvm_vcpu_write_tsc_offset()  argument
2614 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,  in kvm_vcpu_write_tsc_offset()
2615 				   vcpu->arch.l1_tsc_offset,  in kvm_vcpu_write_tsc_offset()
2618 	vcpu->arch.l1_tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2625 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_offset()
2626 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(  in kvm_vcpu_write_tsc_offset()
2628 			static_call(kvm_x86_get_l2_tsc_offset)(vcpu),  in kvm_vcpu_write_tsc_offset()
2629 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_offset()
2631 		vcpu->arch.tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2633 	static_call(kvm_x86_write_tsc_offset)(vcpu);  in kvm_vcpu_write_tsc_offset()
2636 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)  in kvm_vcpu_write_tsc_multiplier()  argument
2638 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2641 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_multiplier()
2642 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(  in kvm_vcpu_write_tsc_multiplier()
2644 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_multiplier()
2646 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2649 		static_call(kvm_x86_write_tsc_multiplier)(vcpu);  in kvm_vcpu_write_tsc_multiplier()
2667  * offset for the vcpu and tracks the TSC matching generation that the vcpu
2670 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,  in __kvm_synchronize_tsc()  argument
2673 	struct kvm *kvm = vcpu->kvm;  in __kvm_synchronize_tsc()
2683 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;  in __kvm_synchronize_tsc()
2686 	vcpu->arch.last_guest_tsc = tsc;  in __kvm_synchronize_tsc()
2688 	kvm_vcpu_write_tsc_offset(vcpu, offset);  in __kvm_synchronize_tsc()
2705 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {  in __kvm_synchronize_tsc()
2709 	/* Keep track of which generation this VCPU has synchronized to */  in __kvm_synchronize_tsc()
2710 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;  in __kvm_synchronize_tsc()
2711 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;  in __kvm_synchronize_tsc()
2712 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;  in __kvm_synchronize_tsc()
2714 	kvm_track_tsc_matching(vcpu, !matched);  in __kvm_synchronize_tsc()
2717 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)  in kvm_synchronize_tsc()  argument
2720 	struct kvm *kvm = vcpu->kvm;  in kvm_synchronize_tsc()
2727 	offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2731 	if (vcpu->arch.virtual_tsc_khz) {  in kvm_synchronize_tsc()
2734 			 * Force synchronization when creating a vCPU, or when  in kvm_synchronize_tsc()
2740 						nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2741 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;  in kvm_synchronize_tsc()
2745 			 * previously set vCPU, we assume that they were intended to be  in kvm_synchronize_tsc()
2752 			 * value written by userspace (on any vCPU) should not be subject  in kvm_synchronize_tsc()
2754 			 * come from the kernel's default vCPU creation. Make the 1-second  in kvm_synchronize_tsc()
2772 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {  in kvm_synchronize_tsc()
2776 			u64 delta = nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2778 			offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2783 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched);  in kvm_synchronize_tsc()
2787 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,  in adjust_tsc_offset_guest()  argument
2790 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;  in adjust_tsc_offset_guest()
2791 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);  in adjust_tsc_offset_guest()
2794 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)  in adjust_tsc_offset_host()  argument
2796 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)  in adjust_tsc_offset_host()
2799 				   vcpu->arch.l1_tsc_scaling_ratio);  in adjust_tsc_offset_host()
2800 	adjust_tsc_offset_guest(vcpu, adjustment);  in adjust_tsc_offset_host()
3018 	struct kvm_vcpu *vcpu;  in kvm_end_pvclock_update()  local
3023 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3024 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_end_pvclock_update()
3027 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3028 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);  in kvm_end_pvclock_update()
3115 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_setup_guest_pvclock()  local
3132 	 * This VCPU is paused, but it's legal for a guest to read another  in kvm_setup_guest_pvclock()
3133 	 * VCPU's kvmclock, so we really have to follow the specification where  in kvm_setup_guest_pvclock()
3138 	guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;  in kvm_setup_guest_pvclock()
3142 	vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);  in kvm_setup_guest_pvclock()
3144 	if (vcpu->pvclock_set_guest_stopped_request) {  in kvm_setup_guest_pvclock()
3145 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;  in kvm_setup_guest_pvclock()
3146 		vcpu->pvclock_set_guest_stopped_request = false;  in kvm_setup_guest_pvclock()
3149 	memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));  in kvm_setup_guest_pvclock()
3156 	guest_hv_clock->version = ++vcpu->hv_clock.version;  in kvm_setup_guest_pvclock()
3161 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);  in kvm_setup_guest_pvclock()
3168 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_guest_time_update()  local
3220 	 *   2) Broken TSC compensation resets the base at each VCPU  in kvm_guest_time_update()
3226 	if (vcpu->tsc_catchup) {  in kvm_guest_time_update()
3242 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {  in kvm_guest_time_update()
3244 				   &vcpu->hv_clock.tsc_shift,  in kvm_guest_time_update()
3245 				   &vcpu->hv_clock.tsc_to_system_mul);  in kvm_guest_time_update()
3246 		vcpu->hw_tsc_khz = tgt_tsc_khz;  in kvm_guest_time_update()
3250 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;  in kvm_guest_time_update()
3251 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;  in kvm_guest_time_update()
3252 	vcpu->last_guest_tsc = tsc_timestamp;  in kvm_guest_time_update()
3259 	vcpu->hv_clock.flags = pvclock_flags;  in kvm_guest_time_update()
3261 	if (vcpu->pv_time.active)  in kvm_guest_time_update()
3262 		kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0, false);  in kvm_guest_time_update()
3264 	if (vcpu->xen.vcpu_info_cache.active)  in kvm_guest_time_update()
3265 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,  in kvm_guest_time_update()
3268 	if (vcpu->xen.vcpu_time_info_cache.active)  in kvm_guest_time_update()
3269 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0,  in kvm_guest_time_update()
3272 	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);  in kvm_guest_time_update()
3353  * kvmclock updates which are isolated to a given vcpu, such as
3354  * vcpu->cpu migration, should not allow system_timestamp from
3356  * correction applies to one vcpu's system_timestamp but not
3362  * The time for a remote vcpu to update its kvmclock is bound
3375 	struct kvm_vcpu *vcpu;  in kvmclock_update_fn()  local
3377 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmclock_update_fn()
3378 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvmclock_update_fn()
3379 		kvm_vcpu_kick(vcpu);  in kvmclock_update_fn()
3419 static bool can_set_mci_status(struct kvm_vcpu *vcpu)  in can_set_mci_status()  argument
3422 	if (guest_cpuid_is_amd_or_hygon(vcpu))  in can_set_mci_status()
3423 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));  in can_set_mci_status()
3428 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_msr_mce()  argument
3430 	u64 mcg_cap = vcpu->arch.mcg_cap;  in set_msr_mce()
3438 		vcpu->arch.mcg_status = data;  in set_msr_mce()
3446 		vcpu->arch.mcg_ctl = data;  in set_msr_mce()
3460 		vcpu->arch.mci_ctl2_banks[offset] = data;  in set_msr_mce()
3487 		    data != 0 && !can_set_mci_status(vcpu))  in set_msr_mce()
3492 		vcpu->arch.mce_banks[offset] = data;  in set_msr_mce()
3500 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)  in kvm_pv_async_pf_enabled()  argument
3504 	return (vcpu->arch.apf.msr_en_val & mask) == mask;  in kvm_pv_async_pf_enabled()
3507 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf()  argument
3515 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&  in kvm_pv_enable_async_pf()
3519 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&  in kvm_pv_enable_async_pf()
3523 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf()
3526 	vcpu->arch.apf.msr_en_val = data;  in kvm_pv_enable_async_pf()
3528 	if (!kvm_pv_async_pf_enabled(vcpu)) {  in kvm_pv_enable_async_pf()
3529 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_pv_enable_async_pf()
3530 		kvm_async_pf_hash_reset(vcpu);  in kvm_pv_enable_async_pf()
3534 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,  in kvm_pv_enable_async_pf()
3538 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);  in kvm_pv_enable_async_pf()
3539 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;  in kvm_pv_enable_async_pf()
3541 	kvm_async_pf_wakeup_all(vcpu);  in kvm_pv_enable_async_pf()
3546 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf_int()  argument
3552 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf_int()
3555 	vcpu->arch.apf.msr_int_val = data;  in kvm_pv_enable_async_pf_int()
3557 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;  in kvm_pv_enable_async_pf_int()
3562 static void kvmclock_reset(struct kvm_vcpu *vcpu)  in kvmclock_reset()  argument
3564 	kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvmclock_reset()
3565 	vcpu->arch.time = 0;  in kvmclock_reset()
3568 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_all()  argument
3570 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_all()
3571 	static_call(kvm_x86_flush_tlb_all)(vcpu);  in kvm_vcpu_flush_tlb_all()
3574 	kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_vcpu_flush_tlb_all()
3577 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_guest()  argument
3579 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_guest()
3588 		kvm_mmu_sync_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3589 		kvm_mmu_sync_prev_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3592 	static_call(kvm_x86_flush_tlb_guest)(vcpu);  in kvm_vcpu_flush_tlb_guest()
3598 	kvm_hv_vcpu_purge_flush_tlb(vcpu);  in kvm_vcpu_flush_tlb_guest()
3602 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_current()  argument
3604 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_current()
3605 	static_call(kvm_x86_flush_tlb_current)(vcpu);  in kvm_vcpu_flush_tlb_current()
3614 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)  in kvm_service_local_tlb_flush_requests()  argument
3616 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))  in kvm_service_local_tlb_flush_requests()
3617 		kvm_vcpu_flush_tlb_current(vcpu);  in kvm_service_local_tlb_flush_requests()
3619 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))  in kvm_service_local_tlb_flush_requests()
3620 		kvm_vcpu_flush_tlb_guest(vcpu);  in kvm_service_local_tlb_flush_requests()
3624 static void record_steal_time(struct kvm_vcpu *vcpu)  in record_steal_time()  argument
3626 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in record_steal_time()
3629 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in record_steal_time()
3633 	if (kvm_xen_msr_enabled(vcpu->kvm)) {  in record_steal_time()
3634 		kvm_xen_runstate_set_running(vcpu);  in record_steal_time()
3638 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in record_steal_time()
3641 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))  in record_steal_time()
3644 	slots = kvm_memslots(vcpu->kvm);  in record_steal_time()
3652 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||  in record_steal_time()
3662 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {  in record_steal_time()
3681 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3683 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,  in record_steal_time()
3686 			kvm_vcpu_flush_tlb_guest(vcpu);  in record_steal_time()
3695 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3709 		vcpu->arch.st.last_steal;  in record_steal_time()
3710 	vcpu->arch.st.last_steal = current->sched_info.run_delay;  in record_steal_time()
3719 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in record_steal_time()
3734 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_msr_common()  argument
3739 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)  in kvm_set_msr_common()
3740 		return kvm_xen_write_hypercall_page(vcpu, data);  in kvm_set_msr_common()
3755 			vcpu->arch.microcode_version = data;  in kvm_set_msr_common()
3760 		vcpu->arch.arch_capabilities = data;  in kvm_set_msr_common()
3770 		 * disallows changing feature MSRs after the vCPU has run; PMU  in kvm_set_msr_common()
3771 		 * refresh will bug the VM if called after the vCPU has run.  in kvm_set_msr_common()
3773 		if (vcpu->arch.perf_capabilities == data)  in kvm_set_msr_common()
3776 		vcpu->arch.perf_capabilities = data;  in kvm_set_msr_common()
3777 		kvm_pmu_refresh(vcpu);  in kvm_set_msr_common()
3783 			if ((!guest_has_pred_cmd_msr(vcpu)))  in kvm_set_msr_common()
3786 			if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&  in kvm_set_msr_common()
3787 			    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))  in kvm_set_msr_common()
3790 			if (!guest_cpuid_has(vcpu, X86_FEATURE_SBPB))  in kvm_set_msr_common()
3811 		    !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D))  in kvm_set_msr_common()
3822 		return set_efer(vcpu, msr_info);  in kvm_set_msr_common()
3834 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3837 		vcpu->arch.msr_hwcr = data;  in kvm_set_msr_common()
3841 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3849 		vcpu->arch.pat = data;  in kvm_set_msr_common()
3853 		return kvm_mtrr_set_msr(vcpu, msr, data);  in kvm_set_msr_common()
3855 		return kvm_set_apic_base(vcpu, msr_info);  in kvm_set_msr_common()
3857 		return kvm_x2apic_msr_write(vcpu, msr, data);  in kvm_set_msr_common()
3859 		kvm_set_lapic_tscdeadline_msr(vcpu, data);  in kvm_set_msr_common()
3862 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {  in kvm_set_msr_common()
3864 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;  in kvm_set_msr_common()
3865 				adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3869 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_msr_common()
3871 			vcpu->arch.ia32_tsc_adjust_msr = data;  in kvm_set_msr_common()
3875 		u64 old_val = vcpu->arch.ia32_misc_enable_msr;  in kvm_set_msr_common()
3887 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&  in kvm_set_msr_common()
3889 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))  in kvm_set_msr_common()
3891 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3892 			kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3894 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3901 		vcpu->arch.smbase = data;  in kvm_set_msr_common()
3904 		vcpu->arch.msr_ia32_power_ctl = data;  in kvm_set_msr_common()
3908 			kvm_synchronize_tsc(vcpu, &data);  in kvm_set_msr_common()
3910 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;  in kvm_set_msr_common()
3911 			adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3912 			vcpu->arch.ia32_tsc_adjust_msr += adj;  in kvm_set_msr_common()
3917 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_set_msr_common()
3926 		vcpu->arch.ia32_xss = data;  in kvm_set_msr_common()
3927 		kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3932 		vcpu->arch.smi_count = data;  in kvm_set_msr_common()
3935 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3938 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3939 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3942 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3945 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3946 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3949 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3952 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);  in kvm_set_msr_common()
3955 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3958 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);  in kvm_set_msr_common()
3961 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_set_msr_common()
3964 		if (kvm_pv_enable_async_pf(vcpu, data))  in kvm_set_msr_common()
3968 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3971 		if (kvm_pv_enable_async_pf_int(vcpu, data))  in kvm_set_msr_common()
3975 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3978 			vcpu->arch.apf.pageready_pending = false;  in kvm_set_msr_common()
3979 			kvm_check_async_pf_completion(vcpu);  in kvm_set_msr_common()
3983 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_set_msr_common()
3992 		vcpu->arch.st.msr_val = data;  in kvm_set_msr_common()
3997 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_set_msr_common()
4001 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_set_msr_common()
4004 		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))  in kvm_set_msr_common()
4009 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_set_msr_common()
4016 		vcpu->arch.msr_kvm_poll_control = data;  in kvm_set_msr_common()
4023 		return set_msr_mce(vcpu, msr_info);  in kvm_set_msr_common()
4029 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
4030 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
4033 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
4056 		return kvm_hv_set_msr_common(vcpu, msr, data,  in kvm_set_msr_common()
4063 		kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
4066 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
4068 		vcpu->arch.osvw.length = data;  in kvm_set_msr_common()
4071 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
4073 		vcpu->arch.osvw.status = data;  in kvm_set_msr_common()
4078 		     cpuid_fault_enabled(vcpu)))  in kvm_set_msr_common()
4080 		vcpu->arch.msr_platform_info = data;  in kvm_set_msr_common()
4085 		     !supports_cpuid_fault(vcpu)))  in kvm_set_msr_common()
4087 		vcpu->arch.msr_misc_features_enables = data;  in kvm_set_msr_common()
4092 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
4095 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
4098 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);  in kvm_set_msr_common()
4102 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
4105 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
4108 		vcpu->arch.guest_fpu.xfd_err = data;  in kvm_set_msr_common()
4112 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
4113 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
4129 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)  in get_msr_mce()  argument
4132 	u64 mcg_cap = vcpu->arch.mcg_cap;  in get_msr_mce()
4142 		data = vcpu->arch.mcg_cap;  in get_msr_mce()
4147 		data = vcpu->arch.mcg_ctl;  in get_msr_mce()
4150 		data = vcpu->arch.mcg_status;  in get_msr_mce()
4161 		data = vcpu->arch.mci_ctl2_banks[offset];  in get_msr_mce()
4170 		data = vcpu->arch.mce_banks[offset];  in get_msr_mce()
4179 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_get_msr_common()  argument
4217 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4218 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4222 		msr_info->data = vcpu->arch.microcode_version;  in kvm_get_msr_common()
4226 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))  in kvm_get_msr_common()
4228 		msr_info->data = vcpu->arch.arch_capabilities;  in kvm_get_msr_common()
4232 		    !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))  in kvm_get_msr_common()
4234 		msr_info->data = vcpu->arch.perf_capabilities;  in kvm_get_msr_common()
4237 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;  in kvm_get_msr_common()
4252 			offset = vcpu->arch.l1_tsc_offset;  in kvm_get_msr_common()
4253 			ratio = vcpu->arch.l1_tsc_scaling_ratio;  in kvm_get_msr_common()
4255 			offset = vcpu->arch.tsc_offset;  in kvm_get_msr_common()
4256 			ratio = vcpu->arch.tsc_scaling_ratio;  in kvm_get_msr_common()
4263 		msr_info->data = vcpu->arch.pat;  in kvm_get_msr_common()
4268 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4287 		msr_info->data = kvm_get_apic_base(vcpu);  in kvm_get_msr_common()
4290 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4292 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);  in kvm_get_msr_common()
4295 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;  in kvm_get_msr_common()
4298 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;  in kvm_get_msr_common()
4303 		msr_info->data = vcpu->arch.smbase;  in kvm_get_msr_common()
4306 		msr_info->data = vcpu->arch.smi_count;  in kvm_get_msr_common()
4315 		msr_info->data = vcpu->arch.efer;  in kvm_get_msr_common()
4318 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4321 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4324 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4327 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4330 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4333 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4336 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4339 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4342 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_get_msr_common()
4345 		msr_info->data = vcpu->arch.apf.msr_en_val;  in kvm_get_msr_common()
4348 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4351 		msr_info->data = vcpu->arch.apf.msr_int_val;  in kvm_get_msr_common()
4354 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4360 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_get_msr_common()
4363 		msr_info->data = vcpu->arch.st.msr_val;  in kvm_get_msr_common()
4366 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_get_msr_common()
4369 		msr_info->data = vcpu->arch.pv_eoi.msr_val;  in kvm_get_msr_common()
4372 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_get_msr_common()
4375 		msr_info->data = vcpu->arch.msr_kvm_poll_control;  in kvm_get_msr_common()
4384 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,  in kvm_get_msr_common()
4388 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_get_msr_common()
4390 		msr_info->data = vcpu->arch.ia32_xss;  in kvm_get_msr_common()
4415 		return kvm_hv_get_msr_common(vcpu,  in kvm_get_msr_common()
4433 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4435 		msr_info->data = vcpu->arch.osvw.length;  in kvm_get_msr_common()
4438 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4440 		msr_info->data = vcpu->arch.osvw.status;  in kvm_get_msr_common()
4444 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)  in kvm_get_msr_common()
4446 		msr_info->data = vcpu->arch.msr_platform_info;  in kvm_get_msr_common()
4449 		msr_info->data = vcpu->arch.msr_misc_features_enables;  in kvm_get_msr_common()
4452 		msr_info->data = vcpu->arch.msr_hwcr;  in kvm_get_msr_common()
4457 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4460 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;  in kvm_get_msr_common()
4464 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4467 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;  in kvm_get_msr_common()
4471 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4472 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4495 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,  in __msr_io()  argument
4497 		    int (*do_msr)(struct kvm_vcpu *vcpu,  in __msr_io()  argument
4503 		if (do_msr(vcpu, entries[i].index, &entries[i].data))  in __msr_io()
4514 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,  in msr_io()  argument
4515 		  int (*do_msr)(struct kvm_vcpu *vcpu,  in msr_io()  argument
4539 	r = __msr_io(vcpu, &msrs, entries, do_msr);  in msr_io()
4557 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,  in kvm_ioctl_get_supported_hv_cpuid()  argument
4567 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_ioctl_get_supported_hv_cpuid()
4954 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)  in need_emulate_wbinvd()  argument
4956 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);  in need_emulate_wbinvd()
4959 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_vcpu_load()  argument
4962 	if (need_emulate_wbinvd(vcpu)) {  in kvm_arch_vcpu_load()
4964 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_load()
4965 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4966 			smp_call_function_single(vcpu->cpu,  in kvm_arch_vcpu_load()
4970 	static_call(kvm_x86_vcpu_load)(vcpu, cpu);  in kvm_arch_vcpu_load()
4973 	vcpu->arch.host_pkru = read_pkru();  in kvm_arch_vcpu_load()
4976 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {  in kvm_arch_vcpu_load()
4977 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);  in kvm_arch_vcpu_load()
4978 		vcpu->arch.tsc_offset_adjustment = 0;  in kvm_arch_vcpu_load()
4979 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4982 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {  in kvm_arch_vcpu_load()
4983 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :  in kvm_arch_vcpu_load()
4984 				rdtsc() - vcpu->arch.last_host_tsc;  in kvm_arch_vcpu_load()
4989 			u64 offset = kvm_compute_l1_tsc_offset(vcpu,  in kvm_arch_vcpu_load()
4990 						vcpu->arch.last_guest_tsc);  in kvm_arch_vcpu_load()
4991 			kvm_vcpu_write_tsc_offset(vcpu, offset);  in kvm_arch_vcpu_load()
4992 			vcpu->arch.tsc_catchup = 1;  in kvm_arch_vcpu_load()
4995 		if (kvm_lapic_hv_timer_in_use(vcpu))  in kvm_arch_vcpu_load()
4996 			kvm_lapic_restart_hv_timer(vcpu);  in kvm_arch_vcpu_load()
5000 		 * kvmclock on vcpu->cpu migration  in kvm_arch_vcpu_load()
5002 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)  in kvm_arch_vcpu_load()
5003 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
5004 		if (vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
5005 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);  in kvm_arch_vcpu_load()
5006 		vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
5009 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_arch_vcpu_load()
5012 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)  in kvm_steal_time_set_preempted()  argument
5014 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in kvm_steal_time_set_preempted()
5018 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in kvm_steal_time_set_preempted()
5021 	 * The vCPU can be marked preempted if and only if the VM-Exit was on  in kvm_steal_time_set_preempted()
5024 	 * when this is true, for example allowing the vCPU to be marked  in kvm_steal_time_set_preempted()
5027 	if (!vcpu->arch.at_instruction_boundary) {  in kvm_steal_time_set_preempted()
5028 		vcpu->stat.preemption_other++;  in kvm_steal_time_set_preempted()
5032 	vcpu->stat.preemption_reported++;  in kvm_steal_time_set_preempted()
5033 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in kvm_steal_time_set_preempted()
5036 	if (vcpu->arch.st.preempted)  in kvm_steal_time_set_preempted()
5040 	if (unlikely(current->mm != vcpu->kvm->mm))  in kvm_steal_time_set_preempted()
5043 	slots = kvm_memslots(vcpu->kvm);  in kvm_steal_time_set_preempted()
5054 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;  in kvm_steal_time_set_preempted()
5056 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in kvm_steal_time_set_preempted()
5059 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_put()  argument
5063 	if (vcpu->preempted) {  in kvm_arch_vcpu_put()
5064 		if (!vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_put()
5065 			vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);  in kvm_arch_vcpu_put()
5071 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_put()
5072 		if (kvm_xen_msr_enabled(vcpu->kvm))  in kvm_arch_vcpu_put()
5073 			kvm_xen_runstate_set_preempted(vcpu);  in kvm_arch_vcpu_put()
5075 			kvm_steal_time_set_preempted(vcpu);  in kvm_arch_vcpu_put()
5076 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_put()
5079 	static_call(kvm_x86_vcpu_put)(vcpu);  in kvm_arch_vcpu_put()
5080 	vcpu->arch.last_host_tsc = rdtsc();  in kvm_arch_vcpu_put()
5083 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_get_lapic()  argument
5086 	static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in kvm_vcpu_ioctl_get_lapic()
5088 	return kvm_apic_get_state(vcpu, s);  in kvm_vcpu_ioctl_get_lapic()
5091 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_set_lapic()  argument
5096 	r = kvm_apic_set_state(vcpu, s);  in kvm_vcpu_ioctl_set_lapic()
5099 	update_cr8_intercept(vcpu);  in kvm_vcpu_ioctl_set_lapic()
5104 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)  in kvm_cpu_accept_dm_intr()  argument
5112 	if (kvm_cpu_has_extint(vcpu))  in kvm_cpu_accept_dm_intr()
5116 	return (!lapic_in_kernel(vcpu) ||  in kvm_cpu_accept_dm_intr()
5117 		kvm_apic_accept_pic_intr(vcpu));  in kvm_cpu_accept_dm_intr()
5120 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)  in kvm_vcpu_ready_for_interrupt_injection()  argument
5129 	return (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
5130 		kvm_cpu_accept_dm_intr(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
5131 		!kvm_event_needs_reinjection(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
5132 		!kvm_is_exception_pending(vcpu));  in kvm_vcpu_ready_for_interrupt_injection()
5135 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_interrupt()  argument
5141 	if (!irqchip_in_kernel(vcpu->kvm)) {  in kvm_vcpu_ioctl_interrupt()
5142 		kvm_queue_interrupt(vcpu, irq->irq, false);  in kvm_vcpu_ioctl_interrupt()
5143 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5151 	if (pic_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_interrupt()
5154 	if (vcpu->arch.pending_external_vector != -1)  in kvm_vcpu_ioctl_interrupt()
5157 	vcpu->arch.pending_external_vector = irq->irq;  in kvm_vcpu_ioctl_interrupt()
5158 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5162 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)  in kvm_vcpu_ioctl_nmi()  argument
5164 	kvm_inject_nmi(vcpu);  in kvm_vcpu_ioctl_nmi()
5169 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,  in vcpu_ioctl_tpr_access_reporting()  argument
5174 	vcpu->arch.tpr_access_reporting = !!tac->enabled;  in vcpu_ioctl_tpr_access_reporting()
5178 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_setup_mce()  argument
5190 	vcpu->arch.mcg_cap = mcg_cap;  in kvm_vcpu_ioctl_x86_setup_mce()
5193 		vcpu->arch.mcg_ctl = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5196 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5198 			vcpu->arch.mci_ctl2_banks[bank] = 0;  in kvm_vcpu_ioctl_x86_setup_mce()
5201 	kvm_apic_after_set_mcg_cap(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5203 	static_call(kvm_x86_setup_mce)(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5225 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)  in kvm_vcpu_x86_set_ucna()  argument
5227 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_x86_set_ucna()
5232 	vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_x86_set_ucna()
5235 	    !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))  in kvm_vcpu_x86_set_ucna()
5238 	if (lapic_in_kernel(vcpu))  in kvm_vcpu_x86_set_ucna()
5239 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);  in kvm_vcpu_x86_set_ucna()
5244 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_mce()  argument
5247 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_ioctl_x86_set_mce()
5249 	u64 *banks = vcpu->arch.mce_banks;  in kvm_vcpu_ioctl_x86_set_mce()
5257 		return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);  in kvm_vcpu_ioctl_x86_set_mce()
5264 	    vcpu->arch.mcg_ctl != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
5273 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||  in kvm_vcpu_ioctl_x86_set_mce()
5274 		    !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) {  in kvm_vcpu_ioctl_x86_set_mce()
5275 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_mce()
5282 		vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_ioctl_x86_set_mce()
5284 		kvm_queue_exception(vcpu, MC_VECTOR);  in kvm_vcpu_ioctl_x86_set_mce()
5297 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_vcpu_events()  argument
5302 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5305 	if (kvm_check_request(KVM_REQ_SMI, vcpu))  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5306 		process_smi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5316 	if (vcpu->arch.exception_vmexit.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5317 	    !vcpu->arch.exception.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5318 	    !vcpu->arch.exception.injected)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5319 		ex = &vcpu->arch.exception_vmexit;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5321 		ex = &vcpu->arch.exception;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5331 	if (!vcpu->kvm->arch.exception_payload_enabled &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5333 		kvm_deliver_exception_payload(vcpu, ex);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5351 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5361 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5362 	events->interrupt.nr = vcpu->arch.interrupt.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5363 	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5365 	events->nmi.injected = vcpu->arch.nmi_injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5366 	events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5367 	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5372 	events->smi.smm = is_smm(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5373 	events->smi.pending = vcpu->arch.smi_pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5375 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5377 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5382 	if (vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5384 	if (vcpu->kvm->arch.triple_fault_event) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5385 		events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5390 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()  argument
5402 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5420 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5423 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5433 	vcpu->arch.exception_from_userspace = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5435 	vcpu->arch.exception_vmexit.pending = false;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5437 	vcpu->arch.exception.injected = events->exception.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5438 	vcpu->arch.exception.pending = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5439 	vcpu->arch.exception.vector = events->exception.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5440 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5441 	vcpu->arch.exception.error_code = events->exception.error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5442 	vcpu->arch.exception.has_payload = events->exception_has_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5443 	vcpu->arch.exception.payload = events->exception_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5445 	vcpu->arch.interrupt.injected = events->interrupt.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5446 	vcpu->arch.interrupt.nr = events->interrupt.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5447 	vcpu->arch.interrupt.soft = events->interrupt.soft;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5449 		static_call(kvm_x86_set_interrupt_shadow)(vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5452 	vcpu->arch.nmi_injected = events->nmi.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5454 		vcpu->arch.nmi_pending = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5455 		atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5457 			kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5459 	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5462 	    lapic_in_kernel(vcpu))  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5463 		vcpu->arch.apic->sipi_vector = events->sipi_vector;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5467 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5468 			kvm_leave_nested(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5469 			kvm_smm_changed(vcpu, events->smi.smm);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5472 		vcpu->arch.smi_pending = events->smi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5476 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5478 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5487 		if (lapic_in_kernel(vcpu)) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5489 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5491 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5496 		if (!vcpu->kvm->arch.triple_fault_event)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5499 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5501 			kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5504 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5509 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_debugregs()  argument
5515 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_get_debugregs()
5516 	kvm_get_dr(vcpu, 6, &val);  in kvm_vcpu_ioctl_x86_get_debugregs()
5518 	dbgregs->dr7 = vcpu->arch.dr7;  in kvm_vcpu_ioctl_x86_get_debugregs()
5521 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_debugregs()  argument
5532 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));  in kvm_vcpu_ioctl_x86_set_debugregs()
5533 	kvm_update_dr0123(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5534 	vcpu->arch.dr6 = dbgregs->dr6;  in kvm_vcpu_ioctl_x86_set_debugregs()
5535 	vcpu->arch.dr7 = dbgregs->dr7;  in kvm_vcpu_ioctl_x86_set_debugregs()
5536 	kvm_update_dr7(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5542 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave2()  argument
5557 	u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |  in kvm_vcpu_ioctl_x86_get_xsave2()
5560 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_get_xsave2()
5563 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,  in kvm_vcpu_ioctl_x86_get_xsave2()
5564 				       supported_xcr0, vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_get_xsave2()
5567 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave()  argument
5570 	kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,  in kvm_vcpu_ioctl_x86_get_xsave()
5574 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xsave()  argument
5577 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_set_xsave()
5580 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,  in kvm_vcpu_ioctl_x86_set_xsave()
5583 					      &vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_set_xsave()
5586 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xcrs()  argument
5597 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5600 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xcrs()  argument
5614 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,  in kvm_vcpu_ioctl_x86_set_xcrs()
5629 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)  in kvm_set_guest_paused()  argument
5631 	if (!vcpu->arch.pv_time.active)  in kvm_set_guest_paused()
5633 	vcpu->arch.pvclock_set_guest_stopped_request = true;  in kvm_set_guest_paused()
5634 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_guest_paused()
5638 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_has_attr()  argument
5654 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_get_attr()  argument
5666 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))  in kvm_arch_tsc_get_attr()
5677 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_set_attr()  argument
5681 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_tsc_set_attr()
5699 		matched = (vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5700 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5703 		tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;  in kvm_arch_tsc_set_attr()
5707 		__kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);  in kvm_arch_tsc_set_attr()
5720 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_device_attr()  argument
5735 		r = kvm_arch_tsc_has_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5738 		r = kvm_arch_tsc_get_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5741 		r = kvm_arch_tsc_set_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5748 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_enable_cap()  argument
5762 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_enable_cap()
5764 		return kvm_hv_activate_synic(vcpu, cap->cap ==  in kvm_vcpu_ioctl_enable_cap()
5774 			r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);  in kvm_vcpu_ioctl_enable_cap()
5787 		return static_call(kvm_x86_enable_l2_tlb_flush)(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5790 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);  in kvm_vcpu_ioctl_enable_cap()
5794 		vcpu->arch.pv_cpuid.enforce = cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
5795 		if (vcpu->arch.pv_cpuid.enforce)  in kvm_vcpu_ioctl_enable_cap()
5796 			kvm_update_pv_runtime(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5807 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()  local
5818 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl()
5824 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5832 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5843 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5851 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5860 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);  in kvm_arch_vcpu_ioctl()
5864 		r = kvm_vcpu_ioctl_nmi(vcpu);  in kvm_arch_vcpu_ioctl()
5868 		r = kvm_inject_smi(vcpu);  in kvm_arch_vcpu_ioctl()
5878 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5888 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5899 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5910 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5911 		r = msr_io(vcpu, argp, do_get_msr, 1);  in kvm_arch_vcpu_ioctl()
5912 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5916 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5917 		r = msr_io(vcpu, argp, do_set_msr, 0);  in kvm_arch_vcpu_ioctl()
5918 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5927 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);  in kvm_arch_vcpu_ioctl()
5941 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5946 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5947 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);  in kvm_arch_vcpu_ioctl()
5948 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5957 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);  in kvm_arch_vcpu_ioctl()
5966 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);  in kvm_arch_vcpu_ioctl()
5972 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
5987 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
5993 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
6010 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
6015 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))  in kvm_arch_vcpu_ioctl()
6023 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
6032 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
6040 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
6045 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
6052 		kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);  in kvm_arch_vcpu_ioctl()
6068 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
6084 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
6100 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))  in kvm_arch_vcpu_ioctl()
6106 		r = vcpu->arch.virtual_tsc_khz;  in kvm_arch_vcpu_ioctl()
6110 		r = kvm_set_guest_paused(vcpu);  in kvm_arch_vcpu_ioctl()
6119 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);  in kvm_arch_vcpu_ioctl()
6135 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,  in kvm_arch_vcpu_ioctl()
6179 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
6180 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);  in kvm_arch_vcpu_ioctl()
6181 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
6186 		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);  in kvm_arch_vcpu_ioctl()
6196 		r = kvm_xen_vcpu_get_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6207 		r = kvm_xen_vcpu_set_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6216 		__get_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6230 		r = __set_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6236 		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);  in kvm_arch_vcpu_ioctl()
6244 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl()
6248 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)  in kvm_arch_vcpu_fault()  argument
6422 	struct kvm_vcpu *vcpu;  in kvm_arch_sync_dirty_log()  local
6428 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_sync_dirty_log()
6429 		kvm_vcpu_kick(vcpu);  in kvm_arch_sync_dirty_log()
6843 	struct kvm_vcpu *vcpu;  in kvm_arch_suspend_notifier()  local
6848 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_suspend_notifier()
6849 		if (!vcpu->arch.pv_time.active)  in kvm_arch_suspend_notifier()
6852 		ret = kvm_set_guest_paused(vcpu);  in kvm_arch_suspend_notifier()
6854 			kvm_err("Failed to pause guest VCPU%d: %d\n",  in kvm_arch_suspend_notifier()
6855 				vcpu->vcpu_id, ret);  in kvm_arch_suspend_notifier()
7410 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,  in vcpu_mmio_write()  argument
7418 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_write()
7419 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))  in vcpu_mmio_write()
7420 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_write()
7431 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)  in vcpu_mmio_read()  argument
7438 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_read()
7439 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,  in vcpu_mmio_read()
7441 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_read()
7453 void kvm_set_segment(struct kvm_vcpu *vcpu,  in kvm_set_segment()  argument
7456 	static_call(kvm_x86_set_segment)(vcpu, var, seg);  in kvm_set_segment()
7459 void kvm_get_segment(struct kvm_vcpu *vcpu,  in kvm_get_segment()  argument
7462 	static_call(kvm_x86_get_segment)(vcpu, var, seg);  in kvm_get_segment()
7465 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,  in translate_nested_gpa()  argument
7468 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in translate_nested_gpa()
7471 	BUG_ON(!mmu_is_nested(vcpu));  in translate_nested_gpa()
7475 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);  in translate_nested_gpa()
7480 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_read()  argument
7483 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_read()
7485 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_read()
7486 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_read()
7490 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_write()  argument
7493 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_write()
7495 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_write()
7497 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_write()
7502 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_system()  argument
7505 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_system()
7507 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);  in kvm_mmu_gva_to_gpa_system()
7511 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_read_guest_virt_helper()  argument
7514 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_read_guest_virt_helper()
7519 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_read_guest_virt_helper()
7526 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,  in kvm_read_guest_virt_helper()
7546 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_fetch_guest_virt()  local
7547 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fetch_guest_virt()
7548 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_fetch_guest_virt()
7553 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,  in kvm_fetch_guest_virt()
7561 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,  in kvm_fetch_guest_virt()
7569 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,  in kvm_read_guest_virt()  argument
7573 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_read_guest_virt()
7582 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,  in kvm_read_guest_virt()
7591 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_std()  local
7596 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)  in emulator_read_std()
7599 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);  in emulator_read_std()
7603 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_write_guest_virt_helper()  argument
7606 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_write_guest_virt_helper()
7611 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_write_guest_virt_helper()
7618 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);  in kvm_write_guest_virt_helper()
7636 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_write_std()  local
7641 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)  in emulator_write_std()
7644 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in emulator_write_std()
7648 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,  in kvm_write_guest_virt_system()  argument
7652 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_write_guest_virt_system()
7654 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in kvm_write_guest_virt_system()
7659 static int kvm_check_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,  in kvm_check_emulate_insn()  argument
7662 	return static_call(kvm_x86_check_emulate_instruction)(vcpu, emul_type,  in kvm_check_emulate_insn()
7666 int handle_ud(struct kvm_vcpu *vcpu)  in handle_ud()  argument
7675 	r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0);  in handle_ud()
7680 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),  in handle_ud()
7684 			kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);  in handle_ud()
7685 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));  in handle_ud()
7689 	return kvm_emulate_instruction(vcpu, emul_type);  in handle_ud()
7693 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_is_mmio_gpa()  argument
7700 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {  in vcpu_is_mmio_gpa()
7708 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_mmio_gva_to_gpa()  argument
7712 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in vcpu_mmio_gva_to_gpa()
7713 	u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)  in vcpu_mmio_gva_to_gpa()
7721 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||  in vcpu_mmio_gva_to_gpa()
7722 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,  in vcpu_mmio_gva_to_gpa()
7723 			      vcpu->arch.mmio_access, 0, access))) {  in vcpu_mmio_gva_to_gpa()
7724 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |  in vcpu_mmio_gva_to_gpa()
7730 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in vcpu_mmio_gva_to_gpa()
7735 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);  in vcpu_mmio_gva_to_gpa()
7738 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,  in emulator_write_phys()  argument
7743 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);  in emulator_write_phys()
7746 	kvm_page_track_write(vcpu, gpa, val, bytes);  in emulator_write_phys()
7751 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7753 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7755 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7757 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7762 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)  in read_prepare()  argument
7764 	if (vcpu->mmio_read_completed) {  in read_prepare()
7766 			       vcpu->mmio_fragments[0].gpa, val);  in read_prepare()
7767 		vcpu->mmio_read_completed = 0;  in read_prepare()
7774 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_emulate()  argument
7777 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);  in read_emulate()
7780 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_emulate()  argument
7783 	return emulator_write_phys(vcpu, gpa, val, bytes);  in write_emulate()
7786 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)  in write_mmio()  argument
7789 	return vcpu_mmio_write(vcpu, gpa, bytes, val);  in write_mmio()
7792 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_exit_mmio()  argument
7799 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_exit_mmio()  argument
7802 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];  in write_exit_mmio()
7804 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in write_exit_mmio()
7825 				       struct kvm_vcpu *vcpu,  in emulator_read_write_onepage()  argument
7832 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()
7844 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);  in emulator_read_write_onepage()
7846 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);  in emulator_read_write_onepage()
7851 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))  in emulator_read_write_onepage()
7857 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);  in emulator_read_write_onepage()
7865 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);  in emulator_read_write_onepage()
7866 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];  in emulator_read_write_onepage()
7879 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_write()  local
7884 		  ops->read_write_prepare(vcpu, val, bytes))  in emulator_read_write()
7887 	vcpu->mmio_nr_fragments = 0;  in emulator_read_write()
7895 						 vcpu, ops);  in emulator_read_write()
7907 					 vcpu, ops);  in emulator_read_write()
7911 	if (!vcpu->mmio_nr_fragments)  in emulator_read_write()
7914 	gpa = vcpu->mmio_fragments[0].gpa;  in emulator_read_write()
7916 	vcpu->mmio_needed = 1;  in emulator_read_write()
7917 	vcpu->mmio_cur_fragment = 0;  in emulator_read_write()
7919 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);  in emulator_read_write()
7920 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;  in emulator_read_write()
7921 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in emulator_read_write()
7922 	vcpu->run->mmio.phys_addr = gpa;  in emulator_read_write()
7924 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);  in emulator_read_write()
7957 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_cmpxchg_emulated()  local
7967 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);  in emulator_cmpxchg_emulated()
7985 	hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
8018 	kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
8023 	kvm_page_track_write(vcpu, gpa, new, bytes);  in emulator_cmpxchg_emulated()
8033 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in_out()  argument
8040 	WARN_ON_ONCE(vcpu->arch.pio.count);  in emulator_pio_in_out()
8043 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
8045 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
8065 	vcpu->arch.pio.port = port;  in emulator_pio_in_out()
8066 	vcpu->arch.pio.in = in;  in emulator_pio_in_out()
8067 	vcpu->arch.pio.count = count;  in emulator_pio_in_out()
8068 	vcpu->arch.pio.size = size;  in emulator_pio_in_out()
8071 		memset(vcpu->arch.pio_data, 0, size * count);  in emulator_pio_in_out()
8073 		memcpy(vcpu->arch.pio_data, data, size * count);  in emulator_pio_in_out()
8075 	vcpu->run->exit_reason = KVM_EXIT_IO;  in emulator_pio_in_out()
8076 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;  in emulator_pio_in_out()
8077 	vcpu->run->io.size = size;  in emulator_pio_in_out()
8078 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;  in emulator_pio_in_out()
8079 	vcpu->run->io.count = count;  in emulator_pio_in_out()
8080 	vcpu->run->io.port = port;  in emulator_pio_in_out()
8084 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in()  argument
8087 	int r = emulator_pio_in_out(vcpu, size, port, val, count, true);  in emulator_pio_in()
8094 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)  in complete_emulator_pio_in()  argument
8096 	int size = vcpu->arch.pio.size;  in complete_emulator_pio_in()
8097 	unsigned int count = vcpu->arch.pio.count;  in complete_emulator_pio_in()
8098 	memcpy(val, vcpu->arch.pio_data, size * count);  in complete_emulator_pio_in()
8099 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);  in complete_emulator_pio_in()
8100 	vcpu->arch.pio.count = 0;  in complete_emulator_pio_in()
8107 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_pio_in_emulated()  local
8108 	if (vcpu->arch.pio.count) {  in emulator_pio_in_emulated()
8112 		 * can modify ECX before rerunning the vCPU.  Ignore any such  in emulator_pio_in_emulated()
8116 		complete_emulator_pio_in(vcpu, val);  in emulator_pio_in_emulated()
8120 	return emulator_pio_in(vcpu, size, port, val, count);  in emulator_pio_in_emulated()
8123 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_out()  argument
8128 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);  in emulator_pio_out()
8138 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)  in get_segment_base()  argument
8140 	return static_call(kvm_x86_get_segment_base)(vcpu, seg);  in get_segment_base()
8148 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd_noskip()  argument
8150 	if (!need_emulate_wbinvd(vcpu))  in kvm_emulate_wbinvd_noskip()
8156 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
8157 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,  in kvm_emulate_wbinvd_noskip()
8160 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
8166 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd()  argument
8168 	kvm_emulate_wbinvd_noskip(vcpu);  in kvm_emulate_wbinvd()
8169 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_wbinvd()
8200 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_cr()  local
8205 		value = kvm_read_cr0(vcpu);  in emulator_get_cr()
8208 		value = vcpu->arch.cr2;  in emulator_get_cr()
8211 		value = kvm_read_cr3(vcpu);  in emulator_get_cr()
8214 		value = kvm_read_cr4(vcpu);  in emulator_get_cr()
8217 		value = kvm_get_cr8(vcpu);  in emulator_get_cr()
8229 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_cr()  local
8234 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));  in emulator_set_cr()
8237 		vcpu->arch.cr2 = val;  in emulator_set_cr()
8240 		res = kvm_set_cr3(vcpu, val);  in emulator_set_cr()
8243 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));  in emulator_set_cr()
8246 		res = kvm_set_cr8(vcpu, val);  in emulator_set_cr()
8327 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_segment()  local
8349 	kvm_set_segment(vcpu, &var, seg);  in emulator_set_segment()
8356 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_msr_with_filter()  local
8359 	r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);  in emulator_get_msr_with_filter()
8364 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,  in emulator_get_msr_with_filter()
8379 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_msr_with_filter()  local
8382 	r = kvm_set_msr_with_filter(vcpu, msr_index, data);  in emulator_set_msr_with_filter()
8387 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,  in emulator_set_msr_with_filter()
8561 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)  in toggle_interruptibility()  argument
8563 	u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in toggle_interruptibility()
8574 		static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);  in toggle_interruptibility()
8576 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in toggle_interruptibility()
8580 static void inject_emulated_exception(struct kvm_vcpu *vcpu)  in inject_emulated_exception()  argument
8582 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()
8585 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
8587 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
8590 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
8593 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)  in alloc_emulate_ctxt()  argument
8599 		pr_err("failed to allocate vcpu's emulator\n");  in alloc_emulate_ctxt()
8603 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
8605 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
8610 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)  in init_emulate_ctxt()  argument
8612 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()
8615 	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in init_emulate_ctxt()
8618 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
8621 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
8622 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
8624 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :  in init_emulate_ctxt()
8633 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in init_emulate_ctxt()
8636 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)  in kvm_inject_realmode_interrupt()  argument
8638 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()
8641 	init_emulate_ctxt(vcpu);  in kvm_inject_realmode_interrupt()
8649 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_inject_realmode_interrupt()
8652 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
8653 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
8658 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in prepare_emulation_failure_exit()  argument
8661 	struct kvm_run *run = vcpu->run;  in prepare_emulation_failure_exit()
8671 	static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],  in prepare_emulation_failure_exit()
8709 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)  in prepare_emulation_ctxt_failure_exit()  argument
8711 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in prepare_emulation_ctxt_failure_exit()
8713 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,  in prepare_emulation_ctxt_failure_exit()
8717 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in __kvm_prepare_emulation_failure_exit()  argument
8720 	prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);  in __kvm_prepare_emulation_failure_exit()
8724 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)  in kvm_prepare_emulation_failure_exit()  argument
8726 	__kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);  in kvm_prepare_emulation_failure_exit()
8730 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)  in handle_emulation_failure()  argument
8732 	struct kvm *kvm = vcpu->kvm;  in handle_emulation_failure()
8734 	++vcpu->stat.insn_emulation_fail;  in handle_emulation_failure()
8735 	trace_kvm_emulate_insn_failed(vcpu);  in handle_emulation_failure()
8738 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in handle_emulation_failure()
8744 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8748 	kvm_queue_exception(vcpu, UD_VECTOR);  in handle_emulation_failure()
8750 	if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {  in handle_emulation_failure()
8751 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8758 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in reexecute_instruction()  argument
8767 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in reexecute_instruction()
8771 	if (!vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8776 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in reexecute_instruction()
8792 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8804 	if (vcpu->arch.mmu->root_role.direct) {  in reexecute_instruction()
8807 		write_lock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8808 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;  in reexecute_instruction()
8809 		write_unlock(&vcpu->kvm->mmu_lock);  in reexecute_instruction()
8812 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8822 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in reexecute_instruction()
8835 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in retry_instruction()  local
8838 	last_retry_eip = vcpu->arch.last_retry_eip;  in retry_instruction()
8839 	last_retry_addr = vcpu->arch.last_retry_addr;  in retry_instruction()
8854 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;  in retry_instruction()
8859 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in retry_instruction()
8869 	vcpu->arch.last_retry_eip = ctxt->eip;  in retry_instruction()
8870 	vcpu->arch.last_retry_addr = cr2_or_gpa;  in retry_instruction()
8872 	if (!vcpu->arch.mmu->root_role.direct)  in retry_instruction()
8873 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);  in retry_instruction()
8875 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));  in retry_instruction()
8880 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8881 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8898 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)  in kvm_vcpu_do_singlestep()  argument
8900 	struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_do_singlestep()
8902 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {  in kvm_vcpu_do_singlestep()
8904 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);  in kvm_vcpu_do_singlestep()
8909 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);  in kvm_vcpu_do_singlestep()
8913 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)  in kvm_skip_emulated_instruction()  argument
8915 	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);  in kvm_skip_emulated_instruction()
8918 	r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);  in kvm_skip_emulated_instruction()
8922 	kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);  in kvm_skip_emulated_instruction()
8933 		r = kvm_vcpu_do_singlestep(vcpu);  in kvm_skip_emulated_instruction()
8938 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)  in kvm_is_code_breakpoint_inhibited()  argument
8942 	if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)  in kvm_is_code_breakpoint_inhibited()
8950 	shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);  in kvm_is_code_breakpoint_inhibited()
8952 	       guest_cpuid_is_intel(vcpu);  in kvm_is_code_breakpoint_inhibited()
8955 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,  in kvm_vcpu_check_code_breakpoint()  argument
8978 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&  in kvm_vcpu_check_code_breakpoint()
8979 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {  in kvm_vcpu_check_code_breakpoint()
8980 		struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_check_code_breakpoint()
8981 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
8983 					   vcpu->arch.guest_debug_dr7,  in kvm_vcpu_check_code_breakpoint()
8984 					   vcpu->arch.eff_db);  in kvm_vcpu_check_code_breakpoint()
8996 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&  in kvm_vcpu_check_code_breakpoint()
8997 	    !kvm_is_code_breakpoint_inhibited(vcpu)) {  in kvm_vcpu_check_code_breakpoint()
8998 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
9000 					   vcpu->arch.dr7,  in kvm_vcpu_check_code_breakpoint()
9001 					   vcpu->arch.db);  in kvm_vcpu_check_code_breakpoint()
9004 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);  in kvm_vcpu_check_code_breakpoint()
9054 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,  in x86_decode_emulated_instruction()  argument
9057 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_decode_emulated_instruction()
9060 	init_emulate_ctxt(vcpu);  in x86_decode_emulated_instruction()
9064 	trace_kvm_emulate_insn_start(vcpu);  in x86_decode_emulated_instruction()
9065 	++vcpu->stat.insn_emulation;  in x86_decode_emulated_instruction()
9071 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in x86_emulate_instruction()  argument
9075 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()
9078 	r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len);  in x86_emulate_instruction()
9084 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9087 	vcpu->arch.l1tf_flush_l1d = true;  in x86_emulate_instruction()
9090 		kvm_clear_exception_queue(vcpu);  in x86_emulate_instruction()
9097 		if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))  in x86_emulate_instruction()
9100 		r = x86_decode_emulated_instruction(vcpu, emulation_type,  in x86_emulate_instruction()
9105 				kvm_queue_exception(vcpu, UD_VECTOR);  in x86_emulate_instruction()
9108 			if (reexecute_instruction(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
9120 				inject_emulated_exception(vcpu);  in x86_emulate_instruction()
9123 			return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9129 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in x86_emulate_instruction()
9150 		kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9152 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
9161 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {  in x86_emulate_instruction()
9162 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in x86_emulate_instruction()
9172 		if (vcpu->arch.mmu->root_role.direct) {  in x86_emulate_instruction()
9187 		if (reexecute_instruction(vcpu, cr2_or_gpa, emulation_type))  in x86_emulate_instruction()
9190 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9194 		WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);  in x86_emulate_instruction()
9195 		vcpu->mmio_needed = false;  in x86_emulate_instruction()
9197 		inject_emulated_exception(vcpu);  in x86_emulate_instruction()
9198 	} else if (vcpu->arch.pio.count) {  in x86_emulate_instruction()
9199 		if (!vcpu->arch.pio.in) {  in x86_emulate_instruction()
9201 			vcpu->arch.pio.count = 0;  in x86_emulate_instruction()
9204 			vcpu->arch.complete_userspace_io = complete_emulated_pio;  in x86_emulate_instruction()
9207 	} else if (vcpu->mmio_needed) {  in x86_emulate_instruction()
9208 		++vcpu->stat.mmio_exits;  in x86_emulate_instruction()
9210 		if (!vcpu->mmio_is_write)  in x86_emulate_instruction()
9213 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in x86_emulate_instruction()
9214 	} else if (vcpu->arch.complete_userspace_io) {  in x86_emulate_instruction()
9224 		unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);  in x86_emulate_instruction()
9225 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
9226 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in x86_emulate_instruction()
9235 			kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);  in x86_emulate_instruction()
9237 				kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);  in x86_emulate_instruction()
9238 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9239 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
9240 				r = kvm_vcpu_do_singlestep(vcpu);  in x86_emulate_instruction()
9241 			static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);  in x86_emulate_instruction()
9242 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
9252 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in x86_emulate_instruction()
9254 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;  in x86_emulate_instruction()
9259 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)  in kvm_emulate_instruction()  argument
9261 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);  in kvm_emulate_instruction()
9265 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,  in kvm_emulate_instruction_from_buffer()  argument
9268 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);  in kvm_emulate_instruction_from_buffer()
9272 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)  in complete_fast_pio_out_port_0x7e()  argument
9274 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out_port_0x7e()
9278 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)  in complete_fast_pio_out()  argument
9280 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out()
9282 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))  in complete_fast_pio_out()
9285 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_out()
9288 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_out()  argument
9291 	unsigned long val = kvm_rax_read(vcpu);  in kvm_fast_pio_out()
9292 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);  in kvm_fast_pio_out()
9302 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {  in kvm_fast_pio_out()
9303 		vcpu->arch.complete_userspace_io =  in kvm_fast_pio_out()
9305 		kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio_out()
9307 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_out()
9308 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;  in kvm_fast_pio_out()
9313 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)  in complete_fast_pio_in()  argument
9318 	BUG_ON(vcpu->arch.pio.count != 1);  in complete_fast_pio_in()
9320 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {  in complete_fast_pio_in()
9321 		vcpu->arch.pio.count = 0;  in complete_fast_pio_in()
9326 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;  in complete_fast_pio_in()
9328 	complete_emulator_pio_in(vcpu, &val);  in complete_fast_pio_in()
9329 	kvm_rax_write(vcpu, val);  in complete_fast_pio_in()
9331 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_in()
9334 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_in()  argument
9341 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;  in kvm_fast_pio_in()
9343 	ret = emulator_pio_in(vcpu, size, port, &val, 1);  in kvm_fast_pio_in()
9345 		kvm_rax_write(vcpu, val);  in kvm_fast_pio_in()
9349 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_in()
9350 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;  in kvm_fast_pio_in()
9355 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)  in kvm_fast_pio()  argument
9360 		ret = kvm_fast_pio_in(vcpu, size, port);  in kvm_fast_pio()
9362 		ret = kvm_fast_pio_out(vcpu, size, port);  in kvm_fast_pio()
9363 	return ret && kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio()
9422 	struct kvm_vcpu *vcpu;  in __kvmclock_cpufreq_notifier()  local
9437 	 * the TSC for each VCPU.  We must flag these local variables  in __kvmclock_cpufreq_notifier()
9456 	 * anytime after the setting of the VCPU's request bit, the  in __kvmclock_cpufreq_notifier()
9469 		kvm_for_each_vcpu(i, vcpu, kvm) {  in __kvmclock_cpufreq_notifier()
9470 			if (vcpu->cpu != cpu)  in __kvmclock_cpufreq_notifier()
9472 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in __kvmclock_cpufreq_notifier()
9473 			if (vcpu->cpu != raw_smp_processor_id())  in __kvmclock_cpufreq_notifier()
9553 	struct kvm_vcpu *vcpu;  in pvclock_gtod_update_fn()  local
9558 		kvm_for_each_vcpu(i, vcpu, kvm)  in pvclock_gtod_update_fn()
9559 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in pvclock_gtod_update_fn()
9661 	 * vCPU's FPU state as a fxregs_state struct.  in __kvm_x86_vendor_init()
9829 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)  in __kvm_emulate_halt()  argument
9832 	 * The vCPU has halted, e.g. executed HLT.  Update the run state if the  in __kvm_emulate_halt()
9834 	 * state and halt the vCPU.  Exit to userspace if the local APIC is  in __kvm_emulate_halt()
9838 	++vcpu->stat.halt_exits;  in __kvm_emulate_halt()
9839 	if (lapic_in_kernel(vcpu)) {  in __kvm_emulate_halt()
9840 		vcpu->arch.mp_state = state;  in __kvm_emulate_halt()
9843 		vcpu->run->exit_reason = reason;  in __kvm_emulate_halt()
9848 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_halt_noskip()  argument
9850 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);  in kvm_emulate_halt_noskip()
9854 int kvm_emulate_halt(struct kvm_vcpu *vcpu)  in kvm_emulate_halt()  argument
9856 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_halt()
9861 	return kvm_emulate_halt_noskip(vcpu) && ret;  in kvm_emulate_halt()
9865 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)  in kvm_emulate_ap_reset_hold()  argument
9867 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_ap_reset_hold()
9869 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,  in kvm_emulate_ap_reset_hold()
9875 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,  in kvm_pv_clock_pairing()  argument
9890 	if (vcpu->arch.tsc_always_catchup)  in kvm_pv_clock_pairing()
9898 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);  in kvm_pv_clock_pairing()
9903 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,  in kvm_pv_clock_pairing()
9912  * kvm_pv_kick_cpu_op:  Kick a vcpu.
9914  * @apicid - apicid of vcpu to be kicked.
9938 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)  in kvm_vcpu_apicv_activated()  argument
9940 	ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);  in kvm_vcpu_apicv_activated()
9941 	ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);  in kvm_vcpu_apicv_activated()
9971 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)  in kvm_sched_yield()  argument
9976 	vcpu->stat.directed_yield_attempted++;  in kvm_sched_yield()
9982 	map = rcu_dereference(vcpu->kvm->arch.apic_map);  in kvm_sched_yield()
9985 		target = map->phys_map[dest_id]->vcpu;  in kvm_sched_yield()
9993 	if (vcpu == target)  in kvm_sched_yield()
9999 	vcpu->stat.directed_yield_successful++;  in kvm_sched_yield()
10005 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)  in complete_hypercall_exit()  argument
10007 	u64 ret = vcpu->run->hypercall.ret;  in complete_hypercall_exit()
10009 	if (!is_64_bit_mode(vcpu))  in complete_hypercall_exit()
10011 	kvm_rax_write(vcpu, ret);  in complete_hypercall_exit()
10012 	++vcpu->stat.hypercalls;  in complete_hypercall_exit()
10013 	return kvm_skip_emulated_instruction(vcpu);  in complete_hypercall_exit()
10016 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)  in kvm_emulate_hypercall()  argument
10021 	if (kvm_xen_hypercall_enabled(vcpu->kvm))  in kvm_emulate_hypercall()
10022 		return kvm_xen_hypercall(vcpu);  in kvm_emulate_hypercall()
10024 	if (kvm_hv_hypercall_enabled(vcpu))  in kvm_emulate_hypercall()
10025 		return kvm_hv_hypercall(vcpu);  in kvm_emulate_hypercall()
10027 	nr = kvm_rax_read(vcpu);  in kvm_emulate_hypercall()
10028 	a0 = kvm_rbx_read(vcpu);  in kvm_emulate_hypercall()
10029 	a1 = kvm_rcx_read(vcpu);  in kvm_emulate_hypercall()
10030 	a2 = kvm_rdx_read(vcpu);  in kvm_emulate_hypercall()
10031 	a3 = kvm_rsi_read(vcpu);  in kvm_emulate_hypercall()
10035 	op_64_bit = is_64_bit_hypercall(vcpu);  in kvm_emulate_hypercall()
10044 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {  in kvm_emulate_hypercall()
10056 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))  in kvm_emulate_hypercall()
10059 		kvm_pv_kick_cpu_op(vcpu->kvm, a1);  in kvm_emulate_hypercall()
10060 		kvm_sched_yield(vcpu, a1);  in kvm_emulate_hypercall()
10065 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);  in kvm_emulate_hypercall()
10069 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))  in kvm_emulate_hypercall()
10072 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);  in kvm_emulate_hypercall()
10075 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))  in kvm_emulate_hypercall()
10078 		kvm_sched_yield(vcpu, a0);  in kvm_emulate_hypercall()
10085 		if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))  in kvm_emulate_hypercall()
10094 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;  in kvm_emulate_hypercall()
10095 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;  in kvm_emulate_hypercall()
10096 		vcpu->run->hypercall.args[0]  = gpa;  in kvm_emulate_hypercall()
10097 		vcpu->run->hypercall.args[1]  = npages;  in kvm_emulate_hypercall()
10098 		vcpu->run->hypercall.args[2]  = attrs;  in kvm_emulate_hypercall()
10099 		vcpu->run->hypercall.flags    = 0;  in kvm_emulate_hypercall()
10101 			vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;  in kvm_emulate_hypercall()
10103 		WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);  in kvm_emulate_hypercall()
10104 		vcpu->arch.complete_userspace_io = complete_hypercall_exit;  in kvm_emulate_hypercall()
10114 	kvm_rax_write(vcpu, ret);  in kvm_emulate_hypercall()
10116 	++vcpu->stat.hypercalls;  in kvm_emulate_hypercall()
10117 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_hypercall()
10123 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_fix_hypercall()  local
10125 	unsigned long rip = kvm_rip_read(vcpu);  in emulator_fix_hypercall()
10131 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {  in emulator_fix_hypercall()
10138 	static_call(kvm_x86_patch_hypercall)(vcpu, instruction);  in emulator_fix_hypercall()
10144 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)  in dm_request_for_irq_injection()  argument
10146 	return vcpu->run->request_interrupt_window &&  in dm_request_for_irq_injection()
10147 		likely(!pic_in_kernel(vcpu->kvm));  in dm_request_for_irq_injection()
10151 static void post_kvm_run_save(struct kvm_vcpu *vcpu)  in post_kvm_run_save()  argument
10153 	struct kvm_run *kvm_run = vcpu->run;  in post_kvm_run_save()
10155 	kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);  in post_kvm_run_save()
10156 	kvm_run->cr8 = kvm_get_cr8(vcpu);  in post_kvm_run_save()
10157 	kvm_run->apic_base = kvm_get_apic_base(vcpu);  in post_kvm_run_save()
10160 		pic_in_kernel(vcpu->kvm) ||  in post_kvm_run_save()
10161 		kvm_vcpu_ready_for_interrupt_injection(vcpu);  in post_kvm_run_save()
10163 	if (is_smm(vcpu))  in post_kvm_run_save()
10167 static void update_cr8_intercept(struct kvm_vcpu *vcpu)  in update_cr8_intercept()  argument
10174 	if (!lapic_in_kernel(vcpu))  in update_cr8_intercept()
10177 	if (vcpu->arch.apic->apicv_active)  in update_cr8_intercept()
10180 	if (!vcpu->arch.apic->vapic_addr)  in update_cr8_intercept()
10181 		max_irr = kvm_lapic_find_highest_irr(vcpu);  in update_cr8_intercept()
10188 	tpr = kvm_lapic_get_cr8(vcpu);  in update_cr8_intercept()
10190 	static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);  in update_cr8_intercept()
10194 int kvm_check_nested_events(struct kvm_vcpu *vcpu)  in kvm_check_nested_events()  argument
10196 	if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in kvm_check_nested_events()
10197 		kvm_x86_ops.nested_ops->triple_fault(vcpu);  in kvm_check_nested_events()
10201 	return kvm_x86_ops.nested_ops->check_events(vcpu);  in kvm_check_nested_events()
10204 static void kvm_inject_exception(struct kvm_vcpu *vcpu)  in kvm_inject_exception()  argument
10207 	 * Suppress the error code if the vCPU is in Real Mode, as Real Mode  in kvm_inject_exception()
10213 	vcpu->arch.exception.has_error_code &= is_protmode(vcpu);  in kvm_inject_exception()
10215 	trace_kvm_inj_exception(vcpu->arch.exception.vector,  in kvm_inject_exception()
10216 				vcpu->arch.exception.has_error_code,  in kvm_inject_exception()
10217 				vcpu->arch.exception.error_code,  in kvm_inject_exception()
10218 				vcpu->arch.exception.injected);  in kvm_inject_exception()
10220 	static_call(kvm_x86_inject_exception)(vcpu);  in kvm_inject_exception()
10262 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,  in kvm_check_and_inject_events()  argument
10273 	if (is_guest_mode(vcpu))  in kvm_check_and_inject_events()
10274 		r = kvm_check_nested_events(vcpu);  in kvm_check_and_inject_events()
10301 	if (vcpu->arch.exception.injected)  in kvm_check_and_inject_events()
10302 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10303 	else if (kvm_is_exception_pending(vcpu))  in kvm_check_and_inject_events()
10305 	else if (vcpu->arch.nmi_injected)  in kvm_check_and_inject_events()
10306 		static_call(kvm_x86_inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10307 	else if (vcpu->arch.interrupt.injected)  in kvm_check_and_inject_events()
10308 		static_call(kvm_x86_inject_irq)(vcpu, true);  in kvm_check_and_inject_events()
10315 	WARN_ON_ONCE(vcpu->arch.exception.injected &&  in kvm_check_and_inject_events()
10316 		     vcpu->arch.exception.pending);  in kvm_check_and_inject_events()
10335 	WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||  in kvm_check_and_inject_events()
10336 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10343 	can_inject = !kvm_event_needs_reinjection(vcpu);  in kvm_check_and_inject_events()
10345 	if (vcpu->arch.exception.pending) {  in kvm_check_and_inject_events()
10356 		if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)  in kvm_check_and_inject_events()
10357 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |  in kvm_check_and_inject_events()
10360 		if (vcpu->arch.exception.vector == DB_VECTOR) {  in kvm_check_and_inject_events()
10361 			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);  in kvm_check_and_inject_events()
10362 			if (vcpu->arch.dr7 & DR7_GD) {  in kvm_check_and_inject_events()
10363 				vcpu->arch.dr7 &= ~DR7_GD;  in kvm_check_and_inject_events()
10364 				kvm_update_dr7(vcpu);  in kvm_check_and_inject_events()
10368 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10370 		vcpu->arch.exception.pending = false;  in kvm_check_and_inject_events()
10371 		vcpu->arch.exception.injected = true;  in kvm_check_and_inject_events()
10377 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)  in kvm_check_and_inject_events()
10392 	if (vcpu->arch.smi_pending) {  in kvm_check_and_inject_events()
10393 		r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10397 			vcpu->arch.smi_pending = false;  in kvm_check_and_inject_events()
10398 			++vcpu->arch.smi_count;  in kvm_check_and_inject_events()
10399 			enter_smm(vcpu);  in kvm_check_and_inject_events()
10402 			static_call(kvm_x86_enable_smi_window)(vcpu);  in kvm_check_and_inject_events()
10406 	if (vcpu->arch.nmi_pending) {  in kvm_check_and_inject_events()
10407 		r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10411 			--vcpu->arch.nmi_pending;  in kvm_check_and_inject_events()
10412 			vcpu->arch.nmi_injected = true;  in kvm_check_and_inject_events()
10413 			static_call(kvm_x86_inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10415 			WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10417 		if (vcpu->arch.nmi_pending)  in kvm_check_and_inject_events()
10418 			static_call(kvm_x86_enable_nmi_window)(vcpu);  in kvm_check_and_inject_events()
10421 	if (kvm_cpu_has_injectable_intr(vcpu)) {  in kvm_check_and_inject_events()
10422 		r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;  in kvm_check_and_inject_events()
10426 			int irq = kvm_cpu_get_interrupt(vcpu);  in kvm_check_and_inject_events()
10429 				kvm_queue_interrupt(vcpu, irq, false);  in kvm_check_and_inject_events()
10430 				static_call(kvm_x86_inject_irq)(vcpu, false);  in kvm_check_and_inject_events()
10431 				WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10434 		if (kvm_cpu_has_injectable_intr(vcpu))  in kvm_check_and_inject_events()
10435 			static_call(kvm_x86_enable_irq_window)(vcpu);  in kvm_check_and_inject_events()
10438 	if (is_guest_mode(vcpu) &&  in kvm_check_and_inject_events()
10440 	    kvm_x86_ops.nested_ops->has_events(vcpu))  in kvm_check_and_inject_events()
10446 	 * to the VMCS/VMCB.  Queueing a new exception can put the vCPU into an  in kvm_check_and_inject_events()
10451 	 * vCPU into an infinite loop.  Triple fault can be queued when running  in kvm_check_and_inject_events()
10455 	WARN_ON_ONCE(vcpu->arch.exception.pending ||  in kvm_check_and_inject_events()
10456 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10467 static void process_nmi(struct kvm_vcpu *vcpu)  in process_nmi()  argument
10473 	 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is  in process_nmi()
10481 	if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)  in process_nmi()
10488 	 * tracked in vcpu->arch.nmi_pending.  in process_nmi()
10490 	if (static_call(kvm_x86_is_vnmi_pending)(vcpu))  in process_nmi()
10493 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);  in process_nmi()
10494 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);  in process_nmi()
10496 	if (vcpu->arch.nmi_pending &&  in process_nmi()
10497 	    (static_call(kvm_x86_set_vnmi_pending)(vcpu)))  in process_nmi()
10498 		vcpu->arch.nmi_pending--;  in process_nmi()
10500 	if (vcpu->arch.nmi_pending)  in process_nmi()
10501 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in process_nmi()
10505 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu)  in kvm_get_nr_pending_nmis()  argument
10507 	return vcpu->arch.nmi_pending +  in kvm_get_nr_pending_nmis()
10508 	       static_call(kvm_x86_is_vnmi_pending)(vcpu);  in kvm_get_nr_pending_nmis()
10522 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in __kvm_vcpu_update_apicv()  argument
10524 	struct kvm_lapic *apic = vcpu->arch.apic;  in __kvm_vcpu_update_apicv()
10527 	if (!lapic_in_kernel(vcpu))  in __kvm_vcpu_update_apicv()
10530 	down_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10534 	activate = kvm_vcpu_apicv_activated(vcpu) &&  in __kvm_vcpu_update_apicv()
10535 		   (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);  in __kvm_vcpu_update_apicv()
10541 	kvm_apic_update_apicv(vcpu);  in __kvm_vcpu_update_apicv()
10542 	static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);  in __kvm_vcpu_update_apicv()
10551 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __kvm_vcpu_update_apicv()
10555 	up_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10559 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in kvm_vcpu_update_apicv()  argument
10561 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_update_apicv()
10566 	 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but  in kvm_vcpu_update_apicv()
10571 	 * the vCPU would incorrectly be able to access the vAPIC page via MMIO  in kvm_vcpu_update_apicv()
10575 	if (apic_x2apic_mode(vcpu->arch.apic) &&  in kvm_vcpu_update_apicv()
10577 		kvm_inhibit_apic_access_page(vcpu);  in kvm_vcpu_update_apicv()
10579 	__kvm_vcpu_update_apicv(vcpu);  in kvm_vcpu_update_apicv()
10635 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)  in vcpu_scan_ioapic()  argument
10637 	if (!kvm_apic_present(vcpu))  in vcpu_scan_ioapic()
10640 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);  in vcpu_scan_ioapic()
10642 	if (irqchip_split(vcpu->kvm))  in vcpu_scan_ioapic()
10643 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10645 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_scan_ioapic()
10646 		if (ioapic_in_kernel(vcpu->kvm))  in vcpu_scan_ioapic()
10647 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10650 	if (is_guest_mode(vcpu))  in vcpu_scan_ioapic()
10651 		vcpu->arch.load_eoi_exitmap_pending = true;  in vcpu_scan_ioapic()
10653 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);  in vcpu_scan_ioapic()
10656 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)  in vcpu_load_eoi_exitmap()  argument
10658 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))  in vcpu_load_eoi_exitmap()
10662 	if (to_hv_vcpu(vcpu)) {  in vcpu_load_eoi_exitmap()
10666 			  vcpu->arch.ioapic_handled_vectors,  in vcpu_load_eoi_exitmap()
10667 			  to_hv_synic(vcpu)->vec_bitmap, 256);  in vcpu_load_eoi_exitmap()
10668 		static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);  in vcpu_load_eoi_exitmap()
10673 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);  in vcpu_load_eoi_exitmap()
10681 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)  in kvm_vcpu_reload_apic_access_page()  argument
10683 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_reload_apic_access_page()
10686 	static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);  in kvm_vcpu_reload_apic_access_page()
10689 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)  in __kvm_request_immediate_exit()  argument
10691 	smp_send_reschedule(vcpu->cpu);  in __kvm_request_immediate_exit()
10701 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)  in vcpu_enter_guest()  argument
10705 		dm_request_for_irq_injection(vcpu) &&  in vcpu_enter_guest()
10706 		kvm_cpu_accept_dm_intr(vcpu);  in vcpu_enter_guest()
10711 	if (kvm_request_pending(vcpu)) {  in vcpu_enter_guest()
10712 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {  in vcpu_enter_guest()
10717 		if (kvm_dirty_ring_check_request(vcpu)) {  in vcpu_enter_guest()
10722 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {  in vcpu_enter_guest()
10723 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {  in vcpu_enter_guest()
10728 		if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))  in vcpu_enter_guest()
10729 			kvm_mmu_free_obsolete_roots(vcpu);  in vcpu_enter_guest()
10730 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))  in vcpu_enter_guest()
10731 			__kvm_migrate_timers(vcpu);  in vcpu_enter_guest()
10732 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10733 			kvm_update_masterclock(vcpu->kvm);  in vcpu_enter_guest()
10734 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10735 			kvm_gen_kvmclock_update(vcpu);  in vcpu_enter_guest()
10736 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {  in vcpu_enter_guest()
10737 			r = kvm_guest_time_update(vcpu);  in vcpu_enter_guest()
10741 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))  in vcpu_enter_guest()
10742 			kvm_mmu_sync_roots(vcpu);  in vcpu_enter_guest()
10743 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))  in vcpu_enter_guest()
10744 			kvm_mmu_load_pgd(vcpu);  in vcpu_enter_guest()
10751 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))  in vcpu_enter_guest()
10752 			kvm_vcpu_flush_tlb_all(vcpu);  in vcpu_enter_guest()
10754 		kvm_service_local_tlb_flush_requests(vcpu);  in vcpu_enter_guest()
10758 		 * flushing fails.  Note, Hyper-V's flushing is per-vCPU, but  in vcpu_enter_guest()
10763 		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) &&  in vcpu_enter_guest()
10764 		    kvm_hv_vcpu_flush_tlb(vcpu))  in vcpu_enter_guest()
10765 			kvm_vcpu_flush_tlb_guest(vcpu);  in vcpu_enter_guest()
10768 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {  in vcpu_enter_guest()
10769 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;  in vcpu_enter_guest()
10773 		if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10774 			if (is_guest_mode(vcpu))  in vcpu_enter_guest()
10775 				kvm_x86_ops.nested_ops->triple_fault(vcpu);  in vcpu_enter_guest()
10777 			if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10778 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;  in vcpu_enter_guest()
10779 				vcpu->mmio_needed = 0;  in vcpu_enter_guest()
10784 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {  in vcpu_enter_guest()
10786 			vcpu->arch.apf.halted = true;  in vcpu_enter_guest()
10790 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))  in vcpu_enter_guest()
10791 			record_steal_time(vcpu);  in vcpu_enter_guest()
10792 		if (kvm_check_request(KVM_REQ_PMU, vcpu))  in vcpu_enter_guest()
10793 			kvm_pmu_handle_event(vcpu);  in vcpu_enter_guest()
10794 		if (kvm_check_request(KVM_REQ_PMI, vcpu))  in vcpu_enter_guest()
10795 			kvm_pmu_deliver_pmi(vcpu);  in vcpu_enter_guest()
10797 		if (kvm_check_request(KVM_REQ_SMI, vcpu))  in vcpu_enter_guest()
10798 			process_smi(vcpu);  in vcpu_enter_guest()
10800 		if (kvm_check_request(KVM_REQ_NMI, vcpu))  in vcpu_enter_guest()
10801 			process_nmi(vcpu);  in vcpu_enter_guest()
10802 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {  in vcpu_enter_guest()
10803 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);  in vcpu_enter_guest()
10804 			if (test_bit(vcpu->arch.pending_ioapic_eoi,  in vcpu_enter_guest()
10805 				     vcpu->arch.ioapic_handled_vectors)) {  in vcpu_enter_guest()
10806 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;  in vcpu_enter_guest()
10807 				vcpu->run->eoi.vector =  in vcpu_enter_guest()
10808 						vcpu->arch.pending_ioapic_eoi;  in vcpu_enter_guest()
10813 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))  in vcpu_enter_guest()
10814 			vcpu_scan_ioapic(vcpu);  in vcpu_enter_guest()
10815 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))  in vcpu_enter_guest()
10816 			vcpu_load_eoi_exitmap(vcpu);  in vcpu_enter_guest()
10817 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))  in vcpu_enter_guest()
10818 			kvm_vcpu_reload_apic_access_page(vcpu);  in vcpu_enter_guest()
10820 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {  in vcpu_enter_guest()
10821 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10822 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;  in vcpu_enter_guest()
10823 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10827 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {  in vcpu_enter_guest()
10828 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10829 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;  in vcpu_enter_guest()
10830 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10834 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {  in vcpu_enter_guest()
10835 			struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);  in vcpu_enter_guest()
10837 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;  in vcpu_enter_guest()
10838 			vcpu->run->hyperv = hv_vcpu->exit;  in vcpu_enter_guest()
10848 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))  in vcpu_enter_guest()
10849 			kvm_hv_process_stimers(vcpu);  in vcpu_enter_guest()
10851 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))  in vcpu_enter_guest()
10852 			kvm_vcpu_update_apicv(vcpu);  in vcpu_enter_guest()
10853 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))  in vcpu_enter_guest()
10854 			kvm_check_async_pf_completion(vcpu);  in vcpu_enter_guest()
10855 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))  in vcpu_enter_guest()
10856 			static_call(kvm_x86_msr_filter_changed)(vcpu);  in vcpu_enter_guest()
10858 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))  in vcpu_enter_guest()
10859 			static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);  in vcpu_enter_guest()
10862 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||  in vcpu_enter_guest()
10863 	    kvm_xen_has_interrupt(vcpu)) {  in vcpu_enter_guest()
10864 		++vcpu->stat.req_event;  in vcpu_enter_guest()
10865 		r = kvm_apic_accept_events(vcpu);  in vcpu_enter_guest()
10870 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {  in vcpu_enter_guest()
10875 		r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);  in vcpu_enter_guest()
10881 			static_call(kvm_x86_enable_irq_window)(vcpu);  in vcpu_enter_guest()
10883 		if (kvm_lapic_enabled(vcpu)) {  in vcpu_enter_guest()
10884 			update_cr8_intercept(vcpu);  in vcpu_enter_guest()
10885 			kvm_lapic_sync_to_vapic(vcpu);  in vcpu_enter_guest()
10889 	r = kvm_mmu_reload(vcpu);  in vcpu_enter_guest()
10896 	static_call(kvm_x86_prepare_switch_to_guest)(vcpu);  in vcpu_enter_guest()
10905 	/* Store vcpu->apicv_active before vcpu->mode.  */  in vcpu_enter_guest()
10906 	smp_store_release(&vcpu->mode, IN_GUEST_MODE);  in vcpu_enter_guest()
10908 	kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_enter_guest()
10919 	 * tables done while the VCPU is running.  Please see the comment  in vcpu_enter_guest()
10927 	 * target vCPU wasn't running).  Do this regardless of the vCPU's APICv  in vcpu_enter_guest()
10931 	if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10932 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10934 	if (kvm_vcpu_exit_request(vcpu)) {  in vcpu_enter_guest()
10935 		vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10939 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
10945 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
10946 		static_call(kvm_x86_request_immediate_exit)(vcpu);  in vcpu_enter_guest()
10953 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10954 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);  in vcpu_enter_guest()
10956 	if (unlikely(vcpu->arch.switch_db_regs)) {  in vcpu_enter_guest()
10958 		set_debugreg(vcpu->arch.eff_db[0], 0);  in vcpu_enter_guest()
10959 		set_debugreg(vcpu->arch.eff_db[1], 1);  in vcpu_enter_guest()
10960 		set_debugreg(vcpu->arch.eff_db[2], 2);  in vcpu_enter_guest()
10961 		set_debugreg(vcpu->arch.eff_db[3], 3);  in vcpu_enter_guest()
10970 		 * Assert that vCPU vs. VM APICv state is consistent.  An APICv  in vcpu_enter_guest()
10975 		WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&  in vcpu_enter_guest()
10976 			     (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));  in vcpu_enter_guest()
10978 		exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu);  in vcpu_enter_guest()
10982 		if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10983 			static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10985 		if (unlikely(kvm_vcpu_exit_request(vcpu))) {  in vcpu_enter_guest()
10991 		++vcpu->stat.exits;  in vcpu_enter_guest()
11000 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {  in vcpu_enter_guest()
11001 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);  in vcpu_enter_guest()
11002 		static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);  in vcpu_enter_guest()
11003 		kvm_update_dr0123(vcpu);  in vcpu_enter_guest()
11004 		kvm_update_dr7(vcpu);  in vcpu_enter_guest()
11017 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;  in vcpu_enter_guest()
11018 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());  in vcpu_enter_guest()
11020 	vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
11028 	if (vcpu->arch.xfd_no_write_intercept)  in vcpu_enter_guest()
11031 	static_call(kvm_x86_handle_exit_irqoff)(vcpu);  in vcpu_enter_guest()
11033 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
11043 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);  in vcpu_enter_guest()
11045 	++vcpu->stat.exits;  in vcpu_enter_guest()
11047 	kvm_after_interrupt(vcpu);  in vcpu_enter_guest()
11061 	kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
11067 		unsigned long rip = kvm_rip_read(vcpu);  in vcpu_enter_guest()
11071 	if (unlikely(vcpu->arch.tsc_always_catchup))  in vcpu_enter_guest()
11072 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in vcpu_enter_guest()
11074 	if (vcpu->arch.apic_attention)  in vcpu_enter_guest()
11075 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
11077 	r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);  in vcpu_enter_guest()
11082 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
11083 	static_call(kvm_x86_cancel_injection)(vcpu);  in vcpu_enter_guest()
11084 	if (unlikely(vcpu->arch.apic_attention))  in vcpu_enter_guest()
11085 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
11091 static inline int vcpu_block(struct kvm_vcpu *vcpu)  in vcpu_block()  argument
11095 	if (!kvm_arch_vcpu_runnable(vcpu)) {  in vcpu_block()
11098 		 * the guest's timer may be a break event for the vCPU, and the  in vcpu_block()
11103 		hv_timer = kvm_lapic_hv_timer_in_use(vcpu);  in vcpu_block()
11105 			kvm_lapic_switch_to_sw_timer(vcpu);  in vcpu_block()
11107 		kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_block()
11108 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)  in vcpu_block()
11109 			kvm_vcpu_halt(vcpu);  in vcpu_block()
11111 			kvm_vcpu_block(vcpu);  in vcpu_block()
11112 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_block()
11115 			kvm_lapic_switch_to_hv_timer(vcpu);  in vcpu_block()
11118 		 * If the vCPU is not runnable, a signal or another host event  in vcpu_block()
11120 		 * vCPU's activity state.  in vcpu_block()
11122 		if (!kvm_arch_vcpu_runnable(vcpu))  in vcpu_block()
11132 	if (is_guest_mode(vcpu)) {  in vcpu_block()
11133 		if (kvm_check_nested_events(vcpu) < 0)  in vcpu_block()
11137 	if (kvm_apic_accept_events(vcpu) < 0)  in vcpu_block()
11139 	switch(vcpu->arch.mp_state) {  in vcpu_block()
11142 		vcpu->arch.pv.pv_unhalted = false;  in vcpu_block()
11143 		vcpu->arch.mp_state =  in vcpu_block()
11147 		vcpu->arch.apf.halted = false;  in vcpu_block()
11158 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)  in kvm_vcpu_running()  argument
11160 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&  in kvm_vcpu_running()
11161 		!vcpu->arch.apf.halted);  in kvm_vcpu_running()
11165 static int vcpu_run(struct kvm_vcpu *vcpu)  in vcpu_run()  argument
11169 	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;  in vcpu_run()
11170 	vcpu->arch.l1tf_flush_l1d = true;  in vcpu_run()
11174 		 * If another guest vCPU requests a PV TLB flush in the middle  in vcpu_run()
11179 		vcpu->arch.at_instruction_boundary = false;  in vcpu_run()
11180 		if (kvm_vcpu_running(vcpu)) {  in vcpu_run()
11181 			r = vcpu_enter_guest(vcpu);  in vcpu_run()
11183 			r = vcpu_block(vcpu);  in vcpu_run()
11189 		kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);  in vcpu_run()
11190 		if (kvm_xen_has_pending_events(vcpu))  in vcpu_run()
11191 			kvm_xen_inject_pending_events(vcpu);  in vcpu_run()
11193 		if (kvm_cpu_has_pending_timer(vcpu))  in vcpu_run()
11194 			kvm_inject_pending_timer_irqs(vcpu);  in vcpu_run()
11196 		if (dm_request_for_irq_injection(vcpu) &&  in vcpu_run()
11197 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {  in vcpu_run()
11199 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;  in vcpu_run()
11200 			++vcpu->stat.request_irq_exits;  in vcpu_run()
11205 			kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_run()
11206 			r = xfer_to_guest_mode_handle_work(vcpu);  in vcpu_run()
11207 			kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_run()
11216 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)  in complete_emulated_io()  argument
11218 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);  in complete_emulated_io()
11221 static int complete_emulated_pio(struct kvm_vcpu *vcpu)  in complete_emulated_pio()  argument
11223 	BUG_ON(!vcpu->arch.pio.count);  in complete_emulated_pio()
11225 	return complete_emulated_io(vcpu);  in complete_emulated_pio()
11246 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_emulated_mmio()  argument
11248 	struct kvm_run *run = vcpu->run;  in complete_emulated_mmio()
11252 	BUG_ON(!vcpu->mmio_needed);  in complete_emulated_mmio()
11255 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_emulated_mmio()
11257 	if (!vcpu->mmio_is_write)  in complete_emulated_mmio()
11263 		vcpu->mmio_cur_fragment++;  in complete_emulated_mmio()
11271 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_emulated_mmio()
11272 		vcpu->mmio_needed = 0;  in complete_emulated_mmio()
11275 		if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11277 		vcpu->mmio_read_completed = 1;  in complete_emulated_mmio()
11278 		return complete_emulated_io(vcpu);  in complete_emulated_mmio()
11283 	if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11286 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_emulated_mmio()
11287 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in complete_emulated_mmio()
11292 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_load_guest_fpu()  argument
11295 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);  in kvm_load_guest_fpu()
11300 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_put_guest_fpu()  argument
11302 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);  in kvm_put_guest_fpu()
11303 	++vcpu->stat.fpu_reload;  in kvm_put_guest_fpu()
11307 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_ioctl_run()  argument
11309 	struct kvm_queued_exception *ex = &vcpu->arch.exception;  in kvm_arch_vcpu_ioctl_run()
11310 	struct kvm_run *kvm_run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
11313 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_run()
11314 	kvm_sigset_activate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11316 	kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11318 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11319 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {  in kvm_arch_vcpu_ioctl_run()
11328 		 * APIC timer to be active is if userspace stuffed vCPU state,  in kvm_arch_vcpu_ioctl_run()
11329 		 * i.e. put the vCPU into a nonsensical state.  Only an INIT  in kvm_arch_vcpu_ioctl_run()
11330 		 * will transition the vCPU out of UNINITIALIZED (without more  in kvm_arch_vcpu_ioctl_run()
11335 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11336 		kvm_vcpu_block(vcpu);  in kvm_arch_vcpu_ioctl_run()
11337 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11339 		if (kvm_apic_accept_events(vcpu) < 0) {  in kvm_arch_vcpu_ioctl_run()
11347 			++vcpu->stat.signal_exits;  in kvm_arch_vcpu_ioctl_run()
11359 		r = sync_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11365 	if (!lapic_in_kernel(vcpu)) {  in kvm_arch_vcpu_ioctl_run()
11366 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {  in kvm_arch_vcpu_ioctl_run()
11376 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&  in kvm_arch_vcpu_ioctl_run()
11377 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11379 		kvm_queue_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11385 	vcpu->arch.exception_from_userspace = false;  in kvm_arch_vcpu_ioctl_run()
11387 	if (unlikely(vcpu->arch.complete_userspace_io)) {  in kvm_arch_vcpu_ioctl_run()
11388 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;  in kvm_arch_vcpu_ioctl_run()
11389 		vcpu->arch.complete_userspace_io = NULL;  in kvm_arch_vcpu_ioctl_run()
11390 		r = cui(vcpu);  in kvm_arch_vcpu_ioctl_run()
11394 		WARN_ON_ONCE(vcpu->arch.pio.count);  in kvm_arch_vcpu_ioctl_run()
11395 		WARN_ON_ONCE(vcpu->mmio_needed);  in kvm_arch_vcpu_ioctl_run()
11403 	r = static_call(kvm_x86_vcpu_pre_run)(vcpu);  in kvm_arch_vcpu_ioctl_run()
11407 	r = vcpu_run(vcpu);  in kvm_arch_vcpu_ioctl_run()
11410 	kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11412 		store_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11413 	post_kvm_run_save(vcpu);  in kvm_arch_vcpu_ioctl_run()
11414 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11416 	kvm_sigset_deactivate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11417 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_run()
11421 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __get_regs()  argument
11423 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {  in __get_regs()
11427 		 * back from emulation context to vcpu. Userspace shouldn't do  in __get_regs()
11431 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);  in __get_regs()
11432 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __get_regs()
11434 	regs->rax = kvm_rax_read(vcpu);  in __get_regs()
11435 	regs->rbx = kvm_rbx_read(vcpu);  in __get_regs()
11436 	regs->rcx = kvm_rcx_read(vcpu);  in __get_regs()
11437 	regs->rdx = kvm_rdx_read(vcpu);  in __get_regs()
11438 	regs->rsi = kvm_rsi_read(vcpu);  in __get_regs()
11439 	regs->rdi = kvm_rdi_read(vcpu);  in __get_regs()
11440 	regs->rsp = kvm_rsp_read(vcpu);  in __get_regs()
11441 	regs->rbp = kvm_rbp_read(vcpu);  in __get_regs()
11443 	regs->r8 = kvm_r8_read(vcpu);  in __get_regs()
11444 	regs->r9 = kvm_r9_read(vcpu);  in __get_regs()
11445 	regs->r10 = kvm_r10_read(vcpu);  in __get_regs()
11446 	regs->r11 = kvm_r11_read(vcpu);  in __get_regs()
11447 	regs->r12 = kvm_r12_read(vcpu);  in __get_regs()
11448 	regs->r13 = kvm_r13_read(vcpu);  in __get_regs()
11449 	regs->r14 = kvm_r14_read(vcpu);  in __get_regs()
11450 	regs->r15 = kvm_r15_read(vcpu);  in __get_regs()
11453 	regs->rip = kvm_rip_read(vcpu);  in __get_regs()
11454 	regs->rflags = kvm_get_rflags(vcpu);  in __get_regs()
11457 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_get_regs()  argument
11459 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11460 	__get_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_get_regs()
11461 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11465 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __set_regs()  argument
11467 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;  in __set_regs()
11468 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __set_regs()
11470 	kvm_rax_write(vcpu, regs->rax);  in __set_regs()
11471 	kvm_rbx_write(vcpu, regs->rbx);  in __set_regs()
11472 	kvm_rcx_write(vcpu, regs->rcx);  in __set_regs()
11473 	kvm_rdx_write(vcpu, regs->rdx);  in __set_regs()
11474 	kvm_rsi_write(vcpu, regs->rsi);  in __set_regs()
11475 	kvm_rdi_write(vcpu, regs->rdi);  in __set_regs()
11476 	kvm_rsp_write(vcpu, regs->rsp);  in __set_regs()
11477 	kvm_rbp_write(vcpu, regs->rbp);  in __set_regs()
11479 	kvm_r8_write(vcpu, regs->r8);  in __set_regs()
11480 	kvm_r9_write(vcpu, regs->r9);  in __set_regs()
11481 	kvm_r10_write(vcpu, regs->r10);  in __set_regs()
11482 	kvm_r11_write(vcpu, regs->r11);  in __set_regs()
11483 	kvm_r12_write(vcpu, regs->r12);  in __set_regs()
11484 	kvm_r13_write(vcpu, regs->r13);  in __set_regs()
11485 	kvm_r14_write(vcpu, regs->r14);  in __set_regs()
11486 	kvm_r15_write(vcpu, regs->r15);  in __set_regs()
11489 	kvm_rip_write(vcpu, regs->rip);  in __set_regs()
11490 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);  in __set_regs()
11492 	vcpu->arch.exception.pending = false;  in __set_regs()
11493 	vcpu->arch.exception_vmexit.pending = false;  in __set_regs()
11495 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_regs()
11498 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_set_regs()  argument
11500 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11501 	__set_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_set_regs()
11502 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11506 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs_common()  argument
11510 	if (vcpu->arch.guest_state_protected)  in __get_sregs_common()
11513 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __get_sregs_common()
11514 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __get_sregs_common()
11515 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __get_sregs_common()
11516 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __get_sregs_common()
11517 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __get_sregs_common()
11518 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __get_sregs_common()
11520 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __get_sregs_common()
11521 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __get_sregs_common()
11523 	static_call(kvm_x86_get_idt)(vcpu, &dt);  in __get_sregs_common()
11526 	static_call(kvm_x86_get_gdt)(vcpu, &dt);  in __get_sregs_common()
11530 	sregs->cr2 = vcpu->arch.cr2;  in __get_sregs_common()
11531 	sregs->cr3 = kvm_read_cr3(vcpu);  in __get_sregs_common()
11534 	sregs->cr0 = kvm_read_cr0(vcpu);  in __get_sregs_common()
11535 	sregs->cr4 = kvm_read_cr4(vcpu);  in __get_sregs_common()
11536 	sregs->cr8 = kvm_get_cr8(vcpu);  in __get_sregs_common()
11537 	sregs->efer = vcpu->arch.efer;  in __get_sregs_common()
11538 	sregs->apic_base = kvm_get_apic_base(vcpu);  in __get_sregs_common()
11541 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs()  argument
11543 	__get_sregs_common(vcpu, sregs);  in __get_sregs()
11545 	if (vcpu->arch.guest_state_protected)  in __get_sregs()
11548 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)  in __get_sregs()
11549 		set_bit(vcpu->arch.interrupt.nr,  in __get_sregs()
11553 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __get_sregs2()  argument
11557 	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);  in __get_sregs2()
11559 	if (vcpu->arch.guest_state_protected)  in __get_sregs2()
11562 	if (is_pae_paging(vcpu)) {  in __get_sregs2()
11564 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);  in __get_sregs2()
11569 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_sregs()  argument
11572 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11573 	__get_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_get_sregs()
11574 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11578 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_mpstate()  argument
11583 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11585 		kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11587 	r = kvm_apic_accept_events(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11592 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||  in kvm_arch_vcpu_ioctl_get_mpstate()
11593 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&  in kvm_arch_vcpu_ioctl_get_mpstate()
11594 	    vcpu->arch.pv.pv_unhalted)  in kvm_arch_vcpu_ioctl_get_mpstate()
11597 		mp_state->mp_state = vcpu->arch.mp_state;  in kvm_arch_vcpu_ioctl_get_mpstate()
11601 		kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11602 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11606 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_mpstate()  argument
11611 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11619 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl_set_mpstate()
11636 	if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
11642 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;  in kvm_arch_vcpu_ioctl_set_mpstate()
11643 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);  in kvm_arch_vcpu_ioctl_set_mpstate()
11645 		vcpu->arch.mp_state = mp_state->mp_state;  in kvm_arch_vcpu_ioctl_set_mpstate()
11646 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11650 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11654 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,  in kvm_task_switch()  argument
11657 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()
11660 	init_emulate_ctxt(vcpu);  in kvm_task_switch()
11665 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_task_switch()
11666 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_task_switch()
11667 		vcpu->run->internal.ndata = 0;  in kvm_task_switch()
11671 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
11672 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()
11677 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvm_is_valid_sregs()  argument
11687 		if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3))  in kvm_is_valid_sregs()
11698 	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&  in kvm_is_valid_sregs()
11699 	       kvm_is_valid_cr0(vcpu, sregs->cr0);  in kvm_is_valid_sregs()
11702 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,  in __set_sregs_common()  argument
11709 	if (!kvm_is_valid_sregs(vcpu, sregs))  in __set_sregs_common()
11714 	if (kvm_set_apic_base(vcpu, &apic_base_msr))  in __set_sregs_common()
11717 	if (vcpu->arch.guest_state_protected)  in __set_sregs_common()
11722 	static_call(kvm_x86_set_idt)(vcpu, &dt);  in __set_sregs_common()
11725 	static_call(kvm_x86_set_gdt)(vcpu, &dt);  in __set_sregs_common()
11727 	vcpu->arch.cr2 = sregs->cr2;  in __set_sregs_common()
11728 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;  in __set_sregs_common()
11729 	vcpu->arch.cr3 = sregs->cr3;  in __set_sregs_common()
11730 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in __set_sregs_common()
11731 	static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);  in __set_sregs_common()
11733 	kvm_set_cr8(vcpu, sregs->cr8);  in __set_sregs_common()
11735 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;  in __set_sregs_common()
11736 	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);  in __set_sregs_common()
11738 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;  in __set_sregs_common()
11739 	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);  in __set_sregs_common()
11741 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;  in __set_sregs_common()
11742 	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);  in __set_sregs_common()
11745 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in __set_sregs_common()
11746 		if (is_pae_paging(vcpu)) {  in __set_sregs_common()
11747 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));  in __set_sregs_common()
11750 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in __set_sregs_common()
11753 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __set_sregs_common()
11754 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __set_sregs_common()
11755 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __set_sregs_common()
11756 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __set_sregs_common()
11757 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __set_sregs_common()
11758 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __set_sregs_common()
11760 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __set_sregs_common()
11761 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __set_sregs_common()
11763 	update_cr8_intercept(vcpu);  in __set_sregs_common()
11765 	/* Older userspace won't unhalt the vcpu on reset. */  in __set_sregs_common()
11766 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&  in __set_sregs_common()
11768 	    !is_protmode(vcpu))  in __set_sregs_common()
11769 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in __set_sregs_common()
11774 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __set_sregs()  argument
11778 	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);  in __set_sregs()
11784 		kvm_mmu_reset_context(vcpu);  in __set_sregs()
11785 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in __set_sregs()
11793 		kvm_queue_interrupt(vcpu, pending_vec, false);  in __set_sregs()
11795 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_sregs()
11800 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __set_sregs2()  argument
11811 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))  in __set_sregs2()
11814 	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,  in __set_sregs2()
11821 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);  in __set_sregs2()
11823 		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in __set_sregs2()
11825 		vcpu->arch.pdptrs_from_userspace = true;  in __set_sregs2()
11828 		kvm_mmu_reset_context(vcpu);  in __set_sregs2()
11829 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in __set_sregs2()
11834 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_sregs()  argument
11839 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
11840 	ret = __set_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_set_sregs()
11841 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
11848 	struct kvm_vcpu *vcpu;  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()  local
11856 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11857 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
11866 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_guest_debug()  argument
11872 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11875 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11879 		if (kvm_is_exception_pending(vcpu))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11882 			kvm_queue_exception(vcpu, DB_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11884 			kvm_queue_exception(vcpu, BP_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11891 	rflags = kvm_get_rflags(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11893 	vcpu->guest_debug = dbg->control;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11894 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))  in kvm_arch_vcpu_ioctl_set_guest_debug()
11895 		vcpu->guest_debug = 0;  in kvm_arch_vcpu_ioctl_set_guest_debug()
11897 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
11899 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11900 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11903 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
11905 	kvm_update_dr7(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11907 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_arch_vcpu_ioctl_set_guest_debug()
11908 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11914 	kvm_set_rflags(vcpu, rflags);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11916 	static_call(kvm_x86_update_exception_bitmap)(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11918 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11923 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
11930 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_translate()  argument
11937 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_translate()
11939 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl_translate()
11940 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);  in kvm_arch_vcpu_ioctl_translate()
11941 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl_translate()
11947 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_translate()
11951 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_get_fpu()  argument
11955 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_get_fpu()
11958 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
11960 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_get_fpu()
11970 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
11974 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_set_fpu()  argument
11978 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_set_fpu()
11981 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
11983 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_set_fpu()
11994 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
11998 static void store_regs(struct kvm_vcpu *vcpu)  in store_regs()  argument
12002 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)  in store_regs()
12003 		__get_regs(vcpu, &vcpu->run->s.regs.regs);  in store_regs()
12005 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)  in store_regs()
12006 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);  in store_regs()
12008 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)  in store_regs()
12010 				vcpu, &vcpu->run->s.regs.events);  in store_regs()
12013 static int sync_regs(struct kvm_vcpu *vcpu)  in sync_regs()  argument
12015 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {  in sync_regs()
12016 		__set_regs(vcpu, &vcpu->run->s.regs.regs);  in sync_regs()
12017 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;  in sync_regs()
12020 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {  in sync_regs()
12021 		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;  in sync_regs()
12023 		if (__set_sregs(vcpu, &sregs))  in sync_regs()
12026 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;  in sync_regs()
12029 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {  in sync_regs()
12030 		struct kvm_vcpu_events events = vcpu->run->s.regs.events;  in sync_regs()
12032 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events))  in sync_regs()
12035 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;  in sync_regs()
12056 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_create()  argument
12061 	vcpu->arch.last_vmentry_cpu = -1;  in kvm_arch_vcpu_create()
12062 	vcpu->arch.regs_avail = ~0;  in kvm_arch_vcpu_create()
12063 	vcpu->arch.regs_dirty = ~0;  in kvm_arch_vcpu_create()
12065 	kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN);  in kvm_arch_vcpu_create()
12067 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))  in kvm_arch_vcpu_create()
12068 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_create()
12070 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;  in kvm_arch_vcpu_create()
12072 	r = kvm_mmu_create(vcpu);  in kvm_arch_vcpu_create()
12076 	if (irqchip_in_kernel(vcpu->kvm)) {  in kvm_arch_vcpu_create()
12077 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);  in kvm_arch_vcpu_create()
12082 		 * Defer evaluating inhibits until the vCPU is first run, as  in kvm_arch_vcpu_create()
12083 		 * this vCPU will not get notified of any changes until this  in kvm_arch_vcpu_create()
12084 		 * vCPU is visible to other vCPUs (marked online and added to  in kvm_arch_vcpu_create()
12087 		 * Ignore the current per-VM APICv state so that vCPU creation  in kvm_arch_vcpu_create()
12089 		 * will ensure the vCPU gets the correct state before VM-Entry.  in kvm_arch_vcpu_create()
12092 			vcpu->arch.apic->apicv_active = true;  in kvm_arch_vcpu_create()
12093 			kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);  in kvm_arch_vcpu_create()
12103 	vcpu->arch.pio_data = page_address(page);  in kvm_arch_vcpu_create()
12105 	vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),  in kvm_arch_vcpu_create()
12107 	vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),  in kvm_arch_vcpu_create()
12109 	if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)  in kvm_arch_vcpu_create()
12111 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;  in kvm_arch_vcpu_create()
12113 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,  in kvm_arch_vcpu_create()
12117 	if (!alloc_emulate_ctxt(vcpu))  in kvm_arch_vcpu_create()
12120 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {  in kvm_arch_vcpu_create()
12121 		pr_err("failed to allocate vcpu's fpu\n");  in kvm_arch_vcpu_create()
12125 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);  in kvm_arch_vcpu_create()
12126 	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);  in kvm_arch_vcpu_create()
12128 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;  in kvm_arch_vcpu_create()
12130 	kvm_async_pf_hash_reset(vcpu);  in kvm_arch_vcpu_create()
12132 	vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;  in kvm_arch_vcpu_create()
12133 	kvm_pmu_init(vcpu);  in kvm_arch_vcpu_create()
12135 	vcpu->arch.pending_external_vector = -1;  in kvm_arch_vcpu_create()
12136 	vcpu->arch.preempted_in_kernel = false;  in kvm_arch_vcpu_create()
12139 	vcpu->arch.hv_root_tdp = INVALID_PAGE;  in kvm_arch_vcpu_create()
12142 	r = static_call(kvm_x86_vcpu_create)(vcpu);  in kvm_arch_vcpu_create()
12146 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();  in kvm_arch_vcpu_create()
12147 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;  in kvm_arch_vcpu_create()
12148 	kvm_xen_init_vcpu(vcpu);  in kvm_arch_vcpu_create()
12149 	kvm_vcpu_mtrr_init(vcpu);  in kvm_arch_vcpu_create()
12150 	vcpu_load(vcpu);  in kvm_arch_vcpu_create()
12151 	kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);  in kvm_arch_vcpu_create()
12152 	kvm_vcpu_reset(vcpu, false);  in kvm_arch_vcpu_create()
12153 	kvm_init_mmu(vcpu);  in kvm_arch_vcpu_create()
12154 	vcpu_put(vcpu);  in kvm_arch_vcpu_create()
12158 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_create()
12160 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_create()
12162 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_create()
12164 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_create()
12165 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_create()
12166 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_create()
12168 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_create()
12170 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_create()
12174 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_postcreate()  argument
12176 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_vcpu_postcreate()
12178 	if (mutex_lock_killable(&vcpu->mutex))  in kvm_arch_vcpu_postcreate()
12180 	vcpu_load(vcpu);  in kvm_arch_vcpu_postcreate()
12181 	kvm_synchronize_tsc(vcpu, NULL);  in kvm_arch_vcpu_postcreate()
12182 	vcpu_put(vcpu);  in kvm_arch_vcpu_postcreate()
12185 	vcpu->arch.msr_kvm_poll_control = 1;  in kvm_arch_vcpu_postcreate()
12187 	mutex_unlock(&vcpu->mutex);  in kvm_arch_vcpu_postcreate()
12189 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)  in kvm_arch_vcpu_postcreate()
12194 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_destroy()  argument
12198 	kvmclock_reset(vcpu);  in kvm_arch_vcpu_destroy()
12200 	static_call(kvm_x86_vcpu_free)(vcpu);  in kvm_arch_vcpu_destroy()
12202 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_destroy()
12203 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_destroy()
12204 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_destroy()
12206 	kvm_xen_destroy_vcpu(vcpu);  in kvm_arch_vcpu_destroy()
12207 	kvm_hv_vcpu_uninit(vcpu);  in kvm_arch_vcpu_destroy()
12208 	kvm_pmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12209 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_destroy()
12210 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_destroy()
12211 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_destroy()
12212 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_destroy()
12213 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12214 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_destroy()
12215 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_destroy()
12216 	kvfree(vcpu->arch.cpuid_entries);  in kvm_arch_vcpu_destroy()
12217 	if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_destroy()
12221 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)  in kvm_vcpu_reset()  argument
12224 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_vcpu_reset()
12231 	 * into hardware, to be zeroed at vCPU creation.  Use CRs as a sentinel  in kvm_vcpu_reset()
12235 		     (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));  in kvm_vcpu_reset()
12239 	 * possible to INIT the vCPU while L2 is active.  Force the vCPU back  in kvm_vcpu_reset()
12243 	if (is_guest_mode(vcpu))  in kvm_vcpu_reset()
12244 		kvm_leave_nested(vcpu);  in kvm_vcpu_reset()
12246 	kvm_lapic_reset(vcpu, init_event);  in kvm_vcpu_reset()
12248 	WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));  in kvm_vcpu_reset()
12249 	vcpu->arch.hflags = 0;  in kvm_vcpu_reset()
12251 	vcpu->arch.smi_pending = 0;  in kvm_vcpu_reset()
12252 	vcpu->arch.smi_count = 0;  in kvm_vcpu_reset()
12253 	atomic_set(&vcpu->arch.nmi_queued, 0);  in kvm_vcpu_reset()
12254 	vcpu->arch.nmi_pending = 0;  in kvm_vcpu_reset()
12255 	vcpu->arch.nmi_injected = false;  in kvm_vcpu_reset()
12256 	kvm_clear_interrupt_queue(vcpu);  in kvm_vcpu_reset()
12257 	kvm_clear_exception_queue(vcpu);  in kvm_vcpu_reset()
12259 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));  in kvm_vcpu_reset()
12260 	kvm_update_dr0123(vcpu);  in kvm_vcpu_reset()
12261 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;  in kvm_vcpu_reset()
12262 	vcpu->arch.dr7 = DR7_FIXED_1;  in kvm_vcpu_reset()
12263 	kvm_update_dr7(vcpu);  in kvm_vcpu_reset()
12265 	vcpu->arch.cr2 = 0;  in kvm_vcpu_reset()
12267 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_reset()
12268 	vcpu->arch.apf.msr_en_val = 0;  in kvm_vcpu_reset()
12269 	vcpu->arch.apf.msr_int_val = 0;  in kvm_vcpu_reset()
12270 	vcpu->arch.st.msr_val = 0;  in kvm_vcpu_reset()
12272 	kvmclock_reset(vcpu);  in kvm_vcpu_reset()
12274 	kvm_clear_async_pf_completion_queue(vcpu);  in kvm_vcpu_reset()
12275 	kvm_async_pf_hash_reset(vcpu);  in kvm_vcpu_reset()
12276 	vcpu->arch.apf.halted = false;  in kvm_vcpu_reset()
12278 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {  in kvm_vcpu_reset()
12279 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;  in kvm_vcpu_reset()
12286 			kvm_put_guest_fpu(vcpu);  in kvm_vcpu_reset()
12292 			kvm_load_guest_fpu(vcpu);  in kvm_vcpu_reset()
12296 		vcpu->arch.smbase = 0x30000;  in kvm_vcpu_reset()
12298 		vcpu->arch.msr_misc_features_enables = 0;  in kvm_vcpu_reset()
12299 		vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |  in kvm_vcpu_reset()
12302 		__kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);  in kvm_vcpu_reset()
12303 		__kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);  in kvm_vcpu_reset()
12307 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));  in kvm_vcpu_reset()
12308 	kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);  in kvm_vcpu_reset()
12313 	 * RESET since KVM emulates RESET before exposing the vCPU to userspace,  in kvm_vcpu_reset()
12317 	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);  in kvm_vcpu_reset()
12318 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);  in kvm_vcpu_reset()
12320 	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);  in kvm_vcpu_reset()
12322 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);  in kvm_vcpu_reset()
12323 	kvm_rip_write(vcpu, 0xfff0);  in kvm_vcpu_reset()
12325 	vcpu->arch.cr3 = 0;  in kvm_vcpu_reset()
12326 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_vcpu_reset()
12339 	static_call(kvm_x86_set_cr0)(vcpu, new_cr0);  in kvm_vcpu_reset()
12340 	static_call(kvm_x86_set_cr4)(vcpu, 0);  in kvm_vcpu_reset()
12341 	static_call(kvm_x86_set_efer)(vcpu, 0);  in kvm_vcpu_reset()
12342 	static_call(kvm_x86_update_exception_bitmap)(vcpu);  in kvm_vcpu_reset()
12353 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12354 		kvm_mmu_reset_context(vcpu);  in kvm_vcpu_reset()
12367 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12371 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)  in kvm_vcpu_deliver_sipi_vector()  argument
12375 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12378 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12379 	kvm_rip_write(vcpu, 0);  in kvm_vcpu_deliver_sipi_vector()
12386 	struct kvm_vcpu *vcpu;  in kvm_arch_hardware_enable()  local
12406 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
12407 			if (!stable && vcpu->cpu == smp_processor_id())  in kvm_arch_hardware_enable()
12408 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
12409 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {  in kvm_arch_hardware_enable()
12411 				if (vcpu->arch.last_host_tsc > max_tsc)  in kvm_arch_hardware_enable()
12412 					max_tsc = vcpu->arch.last_host_tsc;  in kvm_arch_hardware_enable()
12429 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,  in kvm_arch_hardware_enable()
12431 	 * adjustments, in case multiple suspend cycles happen before some VCPU  in kvm_arch_hardware_enable()
12459 			kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_hardware_enable()
12460 				vcpu->arch.tsc_offset_adjustment += delta_cyc;  in kvm_arch_hardware_enable()
12461 				vcpu->arch.last_host_tsc = local_tsc;  in kvm_arch_hardware_enable()
12462 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_arch_hardware_enable()
12485 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_reset_bsp()  argument
12487 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;  in kvm_vcpu_is_reset_bsp()
12490 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_bsp()  argument
12492 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;  in kvm_vcpu_is_bsp()
12498 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_sched_in()  argument
12500 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);  in kvm_arch_sched_in()
12502 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_arch_sched_in()
12505 		kvm_make_request(KVM_REQ_PMU, vcpu);  in kvm_arch_sched_in()
12507 	static_call(kvm_x86_sched_in)(vcpu, cpu);  in kvm_arch_sched_in()
12587 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)  in kvm_unload_vcpu_mmu()  argument
12589 	vcpu_load(vcpu);  in kvm_unload_vcpu_mmu()
12590 	kvm_mmu_unload(vcpu);  in kvm_unload_vcpu_mmu()
12591 	vcpu_put(vcpu);  in kvm_unload_vcpu_mmu()
12597 	struct kvm_vcpu *vcpu;  in kvm_unload_vcpu_mmus()  local
12599 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_unload_vcpu_mmus()
12600 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_unload_vcpu_mmus()
12601 		kvm_unload_vcpu_mmu(vcpu);  in kvm_unload_vcpu_mmus()
12839 	struct kvm_vcpu *vcpu;  in kvm_arch_memslots_updated()  local
12849 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_memslots_updated()
12850 		kvm_vcpu_kick(vcpu);  in kvm_arch_memslots_updated()
13038 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)  in kvm_guest_apic_has_interrupt()  argument
13040 	return (is_guest_mode(vcpu) &&  in kvm_guest_apic_has_interrupt()
13041 		static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));  in kvm_guest_apic_has_interrupt()
13044 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)  in kvm_vcpu_has_events()  argument
13046 	if (!list_empty_careful(&vcpu->async_pf.done))  in kvm_vcpu_has_events()
13049 	if (kvm_apic_has_pending_init_or_sipi(vcpu) &&  in kvm_vcpu_has_events()
13050 	    kvm_apic_init_sipi_allowed(vcpu))  in kvm_vcpu_has_events()
13053 	if (vcpu->arch.pv.pv_unhalted)  in kvm_vcpu_has_events()
13056 	if (kvm_is_exception_pending(vcpu))  in kvm_vcpu_has_events()
13059 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_vcpu_has_events()
13060 	    (vcpu->arch.nmi_pending &&  in kvm_vcpu_has_events()
13061 	     static_call(kvm_x86_nmi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
13065 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_vcpu_has_events()
13066 	    (vcpu->arch.smi_pending &&  in kvm_vcpu_has_events()
13067 	     static_call(kvm_x86_smi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
13071 	if (kvm_test_request(KVM_REQ_PMI, vcpu))  in kvm_vcpu_has_events()
13074 	if (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_has_events()
13075 	    (kvm_cpu_has_interrupt(vcpu) ||  in kvm_vcpu_has_events()
13076 	    kvm_guest_apic_has_interrupt(vcpu)))  in kvm_vcpu_has_events()
13079 	if (kvm_hv_has_stimer_pending(vcpu))  in kvm_vcpu_has_events()
13082 	if (is_guest_mode(vcpu) &&  in kvm_vcpu_has_events()
13084 	    kvm_x86_ops.nested_ops->has_events(vcpu))  in kvm_vcpu_has_events()
13087 	if (kvm_xen_has_pending_events(vcpu))  in kvm_vcpu_has_events()
13093 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_runnable()  argument
13095 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);  in kvm_arch_vcpu_runnable()
13098 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)  in kvm_arch_dy_has_pending_interrupt()  argument
13100 	if (kvm_vcpu_apicv_active(vcpu) &&  in kvm_arch_dy_has_pending_interrupt()
13101 	    static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))  in kvm_arch_dy_has_pending_interrupt()
13107 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_dy_runnable()  argument
13109 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))  in kvm_arch_dy_runnable()
13112 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_arch_dy_runnable()
13114 		kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_arch_dy_runnable()
13116 		 kvm_test_request(KVM_REQ_EVENT, vcpu))  in kvm_arch_dy_runnable()
13119 	return kvm_arch_dy_has_pending_interrupt(vcpu);  in kvm_arch_dy_runnable()
13122 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_in_kernel()  argument
13124 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_in_kernel()
13127 	if (vcpu != kvm_get_running_vcpu())  in kvm_arch_vcpu_in_kernel()
13128 		return vcpu->arch.preempted_in_kernel;  in kvm_arch_vcpu_in_kernel()
13130 	return static_call(kvm_x86_get_cpl)(vcpu) == 0;  in kvm_arch_vcpu_in_kernel()
13133 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_get_ip()  argument
13135 	return kvm_rip_read(vcpu);  in kvm_arch_vcpu_get_ip()
13138 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_should_kick()  argument
13140 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;  in kvm_arch_vcpu_should_kick()
13143 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)  in kvm_arch_interrupt_allowed()  argument
13145 	return static_call(kvm_x86_interrupt_allowed)(vcpu, false);  in kvm_arch_interrupt_allowed()
13148 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)  in kvm_get_linear_rip()  argument
13151 	if (vcpu->arch.guest_state_protected)  in kvm_get_linear_rip()
13154 	if (is_64_bit_mode(vcpu))  in kvm_get_linear_rip()
13155 		return kvm_rip_read(vcpu);  in kvm_get_linear_rip()
13156 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +  in kvm_get_linear_rip()
13157 		     kvm_rip_read(vcpu));  in kvm_get_linear_rip()
13161 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)  in kvm_is_linear_rip()  argument
13163 	return kvm_get_linear_rip(vcpu) == linear_rip;  in kvm_is_linear_rip()
13167 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)  in kvm_get_rflags()  argument
13171 	rflags = static_call(kvm_x86_get_rflags)(vcpu);  in kvm_get_rflags()
13172 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_get_rflags()
13178 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in __kvm_set_rflags()  argument
13180 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&  in __kvm_set_rflags()
13181 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))  in __kvm_set_rflags()
13183 	static_call(kvm_x86_set_rflags)(vcpu, rflags);  in __kvm_set_rflags()
13186 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in kvm_set_rflags()  argument
13188 	__kvm_set_rflags(vcpu, rflags);  in kvm_set_rflags()
13189 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_set_rflags()
13205 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_add_async_pf_gfn()  argument
13209 	while (vcpu->arch.apf.gfns[key] != ~0)  in kvm_add_async_pf_gfn()
13212 	vcpu->arch.apf.gfns[key] = gfn;  in kvm_add_async_pf_gfn()
13215 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_async_pf_gfn_slot()  argument
13221 		     (vcpu->arch.apf.gfns[key] != gfn &&  in kvm_async_pf_gfn_slot()
13222 		      vcpu->arch.apf.gfns[key] != ~0); i++)  in kvm_async_pf_gfn_slot()
13228 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_find_async_pf_gfn()  argument
13230 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;  in kvm_find_async_pf_gfn()
13233 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_del_async_pf_gfn()  argument
13237 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);  in kvm_del_async_pf_gfn()
13239 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))  in kvm_del_async_pf_gfn()
13243 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_del_async_pf_gfn()
13246 			if (vcpu->arch.apf.gfns[j] == ~0)  in kvm_del_async_pf_gfn()
13248 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);  in kvm_del_async_pf_gfn()
13255 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];  in kvm_del_async_pf_gfn()
13260 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)  in apf_put_user_notpresent()  argument
13264 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,  in apf_put_user_notpresent()
13268 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)  in apf_put_user_ready()  argument
13272 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_put_user_ready()
13276 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)  in apf_pageready_slot_free()  argument
13281 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_pageready_slot_free()
13288 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_deliver_async_pf()  argument
13291 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_can_deliver_async_pf()
13294 	if (vcpu->arch.apf.send_user_only &&  in kvm_can_deliver_async_pf()
13295 	    static_call(kvm_x86_get_cpl)(vcpu) == 0)  in kvm_can_deliver_async_pf()
13298 	if (is_guest_mode(vcpu)) {  in kvm_can_deliver_async_pf()
13303 		return vcpu->arch.apf.delivery_as_pf_vmexit;  in kvm_can_deliver_async_pf()
13310 		return is_paging(vcpu);  in kvm_can_deliver_async_pf()
13314 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_do_async_pf()  argument
13316 	if (unlikely(!lapic_in_kernel(vcpu) ||  in kvm_can_do_async_pf()
13317 		     kvm_event_needs_reinjection(vcpu) ||  in kvm_can_do_async_pf()
13318 		     kvm_is_exception_pending(vcpu)))  in kvm_can_do_async_pf()
13321 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))  in kvm_can_do_async_pf()
13328 	return kvm_arch_interrupt_allowed(vcpu);  in kvm_can_do_async_pf()
13331 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_not_present()  argument
13337 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_not_present()
13339 	if (kvm_can_deliver_async_pf(vcpu) &&  in kvm_arch_async_page_not_present()
13340 	    !apf_put_user_notpresent(vcpu)) {  in kvm_arch_async_page_not_present()
13347 		kvm_inject_page_fault(vcpu, &fault);  in kvm_arch_async_page_not_present()
13358 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);  in kvm_arch_async_page_not_present()
13363 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_present()  argument
13368 		.vector = vcpu->arch.apf.vec  in kvm_arch_async_page_present()
13374 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_present()
13378 	    kvm_pv_async_pf_enabled(vcpu) &&  in kvm_arch_async_page_present()
13379 	    !apf_put_user_ready(vcpu, work->arch.token)) {  in kvm_arch_async_page_present()
13380 		vcpu->arch.apf.pageready_pending = true;  in kvm_arch_async_page_present()
13381 		kvm_apic_set_irq(vcpu, &irq, NULL);  in kvm_arch_async_page_present()
13384 	vcpu->arch.apf.halted = false;  in kvm_arch_async_page_present()
13385 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_async_page_present()
13388 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)  in kvm_arch_async_page_present_queued()  argument
13390 	kvm_make_request(KVM_REQ_APF_READY, vcpu);  in kvm_arch_async_page_present_queued()
13391 	if (!vcpu->arch.apf.pageready_pending)  in kvm_arch_async_page_present_queued()
13392 		kvm_vcpu_kick(vcpu);  in kvm_arch_async_page_present_queued()
13395 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)  in kvm_arch_can_dequeue_async_page_present()  argument
13397 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_arch_can_dequeue_async_page_present()
13400 		return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);  in kvm_arch_can_dequeue_async_page_present()
13522 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)  in kvm_arch_no_poll()  argument
13524 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;  in kvm_arch_no_poll()
13555 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)  in kvm_fixup_and_inject_pf_error()  argument
13557 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fixup_and_inject_pf_error()
13563 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {  in kvm_fixup_and_inject_pf_error()
13565 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page  in kvm_fixup_and_inject_pf_error()
13576 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);  in kvm_fixup_and_inject_pf_error()
13585 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,  in kvm_handle_memory_failure()  argument
13589 		if (KVM_BUG_ON(!e, vcpu->kvm))  in kvm_handle_memory_failure()
13592 		kvm_inject_emulated_page_fault(vcpu, e);  in kvm_handle_memory_failure()
13603 	kvm_prepare_emulation_failure_exit(vcpu);  in kvm_handle_memory_failure()
13609 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)  in kvm_handle_invpcid()  argument
13619 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);  in kvm_handle_invpcid()
13621 		return kvm_handle_memory_failure(vcpu, r, &e);  in kvm_handle_invpcid()
13624 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13628 	pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);  in kvm_handle_invpcid()
13637 		    is_noncanonical_address(operand.gla, vcpu)) {  in kvm_handle_invpcid()
13638 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13641 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);  in kvm_handle_invpcid()
13642 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13646 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13650 		kvm_invalidate_pcid(vcpu, operand.pcid);  in kvm_handle_invpcid()
13651 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13663 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_handle_invpcid()
13664 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13667 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13673 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_mmio()  argument
13675 	struct kvm_run *run = vcpu->run;  in complete_sev_es_emulated_mmio()
13679 	BUG_ON(!vcpu->mmio_needed);  in complete_sev_es_emulated_mmio()
13682 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_sev_es_emulated_mmio()
13684 	if (!vcpu->mmio_is_write)  in complete_sev_es_emulated_mmio()
13690 		vcpu->mmio_cur_fragment++;  in complete_sev_es_emulated_mmio()
13698 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_sev_es_emulated_mmio()
13699 		vcpu->mmio_needed = 0;  in complete_sev_es_emulated_mmio()
13709 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_sev_es_emulated_mmio()
13714 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in complete_sev_es_emulated_mmio()
13719 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_write()  argument
13728 	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_write()
13737 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_write()
13738 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_write()
13743 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_write()
13744 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_write()
13746 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_write()
13747 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_write()
13748 	vcpu->run->mmio.is_write = 1;  in kvm_sev_es_mmio_write()
13749 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in kvm_sev_es_mmio_write()
13750 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_write()
13752 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_write()
13758 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_read()  argument
13767 	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_read()
13776 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_read()
13777 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_read()
13782 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_read()
13783 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_read()
13785 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_read()
13786 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_read()
13787 	vcpu->run->mmio.is_write = 0;  in kvm_sev_es_mmio_read()
13788 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_read()
13790 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_read()
13796 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)  in advance_sev_es_emulated_pio()  argument
13798 	vcpu->arch.sev_pio_count -= count;  in advance_sev_es_emulated_pio()
13799 	vcpu->arch.sev_pio_data += count * size;  in advance_sev_es_emulated_pio()
13802 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13805 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_outs()  argument
13807 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_outs()
13808 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_outs()
13810 	vcpu->arch.pio.count = 0;  in complete_sev_es_emulated_outs()
13811 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_outs()
13812 		return kvm_sev_es_outs(vcpu, size, port);  in complete_sev_es_emulated_outs()
13816 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_outs()  argument
13821 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_outs()
13822 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);  in kvm_sev_es_outs()
13825 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_outs()
13830 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_outs()
13834 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;  in kvm_sev_es_outs()
13838 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13841 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_ins()  argument
13843 	unsigned count = vcpu->arch.pio.count;  in complete_sev_es_emulated_ins()
13844 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_ins()
13845 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_ins()
13847 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);  in complete_sev_es_emulated_ins()
13848 	advance_sev_es_emulated_pio(vcpu, count, size);  in complete_sev_es_emulated_ins()
13849 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_ins()
13850 		return kvm_sev_es_ins(vcpu, size, port);  in complete_sev_es_emulated_ins()
13854 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_ins()  argument
13859 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_ins()
13860 		if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))  in kvm_sev_es_ins()
13864 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_ins()
13865 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_ins()
13869 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;  in kvm_sev_es_ins()
13873 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_string_io()  argument
13877 	vcpu->arch.sev_pio_data = data;  in kvm_sev_es_string_io()
13878 	vcpu->arch.sev_pio_count = count;  in kvm_sev_es_string_io()
13879 	return in ? kvm_sev_es_ins(vcpu, size, port)  in kvm_sev_es_string_io()
13880 		  : kvm_sev_es_outs(vcpu, size, port);  in kvm_sev_es_string_io()