192 	 * The kernel can not handle #VE's when accessing normal kernel  in tdx_parse_tdinfo()
193 	 * memory.  Ensure that no #VE will be delivered for accesses to  in tdx_parse_tdinfo()
194 	 * TD-private memory.  Only VMM-shared memory (MMIO) will #VE.  in tdx_parse_tdinfo()
209  * The TDX module spec states that #VE may be injected for a limited set of
212  *  - Emulation of the architectural #VE injection on EPT violation;
226  * information if #VE occurred due to instruction execution, but not for EPT
229 static int ve_instr_len(struct ve_info *ve)  in ve_instr_len()  argument
231 	switch (ve->exit_reason) {  in ve_instr_len()
237 		/* It is safe to use ve->instr_len for #VE due instructions */  in ve_instr_len()
238 		return ve->instr_len;  in ve_instr_len()
241 		 * For EPT violations, ve->insn_len is not defined. For those,  in ve_instr_len()
245 		WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");  in ve_instr_len()
248 		WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);  in ve_instr_len()
249 		return ve->instr_len;  in ve_instr_len()
276 static int handle_halt(struct ve_info *ve)  in handle_halt()  argument
283 	return ve_instr_len(ve);  in handle_halt()
297 static int read_msr(struct pt_regs *regs, struct ve_info *ve)  in read_msr()  argument
315 	return ve_instr_len(ve);  in read_msr()
318 static int write_msr(struct pt_regs *regs, struct ve_info *ve)  in write_msr()  argument
335 	return ve_instr_len(ve);  in write_msr()
338 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)  in handle_cpuid()  argument
356 		return ve_instr_len(ve);  in handle_cpuid()
377 	return ve_instr_len(ve);  in handle_cpuid()
404 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)  in handle_mmio()  argument
450 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
455 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
477 	if (!mmio_read(size, ve->gpa, &val))  in handle_mmio()
557 static int handle_io(struct pt_regs *regs, struct ve_info *ve)  in handle_io()  argument
559 	u32 exit_qual = ve->exit_qual;  in handle_io()
578 	return ve_instr_len(ve);  in handle_io()
582  * Early #VE exception handler. Only handles a subset of port I/O.
587 	struct ve_info ve;  in tdx_early_handle_ve()  local
590 	tdx_get_ve_info(&ve);  in tdx_early_handle_ve()
592 	if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION)  in tdx_early_handle_ve()
595 	insn_len = handle_io(regs, &ve);  in tdx_early_handle_ve()
603 void tdx_get_ve_info(struct ve_info *ve)  in tdx_get_ve_info()  argument
608 	 * Called during #VE handling to retrieve the #VE info from the  in tdx_get_ve_info()
611 	 * This has to be called early in #VE handling.  A "nested" #VE which  in tdx_get_ve_info()
614 	 * The call retrieves the #VE info from the TDX module, which also  in tdx_get_ve_info()
615 	 * clears the "#VE valid" flag. This must be done before anything else  in tdx_get_ve_info()
616 	 * because any #VE that occurs while the valid flag is set will lead to  in tdx_get_ve_info()
619 	 * Note, the TDX module treats virtual NMIs as inhibited if the #VE  in tdx_get_ve_info()
620 	 * valid flag is set. It means that NMI=>#VE will not result in a #DF.  in tdx_get_ve_info()
625 	ve->exit_reason = args.rcx;  in tdx_get_ve_info()
626 	ve->exit_qual   = args.rdx;  in tdx_get_ve_info()
627 	ve->gla         = args.r8;  in tdx_get_ve_info()
628 	ve->gpa         = args.r9;  in tdx_get_ve_info()
629 	ve->instr_len   = lower_32_bits(args.r10);  in tdx_get_ve_info()
630 	ve->instr_info  = upper_32_bits(args.r10);  in tdx_get_ve_info()
634  * Handle the user initiated #VE.
639 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_user()  argument
641 	switch (ve->exit_reason) {  in virt_exception_user()
643 		return handle_cpuid(regs, ve);  in virt_exception_user()
645 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_user()
656  * Handle the kernel #VE.
661 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_kernel()  argument
663 	switch (ve->exit_reason) {  in virt_exception_kernel()
665 		return handle_halt(ve);  in virt_exception_kernel()
667 		return read_msr(regs, ve);  in virt_exception_kernel()
669 		return write_msr(regs, ve);  in virt_exception_kernel()
671 		return handle_cpuid(regs, ve);  in virt_exception_kernel()
673 		if (is_private_gpa(ve->gpa))  in virt_exception_kernel()
675 		return handle_mmio(regs, ve);  in virt_exception_kernel()
677 		return handle_io(regs, ve);  in virt_exception_kernel()
679 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_kernel()
684 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve)  in tdx_handle_virt_exception()  argument
689 		insn_len = virt_exception_user(regs, ve);  in tdx_handle_virt_exception()
691 		insn_len = virt_exception_kernel(regs, ve);  in tdx_handle_virt_exception()
695 	/* After successful #VE handling, move the IP */  in tdx_handle_virt_exception()
866          *   - Shared mapping  => Private Page == Recoverable #VE  in tdx_early_init()
875 	 * which can result in a #VE.  But, there is never a private mapping to  in tdx_early_init()
886 	 * bringup low level code. That raises #VE which cannot be handled  in tdx_early_init()