1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021-2022 Intel Corporation */
13 #include <asm/ia32.h>
15 #include <asm/insn-eval.h>
27 #define VE_IS_IO_IN(e)		((e) & BIT(3))
36 #define TDCALL_RETURN_CODE(a)	((a) >> 32)  argument
77  * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT
79  * @reportdata: Address of the input buffer which contains user-defined
87  * Return 0 on success, -EINVAL for invalid operands, or -EIO on
102 			return -EINVAL;  in tdx_mcall_get_report0()
103 		return -EIO;  in tdx_mcall_get_report0()
111  * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote
116  * @size: size of the tdquote buffer (4KB-aligned).
126 	/* Since buf is a shared memory, set the shared (decrypted) bits */  in tdx_hcall_get_quote()
176 	 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL  in tdx_parse_tdinfo()
182 	 * The highest bit of a guest physical address is the "sharing" bit.  in tdx_parse_tdinfo()
189 	*cc_mask = BIT_ULL(gpa_width - 1);  in tdx_parse_tdinfo()
194 	 * TD-private memory.  Only VMM-shared memory (MMIO) will #VE.  in tdx_parse_tdinfo()
209  * The TDX module spec states that #VE may be injected for a limited set of
212  *  - Emulation of the architectural #VE injection on EPT violation;
214  *  - As a result of guest TD execution of a disallowed instruction,
215  *    a disallowed MSR access, or CPUID virtualization;
217  *  - A notification to the guest TD about anomalous behavior;
219  * The last one is opt-in and is not used by the kernel.
231 	switch (ve->exit_reason) {  in ve_instr_len()
237 		/* It is safe to use ve->instr_len for #VE due instructions */  in ve_instr_len()
238 		return ve->instr_len;  in ve_instr_len()
241 		 * For EPT violations, ve->insn_len is not defined. For those,  in ve_instr_len()
245 		WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");  in ve_instr_len()
248 		WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);  in ve_instr_len()
249 		return ve->instr_len;  in ve_instr_len()
263 	 * can be found in TDX Guest-Host-Communication Interface  in __halt()
281 		return -EIO;  in handle_halt()
302 		.r12 = regs->cx,  in read_msr()
307 	 * can be found in TDX Guest-Host-Communication Interface  in read_msr()
311 		return -EIO;  in read_msr()
313 	regs->ax = lower_32_bits(args.r11);  in read_msr()
314 	regs->dx = upper_32_bits(args.r11);  in read_msr()
323 		.r12 = regs->cx,  in write_msr()
324 		.r13 = (u64)regs->dx << 32 | regs->ax,  in write_msr()
329 	 * can be found in TDX Guest-Host-Communication Interface  in write_msr()
333 		return -EIO;  in write_msr()
343 		.r12 = regs->ax,  in handle_cpuid()
344 		.r13 = regs->cx,  in handle_cpuid()
351 	 * Return all-zeros for any CPUID outside the range. It matches CPU  in handle_cpuid()
352 	 * behaviour for non-supported leaf.  in handle_cpuid()
354 	if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) {  in handle_cpuid()
355 		regs->ax = regs->bx = regs->cx = regs->dx = 0;  in handle_cpuid()
360 	 * Emulate the CPUID instruction via a hypercall. More info about  in handle_cpuid()
361 	 * ABI can be found in TDX Guest-Host-Communication Interface  in handle_cpuid()
365 		return -EIO;  in handle_cpuid()
368 	 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of  in handle_cpuid()
372 	regs->ax = args.r12;  in handle_cpuid()
373 	regs->bx = args.r13;  in handle_cpuid()
374 	regs->cx = args.r14;  in handle_cpuid()
375 	regs->dx = args.r15;  in handle_cpuid()
413 	/* Only in-kernel MMIO is supported */  in handle_mmio()
415 		return -EFAULT;  in handle_mmio()
417 	if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE))  in handle_mmio()
418 		return -EFAULT;  in handle_mmio()
421 		return -EINVAL;  in handle_mmio()
425 		return -EINVAL;  in handle_mmio()
430 			return -EINVAL;  in handle_mmio()
438 	 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page.  in handle_mmio()
443 	if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE)  in handle_mmio()
444 		return -EFAULT;  in handle_mmio()
450 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
451 			return -EIO;  in handle_mmio()
455 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
456 			return -EIO;  in handle_mmio()
470 		return -EINVAL;  in handle_mmio()
473 		return -EINVAL;  in handle_mmio()
477 	if (!mmio_read(size, ve->gpa, &val))  in handle_mmio()
478 		return -EIO;  in handle_mmio()
482 		/* Zero-extend for 32-bit operation */  in handle_mmio()
500 		return -EINVAL;  in handle_mmio()
523 	 * in TDX Guest-Host-Communication Interface (GHCI) section titled  in handle_in()
529 	regs->ax &= ~mask;  in handle_in()
531 		regs->ax |= args.r11 & mask;  in handle_in()
542 	 * in TDX Guest-Host-Communication Interface (GHCI) section titled  in handle_out()
546 			       PORT_WRITE, port, regs->ax & mask);  in handle_out()
559 	u32 exit_qual = ve->exit_qual;  in handle_io()
564 		return -EIO;  in handle_io()
576 		return -EIO;  in handle_io()
582  * Early #VE exception handler. Only handles a subset of port I/O.
599 	regs->ip += insn_len;  in tdx_early_handle_ve()
611 	 * This has to be called early in #VE handling.  A "nested" #VE which  in tdx_get_ve_info()
612 	 * occurs before this will raise a #DF and is not recoverable.  in tdx_get_ve_info()
620 	 * valid flag is set. It means that NMI=>#VE will not result in a #DF.  in tdx_get_ve_info()
625 	ve->exit_reason = args.rcx;  in tdx_get_ve_info()
626 	ve->exit_qual   = args.rdx;  in tdx_get_ve_info()
627 	ve->gla         = args.r8;  in tdx_get_ve_info()
628 	ve->gpa         = args.r9;  in tdx_get_ve_info()
629 	ve->instr_len   = lower_32_bits(args.r10);  in tdx_get_ve_info()
630 	ve->instr_info  = upper_32_bits(args.r10);  in tdx_get_ve_info()
637  * or -errno on error.
641 	switch (ve->exit_reason) {  in virt_exception_user()
645 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_user()
646 		return -EIO;  in virt_exception_user()
659  * or -errno on error.
663 	switch (ve->exit_reason) {  in virt_exception_kernel()
673 		if (is_private_gpa(ve->gpa))  in virt_exception_kernel()
674 			panic("Unexpected EPT-violation on private memory.");  in virt_exception_kernel()
679 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_kernel()
680 		return -EIO;  in virt_exception_kernel()
696 	regs->ip += insn_len;  in tdx_handle_virt_exception()
704 	 * TDX guest is responsible for flushing TLB on private->shared  in tdx_tlb_flush_required()
705 	 * transition. VMM is responsible for flushing on shared->private.  in tdx_tlb_flush_required()
713 	 * flush to avoid integrity failures in the face of a buggy or  in tdx_tlb_flush_required()
732  * can be found in TDX Guest-Host-Communication Interface (GHCI),
737 	/* Retrying the hypercall a second time should succeed; use 3 just in case */  in tdx_map_gpa()
738 	const int max_retries_per_page = 3;  in tdx_map_gpa()
752 			.r13 = end - start };  in tdx_map_gpa()
768 		/* "Consume" a retry without forward progress */  in tdx_map_gpa()
794 	/* shared->private conversion requires memory to be accepted before use */  in tdx_enc_status_changed()
805 	 * Only handle shared->private conversion here.  in tdx_enc_status_change_prepare()
817 	 * Only handle private->shared conversion here.  in tdx_enc_status_change_finish()
829 		.r9 = -1ULL,  in tdx_early_init()
832 	u32 eax, sig[3];  in tdx_early_init()
857 	physical_mask &= cc_mask - 1;  in tdx_early_init()
865 	 *   - Private mapping => Shared Page  == Guest shutdown  in tdx_early_init()
866          *   - Shared mapping  => Private Page == Recoverable #VE  in tdx_early_init()
874 	 * In both cases there is a temporary shared mapping to a private page,  in tdx_early_init()
875 	 * which can result in a #VE.  But, there is never a private mapping to  in tdx_early_init()
876 	 * a shared page.  in tdx_early_init()
889 	 * Intel-TDX has a secure RDMSR hypercall, but that needs to be  in tdx_early_init()