3  *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
6  *	Gareth Hughes <gareth@valinux.com>, May 2000
51 #include <asm/text-patching.h>
67 #include <asm/insn-eval.h>
75 #include <asm/processor-flags.h>
100  * FineIBT:      f0 75 f9                lock jne . - 6
103  * static_call:  0f b9 cc                ud1    %esp,%ecx
105  * Notably UBSAN uses EAX, static_call uses ECX.
128 		*len = addr - start;  in decode_bug()
133 		*len = addr - start;  in decode_bug()
145 		*len = addr - start;  in decode_bug()
176 	*len = addr - start;  in decode_bug()
203 		tsk->thread.error_code = error_code;  in do_trap_no_signal()
204 		tsk->thread.trap_nr = trapnr;  in do_trap_no_signal()
220 	tsk->thread.error_code = error_code;  in do_trap_no_signal()
221 	tsk->thread.trap_nr = trapnr;  in do_trap_no_signal()
223 	return -1;  in do_trap_no_signal()
233 			tsk->comm, task_pid_nr(tsk), type, desc,  in show_signal()
234 			regs->ip, regs->sp, error_code);  in show_signal()
235 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
275  * This address is usually regs->ip, but when an uprobe moved the code out
276  * of line then regs->ip points to the XOL code which would confuse
278  * a trap happened in XOL code then uprobe maps regs->ip back to the
309 	unsigned long addr = regs->ip;  in handle_bug()
332 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
337 		if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
355 				(void *)regs->ip);  in handle_bug()
364 	 * When continuing, and regs->ip hasn't changed, move it to the next  in handle_bug()
369 		if (regs->ip == addr)  in handle_bug()
370 			regs->ip += ud_len;  in handle_bug()
372 		regs->ip = addr;  in handle_bug()
375 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
452 	const char *name = stack_type_name(info->type);  in handle_stack_overflow()
455 	       name, (void *)fault_address, info->begin, info->end);  in handle_stack_overflow()
488  * while the stack is read-only are, in fact, recoverable.
512 	 * If IRET takes a non-IST fault on the espfix64 stack, then we  in DEFINE_IDTENTRY_DF()
525 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in DEFINE_IDTENTRY_DF()
526 		regs->cs == __KERNEL_CS &&  in DEFINE_IDTENTRY_DF()
527 		regs->ip == (unsigned long)native_irq_return_iret)  in DEFINE_IDTENTRY_DF()
529 		struct pt_regs *gpregs = (struct pt_regs *)this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1;  in DEFINE_IDTENTRY_DF()
530 		unsigned long *p = (unsigned long *)regs->sp;  in DEFINE_IDTENTRY_DF()
533 		 * regs->sp points to the failing IRET frame on the  in DEFINE_IDTENTRY_DF()
535 		 * in gpregs->ss through gpregs->ip.  in DEFINE_IDTENTRY_DF()
538 		gpregs->ip	= p[0];  in DEFINE_IDTENTRY_DF()
539 		gpregs->cs	= p[1];  in DEFINE_IDTENTRY_DF()
540 		gpregs->flags	= p[2];  in DEFINE_IDTENTRY_DF()
541 		gpregs->sp	= p[3];  in DEFINE_IDTENTRY_DF()
542 		gpregs->ss	= p[4];  in DEFINE_IDTENTRY_DF()
543 		gpregs->orig_ax = 0;  /* Missing (lost) #GP error code */  in DEFINE_IDTENTRY_DF()
556 		regs->ip = (unsigned long)asm_exc_general_protection;  in DEFINE_IDTENTRY_DF()
557 		regs->sp = (unsigned long)&gpregs->orig_ax;  in DEFINE_IDTENTRY_DF()
567 	tsk->thread.error_code = error_code;  in DEFINE_IDTENTRY_DF()
568 	tsk->thread.trap_nr = X86_TRAP_DF;  in DEFINE_IDTENTRY_DF()
574 	 * take any non-IST exception while too close to the bottom of  in DEFINE_IDTENTRY_DF()
578 	 * According to the SDM (footnote in 6.15 under "Interrupt 14 -  in DEFINE_IDTENTRY_DF()
579 	 * Page-Fault Exception (#PF):  in DEFINE_IDTENTRY_DF()
643  * out whether any part of the access to that address was non-canonical.
652 	if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,  in get_kernel_gp_address()
661 	if (*addr == -1UL)  in get_kernel_gp_address()
667 	 *  - the operand is not in the kernel half  in get_kernel_gp_address()
668 	 *  - the last byte of the operand is not in the user canonical half  in get_kernel_gp_address()
671 	    *addr + insn.opnd_bytes - 1 > __VIRTUAL_MASK)  in get_kernel_gp_address()
682 	struct thread_struct *t = &current->thread;  in fixup_iopl_exception()
686 	if (!IS_ENABLED(CONFIG_X86_IOPL_IOPERM) || t->iopl_emul != 3)  in fixup_iopl_exception()
698 	if (!t->iopl_warn && printk_ratelimit()) {  in fixup_iopl_exception()
700 		       current->comm, task_pid_nr(current), ip);  in fixup_iopl_exception()
703 		t->iopl_warn = 1;  in fixup_iopl_exception()
706 	regs->ip += 1;  in fixup_iopl_exception()
725 	 * in *before* interrupts are re-enabled.  in try_fixup_enqcmd_gp()
740 	if (!mm_valid_pasid(current->mm))  in try_fixup_enqcmd_gp()
743 	pasid = mm_get_enqcmd_pasid(current->mm);  in try_fixup_enqcmd_gp()
749 	if (current->pasid_activated)  in try_fixup_enqcmd_gp()
753 	current->pasid_activated = 1;  in try_fixup_enqcmd_gp()
768 	current->thread.error_code = error_code;  in gp_try_fixup_and_notify()
769 	current->thread.trap_nr = trapnr;  in gp_try_fixup_and_notify()
773 	 * from kprobe_running(), we have to be non-preemptible.  in gp_try_fixup_and_notify()
785 	current->thread.error_code = error_code;  in gp_user_force_sig_segv()
786 	current->thread.trap_nr = trapnr;  in gp_user_force_sig_segv()
829 		snprintf(desc, sizeof(desc), "segment-related " GPFSTR);  in DEFINE_IDTENTRY_ERRORCODE()
835 			 (hint == GP_NON_CANONICAL) ? "probably for non-canonical address"  in DEFINE_IDTENTRY_ERRORCODE()
840 	 * KASAN is interested only in the non-canonical case, clear it  in DEFINE_IDTENTRY_ERRORCODE()
918  * Help handler running on a per-cpu (IST or entry trampoline) stack
924 	struct pt_regs *regs = (struct pt_regs *)current_top_of_stack() - 1;  in sync_regs()
938 	 * In the SYSCALL entry path the RSP value comes from user-space - don't  in vc_switch_off_ist()
949 	 * use the fall-back stack instead in this case.  in vc_switch_off_ist()
951 	sp    = regs->sp;  in vc_switch_off_ist()
960 	 * Found a safe stack - switch to it as if the entry didn't happen via  in vc_switch_off_ist()
964 	sp = ALIGN_DOWN(sp, 8) - sizeof(*regs_ret);  in vc_switch_off_ist()
985 	new_stack = (struct pt_regs *)__this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1;  in fixup_bad_iret()
988 	__memcpy(&tmp.ip, (void *)bad_regs->sp, 5*8);  in fixup_bad_iret()
1005 	 * code that can be single-stepped in the SYSENTER entry path, then  in is_sysenter_singlestep()
1006 	 * assume that this is a useless single-step trap due to SYSENTER  in is_sysenter_singlestep()
1012 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
1013 		(unsigned long)__end_SYSENTER_singlestep_region -  in is_sysenter_singlestep()
1016 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
1017 		(unsigned long)__end_entry_SYSENTER_compat -  in is_sysenter_singlestep()
1031 	 *   Certain debug exceptions may clear bits 0-3. The remaining  in debug_read_clear_dr6()
1047  * Our handling of the processor debug registers is non-trivial.
1051  * only set watchpoints on userspace addresses. Therefore the in-kernel
1075 	 * consumed - hw_breakpoint_handler(), single_stop_cont().  in notify_debug()
1078 	 * for signals - ptrace_triggered(), kgdb_hw_overflow_handler().  in notify_debug()
1118 		 * it for userspace, but we just took a kernel #DB, so re-set  in exc_debug_kernel()
1146 	 * The kernel doesn't use TF single-step outside of:  in exc_debug_kernel()
1148 	 *  - Kprobes, consumed through kprobe_debug_handler()  in exc_debug_kernel()
1149 	 *  - KGDB, consumed through notify_debug()  in exc_debug_kernel()
1157 		regs->flags &= ~X86_EFLAGS_TF;  in exc_debug_kernel()
1194 	current->thread.virtual_dr6 = (dr6 & DR_STEP);  in exc_debug_user()
1226 	dr6 |= current->thread.virtual_dr6;  in exc_debug_user()
1298 	struct fpu *fpu = &task->thread.fpu;  in math_error()
1309 		task->thread.error_code = 0;  in math_error()
1310 		task->thread.trap_nr = trapnr;  in math_error()
1324 	task->thread.trap_nr	= trapnr;  in math_error()
1325 	task->thread.error_code = 0;  in math_error()
1404 	case -EPERM:  in handle_xfd_event()
1407 	case -EFAULT:  in handle_xfd_event()
1444 		 * to kill the task than getting stuck in a never-ending  in DEFINE_IDTENTRY()
1482  * never generated on accesses to normal, TD-private memory that has been
1489  * IRET will re-enable NMIs and nested NMI will corrupt the NMI stack.
1492  * the NMI entry code. Entry code paths do not access TD-shared memory,
1506  * the interrupt-disabled region before TDGETVEINFO, a #DF (fault
1519 	 * NMIs/Machine-checks/Interrupts will be in a disabled state  in DEFINE_IDTENTRY()
1557 	/* Init GHCB memory pages when running as an SEV-ES guest */  in trap_init()