1 // SPDX-License-Identifier: GPL-2.0-or-later
7  * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
10  * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
12  * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
14  * 2005-Mar	Roland McGrath <roland@redhat.com>
15  *		Fixed to handle %rip-relative addressing mode correctly.
16  * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
18  *		<prasanna@in.ibm.com> added function-return probes.
19  * 2005-May	Rusty Lynch <rusty.lynch@intel.com>
21  * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
22  *		kprobe-booster and kretprobe-booster for i386.
23  * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
24  *		and kretprobe-booster for x86-64
25  * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
47 #include <asm/text-patching.h>
61 #define stack_addr(regs) ((unsigned long *)regs->sp)
72 	 * This is non-const and volatile to keep gcc from statically
78 	/*      ----------------------------------------------          */
95 	/*      -----------------------------------------------         */
117 	insn->raddr = (s32)((long)(to) - ((long)(from) + 5));  in __synthesize_relative_insn()
118 	insn->op = op;  in __synthesize_relative_insn()
156  * Returns non-zero if INSN is boostable.
166 	/* 2nd-byte opcode */  in can_boost()
167 	if (insn->opcode.nbytes == 2)  in can_boost()
168 		return test_bit(insn->opcode.bytes[1],  in can_boost()
171 	if (insn->opcode.nbytes != 1)  in can_boost()
174 	/* Can't boost Address-size override prefix */  in can_boost()
175 	if (unlikely(inat_is_address_size_prefix(insn->attr)))  in can_boost()
178 	opcode = insn->opcode.bytes[0];  in can_boost()
189 		/* can't boost software-interruptions */  in can_boost()
201 		/* CS override prefix and call are not boostable */  in can_boost()
229 	 * Basically, kp->ainsn.insn has an original instruction.  in __recover_probed_insn()
230 	 * However, RIP-relative instruction can not do single-stepping  in __recover_probed_insn()
233 	 * from the kp->ainsn.insn.  in __recover_probed_insn()
235 	 * On the other hand, in case on normal Kprobe, kp->opcode has a copy  in __recover_probed_insn()
237 	 * by int3. And the instruction at kp->addr is not modified by kprobes  in __recover_probed_insn()
239 	 * from it and kp->opcode.  in __recover_probed_insn()
244 	 * Fortunately, we know that the original code is the ideal 5-byte  in __recover_probed_insn()
254 		buf[0] = kp->opcode;  in __recover_probed_insn()
286 	addr = paddr - offset;  in can_probe()
293 		 * relative-jump. Since the relative-jump itself is  in can_probe()
315  * Returns non-zero if opcode modifies the interrupt flag.
335  * and adjust the displacement if the instruction uses the %rip-relative
362 	if (insn->opcode.bytes[0] == INT3_INSN_OPCODE)  in __copy_instruction()
375 		 * The copied instruction uses the %rip-relative addressing  in __copy_instruction()
380 		 * this calculation, since we need a signed 32-bit result to  in __copy_instruction()
381 		 * be sign-extended to 64 bits when it's added to the %rip  in __copy_instruction()
382 		 * value and yield the same 64-bit result that the sign-  in __copy_instruction()
383 		 * extension of the original signed 32-bit displacement would  in __copy_instruction()
386 		newdisp = (u8 *) src + (s64) insn->displacement.value  in __copy_instruction()
387 			  - (u8 *) real;  in __copy_instruction()
396 	return insn->length;  in __copy_instruction()
403 	int len = insn->length;  in prepare_boost()
405 	if (can_boost(insn, p->addr) &&  in prepare_boost()
406 	    MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) {  in prepare_boost()
411 		synthesize_reljump(buf + len, p->ainsn.insn + len,  in prepare_boost()
412 				   p->addr + insn->length);  in prepare_boost()
414 		p->ainsn.boostable = true;  in prepare_boost()
416 		p->ainsn.boostable = false;  in prepare_boost()
433 	 * First make the page read-only, and only then make it executable to  in alloc_insn_page()
460 	len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn);  in arch_copy_kprobe()
462 		return -EINVAL;  in arch_copy_kprobe()
471 	p->ainsn.if_modifier = is_IF_modifier(buf);  in arch_copy_kprobe()
474 	p->opcode = buf[0];  in arch_copy_kprobe()
476 	p->ainsn.tp_len = len;  in arch_copy_kprobe()
477 	perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len);  in arch_copy_kprobe()
480 	text_poke(p->ainsn.insn, buf, len);  in arch_copy_kprobe()
489 	if (alternatives_text_reserved(p->addr, p->addr))  in arch_prepare_kprobe()
490 		return -EINVAL;  in arch_prepare_kprobe()
492 	if (!can_probe((unsigned long)p->addr))  in arch_prepare_kprobe()
493 		return -EILSEQ;  in arch_prepare_kprobe()
495 	p->ainsn.insn = get_insn_slot();  in arch_prepare_kprobe()
496 	if (!p->ainsn.insn)  in arch_prepare_kprobe()
497 		return -ENOMEM;  in arch_prepare_kprobe()
501 		free_insn_slot(p->ainsn.insn, 0);  in arch_prepare_kprobe()
502 		p->ainsn.insn = NULL;  in arch_prepare_kprobe()
512 	text_poke(p->addr, &int3, 1);  in arch_arm_kprobe()
514 	perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1);  in arch_arm_kprobe()
521 	perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1);  in arch_disarm_kprobe()
522 	text_poke(p->addr, &p->opcode, 1);  in arch_disarm_kprobe()
528 	if (p->ainsn.insn) {  in arch_remove_kprobe()
530 		perf_event_text_poke(p->ainsn.insn, p->ainsn.insn,  in arch_remove_kprobe()
531 				     p->ainsn.tp_len, NULL, 0);  in arch_remove_kprobe()
532 		free_insn_slot(p->ainsn.insn, p->ainsn.boostable);  in arch_remove_kprobe()
533 		p->ainsn.insn = NULL;  in arch_remove_kprobe()
540 	kcb->prev_kprobe.kp = kprobe_running();  in save_previous_kprobe()
541 	kcb->prev_kprobe.status = kcb->kprobe_status;  in save_previous_kprobe()
542 	kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;  in save_previous_kprobe()
543 	kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;  in save_previous_kprobe()
549 	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);  in restore_previous_kprobe()
550 	kcb->kprobe_status = kcb->prev_kprobe.status;  in restore_previous_kprobe()
551 	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;  in restore_previous_kprobe()
552 	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;  in restore_previous_kprobe()
560 	kcb->kprobe_saved_flags = kcb->kprobe_old_flags  in set_current_kprobe()
561 		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));  in set_current_kprobe()
562 	if (p->ainsn.if_modifier)  in set_current_kprobe()
563 		kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;  in set_current_kprobe()
586 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)  in arch_prepare_kretprobe()  argument
590 	ri->ret_addr = (kprobe_opcode_t *) *sara;  in arch_prepare_kretprobe()
591 	ri->fp = sara;  in arch_prepare_kretprobe()
605 	if (p->ainsn.boostable && !p->post_handler) {  in setup_singlestep()
606 		/* Boost up -- we can execute copied instructions directly */  in setup_singlestep()
614 		regs->ip = (unsigned long)p->ainsn.insn;  in setup_singlestep()
621 		kcb->kprobe_status = KPROBE_REENTER;  in setup_singlestep()
623 		kcb->kprobe_status = KPROBE_HIT_SS;  in setup_singlestep()
626 	regs->flags |= X86_EFLAGS_TF;  in setup_singlestep()
627 	regs->flags &= ~X86_EFLAGS_IF;  in setup_singlestep()
629 	if (p->opcode == INT3_INSN_OPCODE)  in setup_singlestep()
630 		regs->ip = (unsigned long)p->addr;  in setup_singlestep()
632 		regs->ip = (unsigned long)p->ainsn.insn;  in setup_singlestep()
644 	switch (kcb->kprobe_status) {  in reenter_kprobe()
653 		 * after, single-stepping of a probed instruction. This entire  in reenter_kprobe()
684 	addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));  in kprobe_int3_handler()
700 			kcb->kprobe_status = KPROBE_HIT_ACTIVE;  in kprobe_int3_handler()
703 			 * If we have no pre-handler or it returned 0, we  in kprobe_int3_handler()
705 			 * pre-handler and it returned non-zero, that means  in kprobe_int3_handler()
709 			if (!p->pre_handler || !p->pre_handler(p, regs))  in kprobe_int3_handler()
725 		regs->ip = (unsigned long)addr;  in kprobe_int3_handler()
765 	".size kretprobe_trampoline, .-kretprobe_trampoline\n"
777 	regs->cs = __KERNEL_CS;  in trampoline_handler()
779 	regs->gs = 0;  in trampoline_handler()
781 	regs->ip = (unsigned long)&kretprobe_trampoline;  in trampoline_handler()
782 	regs->orig_ax = ~0UL;  in trampoline_handler()
784 	return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, &regs->sp);  in trampoline_handler()
789  * Called after single-stepping.  p->addr is the address of the
792  * temporarily put back the original opcode to single-step, we
793  * single-stepped a copy of the instruction.  The address of this
794  * copy is p->ainsn.insn.
796  * This function prepares to return from the post-single-step
803  * 1) If the single-stepped instruction was pushfl, then the TF and IF
804  * flags are set in the just-pushed flags, and may need to be cleared.
806  * 2) If the single-stepped instruction was a call, the return address
810  * If this is the first time we've single-stepped the instruction at
819 	unsigned long copy_ip = (unsigned long)p->ainsn.insn;  in resume_execution()
820 	unsigned long orig_ip = (unsigned long)p->addr;  in resume_execution()
821 	kprobe_opcode_t *insn = p->ainsn.insn;  in resume_execution()
826 	regs->flags &= ~X86_EFLAGS_TF;  in resume_execution()
830 		*tos |= kcb->kprobe_old_flags;  in resume_execution()
837 	case 0xea:	/* jmp absolute -- ip is correct */  in resume_execution()
839 		p->ainsn.boostable = true;  in resume_execution()
841 	case 0xe8:	/* call relative - Fix return addr */  in resume_execution()
842 		*tos = orig_ip + (*tos - copy_ip);  in resume_execution()
845 	case 0x9a:	/* call absolute -- same as call absolute, indirect */  in resume_execution()
846 		*tos = orig_ip + (*tos - copy_ip);  in resume_execution()
856 			*tos = orig_ip + (*tos - copy_ip);  in resume_execution()
864 			p->ainsn.boostable = true;  in resume_execution()
871 	regs->ip += orig_ip - copy_ip;  in resume_execution()
891 	regs->flags |= kcb->kprobe_saved_flags;  in kprobe_debug_handler()
893 	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {  in kprobe_debug_handler()
894 		kcb->kprobe_status = KPROBE_HIT_SSDONE;  in kprobe_debug_handler()
895 		cur->post_handler(cur, regs, 0);  in kprobe_debug_handler()
899 	if (kcb->kprobe_status == KPROBE_REENTER) {  in kprobe_debug_handler()
910 	if (regs->flags & X86_EFLAGS_TF)  in kprobe_debug_handler()
922 	if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) {  in kprobe_fault_handler()
923 		/* This must happen on single-stepping */  in kprobe_fault_handler()
924 		WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS &&  in kprobe_fault_handler()
925 			kcb->kprobe_status != KPROBE_REENTER);  in kprobe_fault_handler()
933 		regs->ip = (unsigned long)cur->addr;  in kprobe_fault_handler()
939 		regs->flags &= ~X86_EFLAGS_TF;  in kprobe_fault_handler()
945 		regs->flags |= kcb->kprobe_old_flags;  in kprobe_fault_handler()
947 		if (kcb->kprobe_status == KPROBE_REENTER)  in kprobe_fault_handler()
951 	} else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE ||  in kprobe_fault_handler()
952 		   kcb->kprobe_status == KPROBE_HIT_SSDONE) {  in kprobe_fault_handler()
965 		 * user-specified handler try to fix it first.  in kprobe_fault_handler()
967 		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))  in kprobe_fault_handler()