1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
18 #include <linux/ftrace.h>
33 #include <asm/ftrace.h>
35 #include <asm/text-patching.h>
47 	 * ftrace has it set to "read/write".  in ftrace_arch_code_modify_prepare()
94 		return -EFAULT;  in ftrace_verify_code()
101 		return -EINVAL;  in ftrace_verify_code()
130 	unsigned long ip = rec->ip;  in ftrace_make_nop()
148 	 * x86 overrides ftrace_replace_code -- this function will never be used  in ftrace_make_nop()
152 	return -EINVAL;  in ftrace_make_nop()
157 	unsigned long ip = rec->ip;  in ftrace_make_call()
164 	return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_call()
179 	return -EINVAL;  in ftrace_modify_call()
219 			old = ftrace_call_replace(rec->ip, ftrace_get_addr_curr(rec));  in ftrace_replace_code()
223 		ret = ftrace_verify_code(rec->ip, old);  in ftrace_replace_code()
242 			new = ftrace_call_replace(rec->ip, ftrace_get_addr_new(rec));  in ftrace_replace_code()
250 		text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL);  in ftrace_replace_code()
264 static inline void *alloc_tramp(unsigned long size)  in alloc_tramp()  argument
266 	return execmem_alloc(EXECMEM_FTRACE, size);  in alloc_tramp()
273 /* Defined as markers to the end of the ftrace default trampolines */
281 /* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
313 	unsigned long size;  in create_trampoline()  local
323 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in create_trampoline()
337 	size = end_offset - start_offset;  in create_trampoline()
340 	 * Allocate enough size to store the ftrace_caller code,  in create_trampoline()
344 	trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *));  in create_trampoline()
348 	*tramp_size = size + RET_SIZE + sizeof(void *);  in create_trampoline()
352 	ret = copy_from_kernel_nofault(trampoline, (void *)start_offset, size);  in create_trampoline()
356 	ip = trampoline + size;  in create_trampoline()
363 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in create_trampoline()
365 		ip = trampoline + (jmp_offset - start_offset);  in create_trampoline()
381 	ptr = (unsigned long *)(trampoline + size + RET_SIZE);  in create_trampoline()
384 	op_offset -= start_offset;  in create_trampoline()
393 	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;  in create_trampoline()
402 	call_offset -= start_offset;  in create_trampoline()
414 	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;  in create_trampoline()
429 	unsigned long size;  in set_ftrace_ops_ro()  local
432 		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in set_ftrace_ops_ro()
435 		if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in set_ftrace_ops_ro()
442 		size = end_offset - start_offset;  in set_ftrace_ops_ro()
443 		size = size + RET_SIZE + sizeof(void *);  in set_ftrace_ops_ro()
444 		npages = DIV_ROUND_UP(size, PAGE_SIZE);  in set_ftrace_ops_ro()
445 		set_memory_ro((unsigned long)ops->trampoline, npages);  in set_ftrace_ops_ro()
462 	return call_offset - start_offset;  in calc_trampoline_call_offset()
470 	unsigned int size;  in arch_ftrace_update_trampoline()  local
473 	if (!ops->trampoline) {  in arch_ftrace_update_trampoline()
474 		ops->trampoline = create_trampoline(ops, &size);  in arch_ftrace_update_trampoline()
475 		if (!ops->trampoline)  in arch_ftrace_update_trampoline()
477 		ops->trampoline_size = size;  in arch_ftrace_update_trampoline()
485 	if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_update_trampoline()
488 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);  in arch_ftrace_update_trampoline()
489 	ip = ops->trampoline + offset;  in arch_ftrace_update_trampoline()
519  * If the ops->trampoline was not allocated, then it probably
520  * has a static trampoline func, or is the ftrace caller itself.
525 	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;  in static_tramp_func()
528 	if (ops && ops->trampoline) {  in static_tramp_func()
535 		if (ops->trampoline == FTRACE_GRAPH_ADDR)  in static_tramp_func()
556 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_trampoline_func()
559 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);  in arch_ftrace_trampoline_func()
560 	return addr_from_call((void *)ops->trampoline + offset);  in arch_ftrace_trampoline_func()
565 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_trampoline_free()
568 	tramp_free((void *)ops->trampoline);  in arch_ftrace_trampoline_free()
569 	ops->trampoline = 0;  in arch_ftrace_trampoline_free()
611 	 * When resuming from suspend-to-ram, this function can be indirectly  in skip_ftrace_return()
625 	if (atomic_read(&current->tracing_graph_pause))  in skip_ftrace_return()
650 	struct pt_regs *regs = &arch_ftrace_regs(fregs)->regs;  in ftrace_graph_func()