1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
18 #include <linux/ftrace.h>
32 #include <asm/ftrace.h>
34 #include <asm/text-patching.h>
46 	 * ftrace has it set to "read/write".  in ftrace_arch_code_modify_prepare()
93 		return -EFAULT;  in ftrace_verify_code()
100 		return -EINVAL;  in ftrace_verify_code()
129 	unsigned long ip = rec->ip;  in ftrace_make_nop()
147 	 * x86 overrides ftrace_replace_code -- this function will never be used  in ftrace_make_nop()
151 	return -EINVAL;  in ftrace_make_nop()
156 	unsigned long ip = rec->ip;  in ftrace_make_call()
163 	return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_call()
178 	return -EINVAL;  in ftrace_modify_call()
218 			old = ftrace_call_replace(rec->ip, ftrace_get_addr_curr(rec));  in ftrace_replace_code()
222 		ret = ftrace_verify_code(rec->ip, old);  in ftrace_replace_code()
241 			new = ftrace_call_replace(rec->ip, ftrace_get_addr_new(rec));  in ftrace_replace_code()
249 		text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL);  in ftrace_replace_code()
266 static inline void *alloc_tramp(unsigned long size)  in alloc_tramp()  argument
268 	return module_alloc(size);  in alloc_tramp()
276 static inline void *alloc_tramp(unsigned long size)  in alloc_tramp()  argument
283 /* Defined as markers to the end of the ftrace default trampolines */
291 /* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
322 	unsigned long size;  in create_trampoline()  local
332 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in create_trampoline()
346 	size = end_offset - start_offset;  in create_trampoline()
349 	 * Allocate enough size to store the ftrace_caller code,  in create_trampoline()
353 	trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *));  in create_trampoline()
357 	*tramp_size = size + RET_SIZE + sizeof(void *);  in create_trampoline()
361 	ret = copy_from_kernel_nofault(trampoline, (void *)start_offset, size);  in create_trampoline()
365 	ip = trampoline + size;  in create_trampoline()
372 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in create_trampoline()
374 		ip = trampoline + (jmp_offset - start_offset);  in create_trampoline()
390 	ptr = (unsigned long *)(trampoline + size + RET_SIZE);  in create_trampoline()
393 	op_offset -= start_offset;  in create_trampoline()
402 	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;  in create_trampoline()
411 	call_offset -= start_offset;  in create_trampoline()
423 	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;  in create_trampoline()
438 	unsigned long size;  in set_ftrace_ops_ro()  local
441 		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in set_ftrace_ops_ro()
444 		if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in set_ftrace_ops_ro()
451 		size = end_offset - start_offset;  in set_ftrace_ops_ro()
452 		size = size + RET_SIZE + sizeof(void *);  in set_ftrace_ops_ro()
453 		npages = DIV_ROUND_UP(size, PAGE_SIZE);  in set_ftrace_ops_ro()
454 		set_memory_ro((unsigned long)ops->trampoline, npages);  in set_ftrace_ops_ro()
471 	return call_offset - start_offset;  in calc_trampoline_call_offset()
479 	unsigned int size;  in arch_ftrace_update_trampoline()  local
482 	if (!ops->trampoline) {  in arch_ftrace_update_trampoline()
483 		ops->trampoline = create_trampoline(ops, &size);  in arch_ftrace_update_trampoline()
484 		if (!ops->trampoline)  in arch_ftrace_update_trampoline()
486 		ops->trampoline_size = size;  in arch_ftrace_update_trampoline()
494 	if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_update_trampoline()
497 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);  in arch_ftrace_update_trampoline()
498 	ip = ops->trampoline + offset;  in arch_ftrace_update_trampoline()
528  * If the ops->trampoline was not allocated, then it probably
529  * has a static trampoline func, or is the ftrace caller itself.
534 	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;  in static_tramp_func()
537 	if (ops && ops->trampoline) {  in static_tramp_func()
544 		if (ops->trampoline == FTRACE_GRAPH_ADDR)  in static_tramp_func()
565 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_trampoline_func()
568 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);  in arch_ftrace_trampoline_func()
569 	return addr_from_call((void *)ops->trampoline + offset);  in arch_ftrace_trampoline_func()
574 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_trampoline_free()
577 	tramp_free((void *)ops->trampoline);  in arch_ftrace_trampoline_free()
578 	ops->trampoline = 0;  in arch_ftrace_trampoline_free()
628 	 * When resuming from suspend-to-ram, this function can be indirectly  in prepare_ftrace_return()
642 	if (unlikely(atomic_read(&current->tracing_graph_pause)))  in prepare_ftrace_return()
659 	struct pt_regs *regs = &fregs->regs;  in ftrace_graph_func()