1 // SPDX-License-Identifier: GPL-2.0
14 #include <linux/trace.h>
105 #define FTRACE_ENTRY(name, struct_name, id, tstruct, print)		\  argument
112 #define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk)  argument
115 #define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print,	regfn)	\  argument
116 	FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print))
119 #define FTRACE_ENTRY_PACKED(name, struct_name, id, tstruct, print)	\  argument
120 	FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print)) __packed
181  * The CPU trace array - it consists of thousands of trace entries
183  * the trace, etc.)
241 static inline bool pid_type_enabled(int type, struct trace_pid_list *pid_list,  in pid_type_enabled()
249 static inline bool still_need_pid_events(int type, struct trace_pid_list *pid_list,  in still_need_pid_events()
263  * struct cond_snapshot - conditional snapshot data and callback
273  * the trace_array.  Because the tr->max_lock is held throughout the
275  * cond_snapshot and cond_data associated with the per-instance
284  * user-defined cond_data by tracing_cond_snapshot_enable().
286  * associated with the trace instance by
292  *	callback function is invoked with the tr->max_lock held.  The
306  * struct trace_func_repeats - used to keep track of the consecutive
322  * The trace array - an array of per-CPU trace arrays. This is the
332 	 * The max_buffer is used to snapshot the trace when a maximum
334 	 * Some tracers will use this to store a maximum trace while
408 	cpumask_var_t		tracing_cpumask; /* only trace on set CPUs */
456 static inline bool module_exists(const char *module)  in module_exists()
480  * The global tracer (top) should be the first trace array added,
483 static inline struct trace_array *top_trace_array(void)  in top_trace_array()
492 	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));  in top_trace_array()
500 #define IF_ASSIGN(var, entry, etype, id)			\  argument
503 		WARN_ON(id != 0 && (entry)->type != id);	\
515  * IF_ASSIGN(var, ent, type, id);
517  *  Where "type" is the trace type that includes the trace_entry
518  *  as the "ent" item. And "id" is the trace identifier that is
521  *  If the type can have more than one id, then use zero.
569 	struct tracer		*trace;  member
584  * struct tracer - a specific tracer and its callbacks to interact with tracefs
591  * @open: called when the trace file is opened
593  * @close: called when the trace file is released
599  * @print_line: callback that prints a trace
624 	int			(*selftest)(struct tracer *trace,
632 	/* Return 0 if OK with change, else return non-zero */
647 static inline struct ring_buffer_iter *
650 	return iter->buffer_iter ? iter->buffer_iter[cpu] : NULL;  in trace_buffer_iter()
724 void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops,
726 int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
779 static inline void latency_fsnotify(struct trace_array *tr) { }  in latency_fsnotify()
785 static inline void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,  in __trace_stack()
811 static inline void ftrace_init_trace_array(struct trace_array *tr) { }  in ftrace_init_trace_array()
824 extern int trace_selftest_startup_function(struct tracer *trace,
826 extern int trace_selftest_startup_function_graph(struct tracer *trace,
828 extern int trace_selftest_startup_irqsoff(struct tracer *trace,
830 extern int trace_selftest_startup_preemptoff(struct tracer *trace,
832 extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
834 extern int trace_selftest_startup_wakeup(struct tracer *trace,
836 extern int trace_selftest_startup_nop(struct tracer *trace,
838 extern int trace_selftest_startup_branch(struct tracer *trace,
847 static inline void __init disable_tracing_selftest(const char *reason)  in disable_tracing_selftest()
896 static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)  in ftrace_hash_empty()
898 	return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));  in ftrace_hash_empty()
928 static inline void ftrace_graph_graph_time_control(bool enable) { }  in ftrace_graph_graph_time_control()
939 			       struct ftrace_graph_ent *trace,
942 				struct ftrace_graph_ent *trace,
946 				 struct ftrace_graph_ret *trace,
959 	 * at a start of graph tracing, and we want to trace
962 	 * trace. In an even more unlikely case, depth could be 2
986 static inline unsigned long ftrace_graph_depth(unsigned long *task_var)  in ftrace_graph_depth()
991 static inline void ftrace_graph_set_depth(unsigned long *task_var, int depth)  in ftrace_graph_set_depth()
1001 static inline int
1002 ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace)  in ftrace_graph_addr()  argument
1004 	unsigned long addr = trace->func;  in ftrace_graph_addr()
1029 		ftrace_graph_set_depth(task_var, trace->depth);  in ftrace_graph_addr()
1034 		 * want to trace it.  in ftrace_graph_addr()
1048 static inline void
1049 ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace)  in ftrace_graph_addr_finish()  argument
1054 	    trace->depth == ftrace_graph_depth(task_var))  in ftrace_graph_addr_finish()
1058 static inline int ftrace_graph_notrace_addr(unsigned long addr)  in ftrace_graph_notrace_addr()
1081 static inline int ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace)  in ftrace_graph_addr()  argument
1086 static inline int ftrace_graph_notrace_addr(unsigned long addr)  in ftrace_graph_notrace_addr()
1090 static inline void ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace)  in ftrace_graph_addr_finish()  argument
1097 static inline bool
1098 ftrace_graph_ignore_func(struct fgraph_ops *gops, struct ftrace_graph_ent *trace)  in ftrace_graph_ignore_func()  argument
1102 	/* trace it when it is-nested-in or is a function enabled. */  in ftrace_graph_ignore_func()
1104 		 ftrace_graph_addr(task_var, trace)) ||  in ftrace_graph_ignore_func()
1105 		(trace->depth < 0) ||  in ftrace_graph_ignore_func()
1106 		(fgraph_max_depth && trace->depth >= fgraph_max_depth);  in ftrace_graph_ignore_func()
1113 static inline enum print_line_t
1118 static inline void free_fgraph_ops(struct trace_array *tr) { }  in free_fgraph_ops()
1128 #define FTRACE_PID_IGNORE	-1
1129 #define FTRACE_PID_TRACE	-2
1140 static inline int ftrace_trace_task(struct trace_array *tr)  in ftrace_trace_task()
1142 	return this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid) !=  in ftrace_trace_task()
1162 static inline int ftrace_trace_task(struct trace_array *tr)  in ftrace_trace_task()
1166 static inline int ftrace_is_dead(void) { return 0; }  in ftrace_is_dead()
1167 static inline int
1173 static inline int ftrace_allocate_ftrace_ops(struct trace_array *tr)  in ftrace_allocate_ftrace_ops()
1177 static inline void ftrace_free_ftrace_ops(struct trace_array *tr) { }  in ftrace_free_ftrace_ops()
1178 static inline void ftrace_destroy_function_files(struct trace_array *tr) { }  in ftrace_destroy_function_files()
1179 static inline __init void
1181 static inline void ftrace_reset_array_ops(struct trace_array *tr) { }  in ftrace_reset_array_ops()
1182 static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }  in ftrace_init_tracefs()
1183 static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { }  in ftrace_init_tracefs_toplevel()
1184 static inline void ftrace_clear_pids(struct trace_array *tr) { }  in ftrace_clear_pids()
1185 static inline int init_function_trace(void) { return 0; }  in init_function_trace()
1186 static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { }  in ftrace_pid_follow_fork()
1187 /* ftace_func_t type is not defined, use macro instead of static inline */
1247 static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)  in register_ftrace_command()
1249 	return -EINVAL;  in register_ftrace_command()
1251 static inline __init int unregister_ftrace_command(char *cmd_name)  in unregister_ftrace_command()
1253 	return -EINVAL;  in unregister_ftrace_command()
1255 static inline void clear_ftrace_function_probes(struct trace_array *tr)  in clear_ftrace_function_probes()
1270  * struct trace_parser - servers for reading the user input separated by spaces
1271  * @cont: set if the input is not complete - no final space char was found
1283 static inline bool trace_parser_loaded(struct trace_parser *parser)  in trace_parser_loaded()
1285 	return (parser->idx != 0);  in trace_parser_loaded()
1288 static inline bool trace_parser_cont(struct trace_parser *parser)  in trace_parser_cont()
1290 	return parser->cont;  in trace_parser_cont()
1293 static inline void trace_parser_clear(struct trace_parser *parser)  in trace_parser_clear()
1295 	parser->cont = false;  in trace_parser_clear()
1296 	parser->idx = 0;  in trace_parser_clear()
1309 		C(DISPLAY_GRAPH,	"display-graph"),
1323 		C(FUNCTION,		"function-trace"),	\
1324 		C(FUNC_FORK,		"function-fork"),
1344  *       trace.c (this macro guarantees it).
1347 		C(PRINT_PARENT,		"print-parent"),	\
1348 		C(SYM_OFFSET,		"sym-offset"),		\
1349 		C(SYM_ADDR,		"sym-addr"),		\
1359 		C(SYM_USEROBJ,		"sym-userobj"),		\
1360 		C(PRINTK_MSGONLY,	"printk-msg-only"),	\
1361 		C(CONTEXT_INFO,		"context-info"),   /* Print pid/cpu/time */ \
1362 		C(LATENCY_FMT,		"latency-format"),	\
1363 		C(RECORD_CMD,		"record-cmd"),		\
1364 		C(RECORD_TGID,		"record-tgid"),		\
1367 		C(IRQ_INFO,		"irq-info"),		\
1369 		C(EVENT_FORK,		"event-fork"),		\
1371 		C(PAUSE_ON_TRACE,	"pause-on-trace"),	\
1372 		C(HASH_PTR,		"hash-ptr"),	/* Print hashed pointer */ \
1412 static inline int trace_branch_enable(struct trace_array *tr)  in trace_branch_enable()
1414 	if (tr->trace_flags & TRACE_ITER_BRANCH)  in trace_branch_enable()
1418 static inline void trace_branch_disable(void)  in trace_branch_disable()
1424 static inline int trace_branch_enable(struct trace_array *tr)  in trace_branch_enable()
1428 static inline void trace_branch_disable(void)  in trace_branch_disable()
1485 static inline void trace_buffer_unlock_commit(struct trace_array *tr,  in trace_buffer_unlock_commit()
1510 static inline void
1537 static inline bool
1544 	unsigned long eflags = file->flags;  in __event_trigger_test_discard()
1549 	if (likely(!(file->flags & (EVENT_FILE_FL_SOFT_DISABLED |  in __event_trigger_test_discard()
1554 	if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)  in __event_trigger_test_discard()
1557 	if (file->flags & EVENT_FILE_FL_FILTERED &&  in __event_trigger_test_discard()
1558 	    !filter_match_preds(file->filter, entry))  in __event_trigger_test_discard()
1561 	if ((file->flags & EVENT_FILE_FL_PID_FILTER) &&  in __event_trigger_test_discard()
1572  * event_trigger_unlock_commit - handle triggers and finish event commit
1583 static inline void
1592 		trace_buffer_unlock_commit(file->tr, buffer, event, trace_ctx);  in event_trigger_unlock_commit()
1598 #define FILTER_PRED_INVALID	((unsigned short)-1)
1632 static inline bool is_string_field(struct ftrace_event_field *field)  in is_string_field()
1634 	return field->filter_type == FILTER_DYN_STRING ||  in is_string_field()
1635 	       field->filter_type == FILTER_RDYN_STRING ||  in is_string_field()
1636 	       field->filter_type == FILTER_STATIC_STRING ||  in is_string_field()
1637 	       field->filter_type == FILTER_PTR_STRING ||  in is_string_field()
1638 	       field->filter_type == FILTER_COMM;  in is_string_field()
1641 static inline bool is_function_field(struct ftrace_event_field *field)  in is_function_field()
1643 	return field->filter_type == FILTER_TRACE_FN;  in is_function_field()
1682 static inline void *event_file_data(struct file *filp)  in event_file_data()
1684 	return READ_ONCE(file_inode(filp)->i_private);  in event_file_data()
1691  * When the trace_event_file is the filp->i_private pointer,
1702 static inline struct trace_event_file *event_file_file(struct file *filp)  in event_file_file()
1707 	file = READ_ONCE(file_inode(filp)->i_private);  in event_file_file()
1708 	if (!file || file->flags & EVENT_FILE_FL_FREED)  in event_file_file()
1722 static inline int register_trigger_hist_cmd(void) { return 0; }  in register_trigger_hist_cmd()
1723 static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }  in register_trigger_hist_enable_disable_cmds()
1827  * struct event_trigger_ops - callbacks for trace event triggers
1829  * The methods in this structure provide per-event trigger hooks for
1849  *	the trace record, rec.
1853  *	function).  This can be used to perform per-trigger
1854  *	initialization such as incrementing a per-trigger reference
1859  * @free: An optional de-initialization function called for the
1862  *	per-trigger de-initialization such as decrementing a
1863  *	per-trigger reference count and freeing corresponding trigger
1885  * struct event_command - callbacks and data members for event commands
1888  * into the 'trigger' file associated with a trace event.  The
1891  * added to the list of trigger instances associated with that trace
1895  * The data members in this structure provide per-event command data
1904  * @trigger_type: A unique id that identifies the event command
1913  *	commands need to do this if they themselves log to the trace
1926  *	the appropriate trace event.  It makes use of the other
1986  * enum event_command_flags - flags for struct event_command
1992  *	the trigger may itself log data into the trace buffer.  Thus
1995  *	in two - the first part checks the filter using the current
1996  *	trace record; if a command has the @post_trigger flag set, it
2004  *	itself logs to the trace buffer, this flag should be set,
2008  *	access to the trace record in order to perform its function,
2010  *	it (filters make a trigger require access to the trace record
2018 static inline bool event_command_post_trigger(struct event_command *cmd_ops)  in event_command_post_trigger()
2020 	return cmd_ops->flags & EVENT_CMD_FL_POST_TRIGGER;  in event_command_post_trigger()
2023 static inline bool event_command_needs_rec(struct event_command *cmd_ops)  in event_command_needs_rec()
2025 	return cmd_ops->flags & EVENT_CMD_FL_NEEDS_REC;  in event_command_needs_rec()
2071  * into sections to display. But the trace infrastructure wants
2080 #define FTRACE_ENTRY(call, struct_name, id, tstruct, print)	\  argument
2084 #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print)	\  argument
2085 	FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
2087 #define FTRACE_ENTRY_PACKED(call, struct_name, id, tstruct, print) \  argument
2088 	FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
2103 static inline void init_ftrace_syscalls(void) { }  in init_ftrace_syscalls()
2104 static inline const char *get_syscall_name(int syscall)  in get_syscall_name()
2117 static inline void __init trace_event_init(void) { }  in trace_event_init()
2118 static inline void trace_event_eval_update(struct trace_eval_map **map, int len) { }  in trace_event_eval_update()
2127 static inline void tracing_snapshot_instance(struct trace_array *tr) { }  in tracing_snapshot_instance()
2128 static inline int tracing_alloc_snapshot_instance(struct trace_array *tr)  in tracing_alloc_snapshot_instance()
2132 static inline int tracing_arm_snapshot(struct trace_array *tr) { return 0; }  in tracing_arm_snapshot()
2133 static inline void tracing_disarm_snapshot(struct trace_array *tr) { }  in tracing_disarm_snapshot()
2140 static inline void tracer_preempt_on(unsigned long a0, unsigned long a1) { }  in tracer_preempt_on()
2141 static inline void tracer_preempt_off(unsigned long a0, unsigned long a1) { }  in tracer_preempt_off()
2147 static inline void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { }  in tracer_hardirqs_on()
2148 static inline void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { }  in tracer_hardirqs_off()
2156 static __always_inline void trace_iterator_reset(struct trace_iterator *iter)  in trace_iterator_reset()
2159 	iter->pos = -1;  in trace_iterator_reset()
2163 static inline bool __is_good_name(const char *name, bool hash_ok)  in __is_good_name()
2165 	if (!isalpha(*name) && *name != '_' && (!hash_ok || *name != '-'))  in __is_good_name()
2169 		    (!hash_ok || *name != '-'))  in __is_good_name()
2176 static inline bool is_good_name(const char *name)  in is_good_name()
2182 static inline bool is_good_system_name(const char *name)  in is_good_system_name()
2188 static inline void sanitize_event_name(char *name)  in sanitize_event_name()
2216 static inline int rv_init_interface(void)  in rv_init_interface()