1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
16 #include "trace.h"
45 	/* Display overruns? (for self-debug purpose) */
46 	{ TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
48 	{ TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
50 	{ TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
52 	{ TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
54 	{ TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
56 	{ TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
58 	{ TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) },
60 	{ TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) },
63 	{ TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) },
65 	{ TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) },
68 	{ TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) },
72 	{ TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) },
104 				struct ftrace_graph_ent *trace,  in __trace_graph_entry()  argument
109 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in __trace_graph_entry()  local
112 	event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,  in __trace_graph_entry()
117 	entry->graph_ent			= *trace;  in __trace_graph_entry()
118 	if (!call_filter_check_discard(call, entry, buffer, event))  in __trace_graph_entry()
119 		trace_buffer_unlock_commit_nostack(buffer, event);  in __trace_graph_entry()
132 int trace_graph_entry(struct ftrace_graph_ent *trace)  in trace_graph_entry()  argument
146 	 * Do not trace a function if it's filtered by set_graph_notrace.  in trace_graph_entry()
152 	if (ftrace_graph_notrace_addr(trace->func)) {  in trace_graph_entry()
164 	if (ftrace_graph_ignore_func(trace))  in trace_graph_entry()
172 	 * events to the ring buffer.  in trace_graph_entry()
179 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in trace_graph_entry()
180 	disabled = atomic_inc_return(&data->disabled);  in trace_graph_entry()
183 		ret = __trace_graph_entry(tr, trace, trace_ctx);  in trace_graph_entry()
188 	atomic_dec(&data->disabled);  in trace_graph_entry()
223 				struct ftrace_graph_ret *trace,  in __trace_graph_return()  argument
228 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in __trace_graph_return()  local
231 	event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,  in __trace_graph_return()
236 	entry->ret				= *trace;  in __trace_graph_return()
237 	if (!call_filter_check_discard(call, entry, buffer, event))  in __trace_graph_return()
238 		trace_buffer_unlock_commit_nostack(buffer, event);  in __trace_graph_return()
241 void trace_graph_return(struct ftrace_graph_ret *trace)  in trace_graph_return()  argument
250 	ftrace_graph_addr_finish(trace);  in trace_graph_return()
259 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in trace_graph_return()
260 	disabled = atomic_inc_return(&data->disabled);  in trace_graph_return()
263 		__trace_graph_return(tr, trace, trace_ctx);  in trace_graph_return()
265 	atomic_dec(&data->disabled);  in trace_graph_return()
278 static void trace_graph_thresh_return(struct ftrace_graph_ret *trace)  in trace_graph_thresh_return()  argument
280 	ftrace_graph_addr_finish(trace);  in trace_graph_thresh_return()
288 	    (trace->rettime - trace->calltime < tracing_thresh))  in trace_graph_thresh_return()
291 		trace_graph_return(trace);  in trace_graph_thresh_return()
340 	 * Start with a space character - to make it stand out  in print_graph_cpu()
341 	 * to the right a bit when trace output is pasted into  in print_graph_cpu()
362 	/* 1 stands for the "-" character */  in print_graph_proc()
366 		spaces = TRACE_GRAPH_PROCINFO_LENGTH - len;  in print_graph_proc()
372 	trace_seq_printf(s, "%s-%s", comm, pid_str);  in print_graph_proc()
375 	for (i = 0; i < spaces - (spaces / 2); i++)  in print_graph_proc()
387 /* If the pid changed since the last trace, output this event */
397 	last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);  in verif_pid()
405 	if (prev_pid == -1)  in verif_pid()
408  * Context-switch trace line:  in verif_pid()
410  ------------------------------------------  in verif_pid()
411  | 1)  migration/0--1  =>  sshd-1755  in verif_pid()
412  ------------------------------------------  in verif_pid()
415 	trace_seq_puts(s, " ------------------------------------------\n");  in verif_pid()
420 	trace_seq_puts(s, "\n ------------------------------------------\n\n");  in verif_pid()
427 	struct fgraph_data *data = iter->private;  in get_return_for_leaf()
433 	 * If the previous output failed to write to the seq buffer,  in get_return_for_leaf()
436 	if (data && data->failed) {  in get_return_for_leaf()
437 		curr = &data->ent;  in get_return_for_leaf()
438 		next = &data->ret;  in get_return_for_leaf()
441 		ring_iter = trace_buffer_iter(iter, iter->cpu);  in get_return_for_leaf()
451 			ring_buffer_consume(iter->array_buffer->buffer, iter->cpu,  in get_return_for_leaf()
453 			event = ring_buffer_peek(iter->array_buffer->buffer, iter->cpu,  in get_return_for_leaf()
467 			data->ent = *curr;  in get_return_for_leaf()
473 			if (next->ent.type == TRACE_GRAPH_RET)  in get_return_for_leaf()
474 				data->ret = *next;  in get_return_for_leaf()
476 				data->ret.ent.type = next->ent.type;  in get_return_for_leaf()
480 	if (next->ent.type != TRACE_GRAPH_RET)  in get_return_for_leaf()
483 	if (curr->ent.pid != next->ent.pid ||  in get_return_for_leaf()
484 			curr->graph_ent.func != next->ret.func)  in get_return_for_leaf()
510 	usecs = iter->ts - iter->array_buffer->time_start;  in print_graph_rel_time()
520 	struct trace_array *tr = iter->tr;  in print_graph_irq()
521 	struct trace_seq *s = &iter->seq;  in print_graph_irq()
522 	struct trace_entry *ent = iter->ent;  in print_graph_irq()
528 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {  in print_graph_irq()
531 			print_graph_abs_time(iter->ts, s);  in print_graph_irq()
548 		if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)  in print_graph_irq()
583 		size_t slen = min_t(size_t, sizeof(nsecs_str), 8UL - len);  in trace_print_graph_duration()
602 	    !(tr->trace_flags & TRACE_ITER_CONTEXT_INFO))  in print_graph_duration()
643 		/* sign extension */  in print_graph_retval()
685 	struct fgraph_data *data = iter->private;  in print_graph_entry_leaf()
686 	struct trace_array *tr = iter->tr;  in print_graph_entry_leaf()
690 	int cpu = iter->cpu;  in print_graph_entry_leaf()
693 	graph_ret = &ret_entry->ret;  in print_graph_entry_leaf()
694 	call = &entry->graph_ent;  in print_graph_entry_leaf()
695 	duration = graph_ret->rettime - graph_ret->calltime;  in print_graph_entry_leaf()
700 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);  in print_graph_entry_leaf()
707 		cpu_data->depth = call->depth - 1;  in print_graph_entry_leaf()
710 		if (call->depth < FTRACE_RETFUNC_DEPTH &&  in print_graph_entry_leaf()
711 		    !WARN_ON_ONCE(call->depth < 0))  in print_graph_entry_leaf()
712 			cpu_data->enter_funcs[call->depth] = 0;  in print_graph_entry_leaf()
719 	for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++)  in print_graph_entry_leaf()
723 	 * Write out the function return value if the option function-retval is  in print_graph_entry_leaf()
727 		print_graph_retval(s, graph_ret->retval, true, (void *)call->func,  in print_graph_entry_leaf()
730 		trace_seq_printf(s, "%ps();\n", (void *)call->func);  in print_graph_entry_leaf()
732 	print_graph_irq(iter, graph_ret->func, TRACE_GRAPH_RET,  in print_graph_entry_leaf()
733 			cpu, iter->ent->pid, flags);  in print_graph_entry_leaf()
743 	struct ftrace_graph_ent *call = &entry->graph_ent;  in print_graph_entry_nested()
744 	struct fgraph_data *data = iter->private;  in print_graph_entry_nested()
745 	struct trace_array *tr = iter->tr;  in print_graph_entry_nested()
750 		int cpu = iter->cpu;  in print_graph_entry_nested()
752 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);  in print_graph_entry_nested()
753 		cpu_data->depth = call->depth;  in print_graph_entry_nested()
756 		if (call->depth < FTRACE_RETFUNC_DEPTH &&  in print_graph_entry_nested()
757 		    !WARN_ON_ONCE(call->depth < 0))  in print_graph_entry_nested()
758 			cpu_data->enter_funcs[call->depth] = call->func;  in print_graph_entry_nested()
765 	for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++)  in print_graph_entry_nested()
768 	trace_seq_printf(s, "%ps() {\n", (void *)call->func);  in print_graph_entry_nested()
784 	struct fgraph_data *data = iter->private;  in print_graph_prologue()
785 	struct trace_entry *ent = iter->ent;  in print_graph_prologue()
786 	struct trace_array *tr = iter->tr;  in print_graph_prologue()
787 	int cpu = iter->cpu;  in print_graph_prologue()
790 	verif_pid(s, ent->pid, cpu, data);  in print_graph_prologue()
794 		print_graph_irq(iter, addr, type, cpu, ent->pid, flags);  in print_graph_prologue()
796 	if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO))  in print_graph_prologue()
801 		print_graph_abs_time(iter->ts, s);  in print_graph_prologue()
813 		print_graph_proc(s, ent->pid);  in print_graph_prologue()
818 	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)  in print_graph_prologue()
828  *  - we are inside irq code
829  *  - we just entered irq code
832  *  - funcgraph-interrupts option is set
833  *  - we are not inside irq code
839 	int cpu = iter->cpu;  in check_irq_entry()
841 	struct fgraph_data *data = iter->private;  in check_irq_entry()
852 	depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq);  in check_irq_entry()
875  *  - we are inside irq code
876  *  - we just left irq code
879  *  - funcgraph-interrupts option is set
880  *  - we are not inside irq code
885 	int cpu = iter->cpu;  in check_irq_return()
887 	struct fgraph_data *data = iter->private;  in check_irq_return()
898 	depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq);  in check_irq_return()
903 	if (*depth_irq == -1)  in check_irq_return()
908 	 * Let's not trace it and clear the entry depth, since  in check_irq_return()
916 		*depth_irq = -1;  in check_irq_return()
930 	struct fgraph_data *data = iter->private;  in print_graph_entry()
931 	struct ftrace_graph_ent *call = &field->graph_ent;  in print_graph_entry()
934 	int cpu = iter->cpu;  in print_graph_entry()
936 	if (check_irq_entry(iter, flags, call->func, call->depth))  in print_graph_entry()
939 	print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags);  in print_graph_entry()
952 		if (s->full) {  in print_graph_entry()
953 			data->failed = 1;  in print_graph_entry()
954 			data->cpu = cpu;  in print_graph_entry()
956 			data->failed = 0;  in print_graph_entry()
963 print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,  in print_graph_return()  argument
967 	unsigned long long duration = trace->rettime - trace->calltime;  in print_graph_return()
968 	struct fgraph_data *data = iter->private;  in print_graph_return()
969 	struct trace_array *tr = iter->tr;  in print_graph_return()
970 	pid_t pid = ent->pid;  in print_graph_return()
971 	int cpu = iter->cpu;  in print_graph_return()
975 	if (check_irq_return(iter, flags, trace->depth))  in print_graph_return()
980 		int cpu = iter->cpu;  in print_graph_return()
982 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);  in print_graph_return()
989 		cpu_data->depth = trace->depth - 1;  in print_graph_return()
991 		if (trace->depth < FTRACE_RETFUNC_DEPTH &&  in print_graph_return()
992 		    !WARN_ON_ONCE(trace->depth < 0)) {  in print_graph_return()
993 			if (cpu_data->enter_funcs[trace->depth] != trace->func)  in print_graph_return()
995 			cpu_data->enter_funcs[trace->depth] = 0;  in print_graph_return()
1005 	for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++)  in print_graph_return()
1010 	 * function-retval option is enabled.  in print_graph_return()
1013 		print_graph_retval(s, trace->retval, false, (void *)trace->func,  in print_graph_return()
1021 		 * that if the funcgraph-tail option is enabled.  in print_graph_return()
1026 			trace_seq_printf(s, "} /* %ps */\n", (void *)trace->func);  in print_graph_return()
1032 				 trace->overrun);  in print_graph_return()
1034 	print_graph_irq(iter, trace->func, TRACE_GRAPH_RET,  in print_graph_return()
1044 	struct trace_array *tr = iter->tr;  in print_graph_comment()
1045 	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);  in print_graph_comment()
1046 	struct fgraph_data *data = iter->private;  in print_graph_comment()
1053 		depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth;  in print_graph_comment()
1068 	switch (iter->ent->type) {  in print_graph_comment()
1085 		event = ftrace_find_event(ent->type);  in print_graph_comment()
1089 		ret = event->funcs->trace(iter, sym_flags, event);  in print_graph_comment()
1098 	if (s->buffer[s->seq.len - 1] == '\n') {  in print_graph_comment()
1099 		s->buffer[s->seq.len - 1] = '\0';  in print_graph_comment()
1100 		s->seq.len--;  in print_graph_comment()
1113 	struct fgraph_data *data = iter->private;  in print_graph_function_flags()
1114 	struct trace_entry *entry = iter->ent;  in print_graph_function_flags()
1115 	struct trace_seq *s = &iter->seq;  in print_graph_function_flags()
1116 	int cpu = iter->cpu;  in print_graph_function_flags()
1119 	if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) {  in print_graph_function_flags()
1120 		per_cpu_ptr(data->cpu_data, cpu)->ignore = 0;  in print_graph_function_flags()
1128 	if (data && data->failed) {  in print_graph_function_flags()
1129 		field = &data->ent;  in print_graph_function_flags()
1130 		iter->cpu = data->cpu;  in print_graph_function_flags()
1132 		if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) {  in print_graph_function_flags()
1133 			per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1;  in print_graph_function_flags()
1136 		iter->cpu = cpu;  in print_graph_function_flags()
1140 	switch (entry->type) {  in print_graph_function_flags()
1156 		return print_graph_return(&field->ret, s, entry, iter, flags);  in print_graph_function_flags()
1160 		/* dont trace stack and functions as comments */  in print_graph_function_flags()
1199 	seq_printf(s, "#%.*s  _-----=> irqs-off        \n", size, spaces);  in print_lat_header()
1200 	seq_printf(s, "#%.*s / _----=> need-resched    \n", size, spaces);  in print_lat_header()
1201 	seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces);  in print_lat_header()
1202 	seq_printf(s, "#%.*s|| / _--=> preempt-depth   \n", size, spaces);  in print_lat_header()
1209 	int lat = tr->trace_flags & TRACE_ITER_LATENCY_FMT;  in __print_graph_headers_flags()
1254 	struct trace_iterator *iter = s->private;  in print_graph_headers_flags()
1255 	struct trace_array *tr = iter->tr;  in print_graph_headers_flags()
1257 	if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO))  in print_graph_headers_flags()
1260 	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) {  in print_graph_headers_flags()
1273 	/* pid and depth on the last trace processed */  in graph_trace_open()
1278 	iter->private = NULL;  in graph_trace_open()
1287 	data->cpu_data = alloc_percpu_gfp(struct fgraph_cpu_data, gfpflags);  in graph_trace_open()
1288 	if (!data->cpu_data)  in graph_trace_open()
1292 		pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);  in graph_trace_open()
1293 		int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);  in graph_trace_open()
1294 		int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore);  in graph_trace_open()
1295 		int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq);  in graph_trace_open()
1297 		*pid = -1;  in graph_trace_open()
1300 		*depth_irq = -1;  in graph_trace_open()
1303 	iter->private = data;  in graph_trace_open()
1315 	struct fgraph_data *data = iter->private;  in graph_trace_close()
1318 		free_percpu(data->cpu_data);  in graph_trace_close()
1339 	.trace		= print_graph_function_event,
1425 	max_bytes_for_cpu = snprintf(NULL, 0, "%u", nr_cpu_ids - 1);  in init_graph_trace()
1428 		pr_warn("Warning: could not register graph trace events\n");  in init_graph_trace()
1433 		pr_warn("Warning: could not register graph trace events\n");  in init_graph_trace()