1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * unlikely profiler
4  *
5  * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6  */
7 #include <linux/kallsyms.h>
8 #include <linux/seq_file.h>
9 #include <linux/spinlock.h>
10 #include <linux/irqflags.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
15 #include <linux/fs.h>
16 #include <asm/local.h>
17 
18 #include "trace.h"
19 #include "trace_stat.h"
20 #include "trace_output.h"
21 
22 #ifdef CONFIG_BRANCH_TRACER
23 
24 static struct tracer branch_trace;
25 static int branch_tracing_enabled __read_mostly;
26 static DEFINE_MUTEX(branch_tracing_mutex);
27 
28 static struct trace_array *branch_tracer;
29 
30 static void
31 probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
32 {
33 	struct trace_array *tr = branch_tracer;
34 	struct trace_buffer *buffer;
35 	struct ring_buffer_event *event;
36 	struct trace_branch *entry;
37 	unsigned long flags;
38 	unsigned int trace_ctx;
39 	const char *p;
40 
41 	if (current->trace_recursion & TRACE_BRANCH_BIT)
42 		return;
43 
44 	/*
45 	 * I would love to save just the ftrace_likely_data pointer, but
46 	 * this code can also be used by modules. Ugly things can happen
47 	 * if the module is unloaded, and then we go and read the
48 	 * pointer.  This is slower, but much safer.
49 	 */
50 
51 	if (unlikely(!tr))
52 		return;
53 
54 	raw_local_irq_save(flags);
55 	current->trace_recursion |= TRACE_BRANCH_BIT;
56 	if (!tracer_tracing_is_on_cpu(tr, raw_smp_processor_id()))
57 		goto out;
58 
59 	trace_ctx = tracing_gen_ctx_flags(flags);
60 	buffer = tr->array_buffer.buffer;
61 	event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
62 					  sizeof(*entry), trace_ctx);
63 	if (!event)
64 		goto out;
65 
66 	entry	= ring_buffer_event_data(event);
67 
68 	/* Strip off the path, only save the file */
69 	p = f->data.file + strlen(f->data.file);
70 	while (p >= f->data.file && *p != '/')
71 		p--;
72 	p++;
73 
74 	strscpy(entry->func, f->data.func);
75 	strscpy(entry->file, p);
76 	entry->constant = f->constant;
77 	entry->line = f->data.line;
78 	entry->correct = val == expect;
79 
80 	trace_buffer_unlock_commit_nostack(buffer, event);
81 
82  out:
83 	current->trace_recursion &= ~TRACE_BRANCH_BIT;
84 	raw_local_irq_restore(flags);
85 }
86 
87 static inline
88 void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
89 {
90 	if (!branch_tracing_enabled)
91 		return;
92 
93 	probe_likely_condition(f, val, expect);
94 }
95 
96 int enable_branch_tracing(struct trace_array *tr)
97 {
98 	mutex_lock(&branch_tracing_mutex);
99 	branch_tracer = tr;
100 	/*
101 	 * Must be seen before enabling. The reader is a condition
102 	 * where we do not need a matching rmb()
103 	 */
104 	smp_wmb();
105 	branch_tracing_enabled++;
106 	mutex_unlock(&branch_tracing_mutex);
107 
108 	return 0;
109 }
110 
111 void disable_branch_tracing(void)
112 {
113 	mutex_lock(&branch_tracing_mutex);
114 
115 	if (!branch_tracing_enabled)
116 		goto out_unlock;
117 
118 	branch_tracing_enabled--;
119 
120  out_unlock:
121 	mutex_unlock(&branch_tracing_mutex);
122 }
123 
124 static int branch_trace_init(struct trace_array *tr)
125 {
126 	return enable_branch_tracing(tr);
127 }
128 
129 static void branch_trace_reset(struct trace_array *tr)
130 {
131 	disable_branch_tracing();
132 }
133 
134 static enum print_line_t trace_branch_print(struct trace_iterator *iter,
135 					    int flags, struct trace_event *event)
136 {
137 	struct trace_branch *field;
138 
139 	trace_assign_type(field, iter->ent);
140 
141 	trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
142 			 field->correct ? "  ok  " : " MISS ",
143 			 field->func,
144 			 field->file,
145 			 field->line);
146 
147 	return trace_handle_return(&iter->seq);
148 }
149 
150 static void branch_print_header(struct seq_file *s)
151 {
152 	seq_puts(s, "#           TASK-PID    CPU#    TIMESTAMP  CORRECT"
153 		    "  FUNC:FILE:LINE\n"
154 		    "#              | |       |          |         |   "
155 		    "    |\n");
156 }
157 
158 static struct trace_event_functions trace_branch_funcs = {
159 	.trace		= trace_branch_print,
160 };
161 
162 static struct trace_event trace_branch_event = {
163 	.type		= TRACE_BRANCH,
164 	.funcs		= &trace_branch_funcs,
165 };
166 
167 static struct tracer branch_trace __read_mostly =
168 {
169 	.name		= "branch",
170 	.init		= branch_trace_init,
171 	.reset		= branch_trace_reset,
172 #ifdef CONFIG_FTRACE_SELFTEST
173 	.selftest	= trace_selftest_startup_branch,
174 #endif /* CONFIG_FTRACE_SELFTEST */
175 	.print_header	= branch_print_header,
176 };
177 
178 __init static int init_branch_tracer(void)
179 {
180 	int ret;
181 
182 	ret = register_trace_event(&trace_branch_event);
183 	if (!ret) {
184 		printk(KERN_WARNING "Warning: could not register "
185 				    "branch events\n");
186 		return 1;
187 	}
188 	return register_tracer(&branch_trace);
189 }
190 core_initcall(init_branch_tracer);
191 
192 #else
193 static inline
194 void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
195 {
196 }
197 #endif /* CONFIG_BRANCH_TRACER */
198 
199 void ftrace_likely_update(struct ftrace_likely_data *f, int val,
200 			  int expect, int is_constant)
201 {
202 	unsigned long flags = user_access_save();
203 
204 	/* A constant is always correct */
205 	if (is_constant) {
206 		f->constant++;
207 		val = expect;
208 	}
209 	/*
210 	 * I would love to have a trace point here instead, but the
211 	 * trace point code is so inundated with unlikely and likely
212 	 * conditions that the recursive nightmare that exists is too
213 	 * much to try to get working. At least for now.
214 	 */
215 	trace_likely_condition(f, val, expect);
216 
217 	/* FIXME: Make this atomic! */
218 	if (val == expect)
219 		f->data.correct++;
220 	else
221 		f->data.incorrect++;
222 
223 	user_access_restore(flags);
224 }
225 EXPORT_SYMBOL(ftrace_likely_update);
226 
227 extern unsigned long __start_annotated_branch_profile[];
228 extern unsigned long __stop_annotated_branch_profile[];
229 
230 static int annotated_branch_stat_headers(struct seq_file *m)
231 {
232 	seq_puts(m, " correct incorrect  % "
233 		    "       Function                "
234 		    "  File              Line\n"
235 		    " ------- ---------  - "
236 		    "       --------                "
237 		    "  ----              ----\n");
238 	return 0;
239 }
240 
241 static inline long get_incorrect_percent(const struct ftrace_branch_data *p)
242 {
243 	long percent;
244 
245 	if (p->correct) {
246 		percent = p->incorrect * 100;
247 		percent /= p->correct + p->incorrect;
248 	} else
249 		percent = p->incorrect ? 100 : -1;
250 
251 	return percent;
252 }
253 
254 static const char *branch_stat_process_file(struct ftrace_branch_data *p)
255 {
256 	const char *f;
257 
258 	/* Only print the file, not the path */
259 	f = p->file + strlen(p->file);
260 	while (f >= p->file && *f != '/')
261 		f--;
262 	return ++f;
263 }
264 
265 static void branch_stat_show(struct seq_file *m,
266 			     struct ftrace_branch_data *p, const char *f)
267 {
268 	long percent;
269 
270 	/*
271 	 * The miss is overlayed on correct, and hit on incorrect.
272 	 */
273 	percent = get_incorrect_percent(p);
274 
275 	if (percent < 0)
276 		seq_puts(m, "  X ");
277 	else
278 		seq_printf(m, "%3ld ", percent);
279 
280 	seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
281 }
282 
283 static int branch_stat_show_normal(struct seq_file *m,
284 				   struct ftrace_branch_data *p, const char *f)
285 {
286 	seq_printf(m, "%8lu %8lu ",  p->correct, p->incorrect);
287 	branch_stat_show(m, p, f);
288 	return 0;
289 }
290 
291 static int annotate_branch_stat_show(struct seq_file *m, void *v)
292 {
293 	struct ftrace_likely_data *p = v;
294 	const char *f;
295 	int l;
296 
297 	f = branch_stat_process_file(&p->data);
298 
299 	if (!p->constant)
300 		return branch_stat_show_normal(m, &p->data, f);
301 
302 	l = snprintf(NULL, 0, "/%lu", p->constant);
303 	l = l > 8 ? 0 : 8 - l;
304 
305 	seq_printf(m, "%8lu/%lu %*lu ",
306 		   p->data.correct, p->constant, l, p->data.incorrect);
307 	branch_stat_show(m, &p->data, f);
308 	return 0;
309 }
310 
311 static void *annotated_branch_stat_start(struct tracer_stat *trace)
312 {
313 	return __start_annotated_branch_profile;
314 }
315 
316 static void *
317 annotated_branch_stat_next(void *v, int idx)
318 {
319 	struct ftrace_likely_data *p = v;
320 
321 	++p;
322 
323 	if ((void *)p >= (void *)__stop_annotated_branch_profile)
324 		return NULL;
325 
326 	return p;
327 }
328 
329 static int annotated_branch_stat_cmp(const void *p1, const void *p2)
330 {
331 	const struct ftrace_branch_data *a = p1;
332 	const struct ftrace_branch_data *b = p2;
333 
334 	long percent_a, percent_b;
335 
336 	percent_a = get_incorrect_percent(a);
337 	percent_b = get_incorrect_percent(b);
338 
339 	if (percent_a < percent_b)
340 		return -1;
341 	if (percent_a > percent_b)
342 		return 1;
343 
344 	if (a->incorrect < b->incorrect)
345 		return -1;
346 	if (a->incorrect > b->incorrect)
347 		return 1;
348 
349 	/*
350 	 * Since the above shows worse (incorrect) cases
351 	 * first, we continue that by showing best (correct)
352 	 * cases last.
353 	 */
354 	if (a->correct > b->correct)
355 		return -1;
356 	if (a->correct < b->correct)
357 		return 1;
358 
359 	return 0;
360 }
361 
362 static struct tracer_stat annotated_branch_stats = {
363 	.name = "branch_annotated",
364 	.stat_start = annotated_branch_stat_start,
365 	.stat_next = annotated_branch_stat_next,
366 	.stat_cmp = annotated_branch_stat_cmp,
367 	.stat_headers = annotated_branch_stat_headers,
368 	.stat_show = annotate_branch_stat_show
369 };
370 
371 __init static int init_annotated_branch_stats(void)
372 {
373 	int ret;
374 
375 	ret = register_stat_tracer(&annotated_branch_stats);
376 	if (!ret) {
377 		printk(KERN_WARNING "Warning: could not register "
378 				    "annotated branches stats\n");
379 		return 1;
380 	}
381 	return 0;
382 }
383 fs_initcall(init_annotated_branch_stats);
384 
385 #ifdef CONFIG_PROFILE_ALL_BRANCHES
386 
387 extern unsigned long __start_branch_profile[];
388 extern unsigned long __stop_branch_profile[];
389 
390 static int all_branch_stat_headers(struct seq_file *m)
391 {
392 	seq_puts(m, "   miss      hit    % "
393 		    "       Function                "
394 		    "  File              Line\n"
395 		    " ------- ---------  - "
396 		    "       --------                "
397 		    "  ----              ----\n");
398 	return 0;
399 }
400 
401 static void *all_branch_stat_start(struct tracer_stat *trace)
402 {
403 	return __start_branch_profile;
404 }
405 
406 static void *
407 all_branch_stat_next(void *v, int idx)
408 {
409 	struct ftrace_branch_data *p = v;
410 
411 	++p;
412 
413 	if ((void *)p >= (void *)__stop_branch_profile)
414 		return NULL;
415 
416 	return p;
417 }
418 
419 static int all_branch_stat_show(struct seq_file *m, void *v)
420 {
421 	struct ftrace_branch_data *p = v;
422 	const char *f;
423 
424 	f = branch_stat_process_file(p);
425 	return branch_stat_show_normal(m, p, f);
426 }
427 
428 static struct tracer_stat all_branch_stats = {
429 	.name = "branch_all",
430 	.stat_start = all_branch_stat_start,
431 	.stat_next = all_branch_stat_next,
432 	.stat_headers = all_branch_stat_headers,
433 	.stat_show = all_branch_stat_show
434 };
435 
436 __init static int all_annotated_branch_stats(void)
437 {
438 	int ret;
439 
440 	ret = register_stat_tracer(&all_branch_stats);
441 	if (!ret) {
442 		printk(KERN_WARNING "Warning: could not register "
443 				    "all branches stats\n");
444 		return 1;
445 	}
446 	return 0;
447 }
448 fs_initcall(all_annotated_branch_stats);
449 #endif /* CONFIG_PROFILE_ALL_BRANCHES */
450