1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Based on code from the latency_tracer, that is: 9 * 10 * Copyright (C) 2004-2006 Ingo Molnar 11 * Copyright (C) 2004 Nadia Yvette Chambers 12 */ 13 #include <linux/ring_buffer.h> 14 #include <linux/debugfs.h> 15 #include <linux/uaccess.h> 16 #include <linux/ftrace.h> 17 #include <linux/slab.h> 18 #include <linux/fs.h> 19 20 #include "trace.h" 21 22 static void tracing_start_function_trace(struct trace_array *tr); 23 static void tracing_stop_function_trace(struct trace_array *tr); 24 static void 25 function_trace_call(unsigned long ip, unsigned long parent_ip, 26 struct ftrace_ops *op, struct ftrace_regs *fregs); 27 static void 28 function_args_trace_call(unsigned long ip, unsigned long parent_ip, 29 struct ftrace_ops *op, struct ftrace_regs *fregs); 30 static void 31 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 32 struct ftrace_ops *op, struct ftrace_regs *fregs); 33 static void 34 function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 35 struct ftrace_ops *op, struct ftrace_regs *fregs); 36 static void 37 function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 38 struct ftrace_ops *op, 39 struct ftrace_regs *fregs); 40 static struct tracer_flags func_flags; 41 42 /* Our option */ 43 enum { 44 45 TRACE_FUNC_NO_OPTS = 0x0, /* No flags set. */ 46 TRACE_FUNC_OPT_STACK = 0x1, 47 TRACE_FUNC_OPT_NO_REPEATS = 0x2, 48 TRACE_FUNC_OPT_ARGS = 0x4, 49 50 /* Update this to next highest bit. */ 51 TRACE_FUNC_OPT_HIGHEST_BIT = 0x8 52 }; 53 54 #define TRACE_FUNC_OPT_MASK (TRACE_FUNC_OPT_HIGHEST_BIT - 1) 55 56 int ftrace_allocate_ftrace_ops(struct trace_array *tr) 57 { 58 struct ftrace_ops *ops; 59 60 /* The top level array uses the "global_ops" */ 61 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 62 return 0; 63 64 ops = kzalloc(sizeof(*ops), GFP_KERNEL); 65 if (!ops) 66 return -ENOMEM; 67 68 /* Currently only the non stack version is supported */ 69 ops->func = function_trace_call; 70 ops->flags = FTRACE_OPS_FL_PID; 71 72 tr->ops = ops; 73 ops->private = tr; 74 75 return 0; 76 } 77 78 void ftrace_free_ftrace_ops(struct trace_array *tr) 79 { 80 kfree(tr->ops); 81 tr->ops = NULL; 82 } 83 84 int ftrace_create_function_files(struct trace_array *tr, 85 struct dentry *parent) 86 { 87 int ret; 88 /* 89 * The top level array uses the "global_ops", and the files are 90 * created on boot up. 91 */ 92 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 93 return 0; 94 95 if (!tr->ops) 96 return -EINVAL; 97 98 ret = allocate_fgraph_ops(tr, tr->ops); 99 if (ret) { 100 kfree(tr->ops); 101 return ret; 102 } 103 104 ftrace_create_filter_files(tr->ops, parent); 105 106 return 0; 107 } 108 109 void ftrace_destroy_function_files(struct trace_array *tr) 110 { 111 ftrace_destroy_filter_files(tr->ops); 112 ftrace_free_ftrace_ops(tr); 113 free_fgraph_ops(tr); 114 } 115 116 static ftrace_func_t select_trace_function(u32 flags_val) 117 { 118 switch (flags_val & TRACE_FUNC_OPT_MASK) { 119 case TRACE_FUNC_NO_OPTS: 120 return function_trace_call; 121 case TRACE_FUNC_OPT_ARGS: 122 return function_args_trace_call; 123 case TRACE_FUNC_OPT_STACK: 124 return function_stack_trace_call; 125 case TRACE_FUNC_OPT_NO_REPEATS: 126 return function_no_repeats_trace_call; 127 case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS: 128 return function_stack_no_repeats_trace_call; 129 default: 130 return NULL; 131 } 132 } 133 134 static bool handle_func_repeats(struct trace_array *tr, u32 flags_val) 135 { 136 if (!tr->last_func_repeats && 137 (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) { 138 tr->last_func_repeats = alloc_percpu(struct trace_func_repeats); 139 if (!tr->last_func_repeats) 140 return false; 141 } 142 143 return true; 144 } 145 146 static int function_trace_init(struct trace_array *tr) 147 { 148 ftrace_func_t func; 149 /* 150 * Instance trace_arrays get their ops allocated 151 * at instance creation. Unless it failed 152 * the allocation. 153 */ 154 if (!tr->ops) 155 return -ENOMEM; 156 157 func = select_trace_function(func_flags.val); 158 if (!func) 159 return -EINVAL; 160 161 if (!handle_func_repeats(tr, func_flags.val)) 162 return -ENOMEM; 163 164 ftrace_init_array_ops(tr, func); 165 166 tr->array_buffer.cpu = raw_smp_processor_id(); 167 168 tracing_start_cmdline_record(); 169 tracing_start_function_trace(tr); 170 return 0; 171 } 172 173 static void function_trace_reset(struct trace_array *tr) 174 { 175 tracing_stop_function_trace(tr); 176 tracing_stop_cmdline_record(); 177 ftrace_reset_array_ops(tr); 178 } 179 180 static void function_trace_start(struct trace_array *tr) 181 { 182 tracing_reset_online_cpus(&tr->array_buffer); 183 } 184 185 /* fregs are guaranteed not to be NULL if HAVE_DYNAMIC_FTRACE_WITH_ARGS is set */ 186 #if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) 187 static __always_inline unsigned long 188 function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs) 189 { 190 unsigned long true_parent_ip; 191 int idx = 0; 192 193 true_parent_ip = parent_ip; 194 if (unlikely(parent_ip == (unsigned long)&return_to_handler) && fregs) 195 true_parent_ip = ftrace_graph_ret_addr(current, &idx, parent_ip, 196 (unsigned long *)ftrace_regs_get_stack_pointer(fregs)); 197 return true_parent_ip; 198 } 199 #else 200 static __always_inline unsigned long 201 function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs) 202 { 203 return parent_ip; 204 } 205 #endif 206 207 static void 208 function_trace_call(unsigned long ip, unsigned long parent_ip, 209 struct ftrace_ops *op, struct ftrace_regs *fregs) 210 { 211 struct trace_array *tr = op->private; 212 unsigned int trace_ctx; 213 int bit; 214 215 if (unlikely(!tr->function_enabled)) 216 return; 217 218 bit = ftrace_test_recursion_trylock(ip, parent_ip); 219 if (bit < 0) 220 return; 221 222 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 223 224 trace_ctx = tracing_gen_ctx_dec(); 225 226 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 227 228 ftrace_test_recursion_unlock(bit); 229 } 230 231 static void 232 function_args_trace_call(unsigned long ip, unsigned long parent_ip, 233 struct ftrace_ops *op, struct ftrace_regs *fregs) 234 { 235 struct trace_array *tr = op->private; 236 unsigned int trace_ctx; 237 int bit; 238 239 if (unlikely(!tr->function_enabled)) 240 return; 241 242 bit = ftrace_test_recursion_trylock(ip, parent_ip); 243 if (bit < 0) 244 return; 245 246 trace_ctx = tracing_gen_ctx(); 247 248 trace_function(tr, ip, parent_ip, trace_ctx, fregs); 249 250 ftrace_test_recursion_unlock(bit); 251 } 252 253 #ifdef CONFIG_UNWINDER_ORC 254 /* 255 * Skip 2: 256 * 257 * function_stack_trace_call() 258 * ftrace_call() 259 */ 260 #define STACK_SKIP 2 261 #else 262 /* 263 * Skip 3: 264 * __trace_stack() 265 * function_stack_trace_call() 266 * ftrace_call() 267 */ 268 #define STACK_SKIP 3 269 #endif 270 271 static void 272 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 273 struct ftrace_ops *op, struct ftrace_regs *fregs) 274 { 275 struct trace_array *tr = op->private; 276 struct trace_array_cpu *data; 277 unsigned long flags; 278 long disabled; 279 int cpu; 280 unsigned int trace_ctx; 281 int skip = STACK_SKIP; 282 283 if (unlikely(!tr->function_enabled)) 284 return; 285 286 /* 287 * Need to use raw, since this must be called before the 288 * recursive protection is performed. 289 */ 290 local_irq_save(flags); 291 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 292 cpu = raw_smp_processor_id(); 293 data = per_cpu_ptr(tr->array_buffer.data, cpu); 294 disabled = local_inc_return(&data->disabled); 295 296 if (likely(disabled == 1)) { 297 trace_ctx = tracing_gen_ctx_flags(flags); 298 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 299 #ifdef CONFIG_UNWINDER_FRAME_POINTER 300 if (ftrace_pids_enabled(op)) 301 skip++; 302 #endif 303 __trace_stack(tr, trace_ctx, skip); 304 } 305 306 local_dec(&data->disabled); 307 local_irq_restore(flags); 308 } 309 310 static inline bool is_repeat_check(struct trace_array *tr, 311 struct trace_func_repeats *last_info, 312 unsigned long ip, unsigned long parent_ip) 313 { 314 if (last_info->ip == ip && 315 last_info->parent_ip == parent_ip && 316 last_info->count < U16_MAX) { 317 last_info->ts_last_call = 318 ring_buffer_time_stamp(tr->array_buffer.buffer); 319 last_info->count++; 320 return true; 321 } 322 323 return false; 324 } 325 326 static inline void process_repeats(struct trace_array *tr, 327 unsigned long ip, unsigned long parent_ip, 328 struct trace_func_repeats *last_info, 329 unsigned int trace_ctx) 330 { 331 if (last_info->count) { 332 trace_last_func_repeats(tr, last_info, trace_ctx); 333 last_info->count = 0; 334 } 335 336 last_info->ip = ip; 337 last_info->parent_ip = parent_ip; 338 } 339 340 static void 341 function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 342 struct ftrace_ops *op, 343 struct ftrace_regs *fregs) 344 { 345 struct trace_func_repeats *last_info; 346 struct trace_array *tr = op->private; 347 unsigned int trace_ctx; 348 int bit; 349 350 if (unlikely(!tr->function_enabled)) 351 return; 352 353 bit = ftrace_test_recursion_trylock(ip, parent_ip); 354 if (bit < 0) 355 return; 356 357 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 358 if (!tracer_tracing_is_on(tr)) 359 goto out; 360 361 /* 362 * An interrupt may happen at any place here. But as far as I can see, 363 * the only damage that this can cause is to mess up the repetition 364 * counter without valuable data being lost. 365 * TODO: think about a solution that is better than just hoping to be 366 * lucky. 367 */ 368 last_info = this_cpu_ptr(tr->last_func_repeats); 369 if (is_repeat_check(tr, last_info, ip, parent_ip)) 370 goto out; 371 372 trace_ctx = tracing_gen_ctx_dec(); 373 process_repeats(tr, ip, parent_ip, last_info, trace_ctx); 374 375 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 376 377 out: 378 ftrace_test_recursion_unlock(bit); 379 } 380 381 static void 382 function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 383 struct ftrace_ops *op, 384 struct ftrace_regs *fregs) 385 { 386 struct trace_func_repeats *last_info; 387 struct trace_array *tr = op->private; 388 struct trace_array_cpu *data; 389 unsigned long flags; 390 long disabled; 391 int cpu; 392 unsigned int trace_ctx; 393 394 if (unlikely(!tr->function_enabled)) 395 return; 396 397 /* 398 * Need to use raw, since this must be called before the 399 * recursive protection is performed. 400 */ 401 local_irq_save(flags); 402 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 403 cpu = raw_smp_processor_id(); 404 data = per_cpu_ptr(tr->array_buffer.data, cpu); 405 disabled = local_inc_return(&data->disabled); 406 407 if (likely(disabled == 1)) { 408 last_info = per_cpu_ptr(tr->last_func_repeats, cpu); 409 if (is_repeat_check(tr, last_info, ip, parent_ip)) 410 goto out; 411 412 trace_ctx = tracing_gen_ctx_flags(flags); 413 process_repeats(tr, ip, parent_ip, last_info, trace_ctx); 414 415 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 416 __trace_stack(tr, trace_ctx, STACK_SKIP); 417 } 418 419 out: 420 local_dec(&data->disabled); 421 local_irq_restore(flags); 422 } 423 424 static struct tracer_opt func_opts[] = { 425 #ifdef CONFIG_STACKTRACE 426 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, 427 #endif 428 { TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) }, 429 #ifdef CONFIG_FUNCTION_TRACE_ARGS 430 { TRACER_OPT(func-args, TRACE_FUNC_OPT_ARGS) }, 431 #endif 432 { } /* Always set a last empty entry */ 433 }; 434 435 static struct tracer_flags func_flags = { 436 .val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */ 437 .opts = func_opts 438 }; 439 440 static void tracing_start_function_trace(struct trace_array *tr) 441 { 442 tr->function_enabled = 0; 443 register_ftrace_function(tr->ops); 444 tr->function_enabled = 1; 445 } 446 447 static void tracing_stop_function_trace(struct trace_array *tr) 448 { 449 tr->function_enabled = 0; 450 unregister_ftrace_function(tr->ops); 451 } 452 453 static struct tracer function_trace; 454 455 static int 456 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 457 { 458 ftrace_func_t func; 459 u32 new_flags; 460 461 /* Do nothing if already set. */ 462 if (!!set == !!(func_flags.val & bit)) 463 return 0; 464 465 /* We can change this flag only when not running. */ 466 if (tr->current_trace != &function_trace) 467 return 0; 468 469 new_flags = (func_flags.val & ~bit) | (set ? bit : 0); 470 func = select_trace_function(new_flags); 471 if (!func) 472 return -EINVAL; 473 474 /* Check if there's anything to change. */ 475 if (tr->ops->func == func) 476 return 0; 477 478 if (!handle_func_repeats(tr, new_flags)) 479 return -ENOMEM; 480 481 unregister_ftrace_function(tr->ops); 482 tr->ops->func = func; 483 register_ftrace_function(tr->ops); 484 485 return 0; 486 } 487 488 static struct tracer function_trace __tracer_data = 489 { 490 .name = "function", 491 .init = function_trace_init, 492 .reset = function_trace_reset, 493 .start = function_trace_start, 494 .flags = &func_flags, 495 .set_flag = func_set_flag, 496 .allow_instances = true, 497 #ifdef CONFIG_FTRACE_SELFTEST 498 .selftest = trace_selftest_startup_function, 499 #endif 500 }; 501 502 #ifdef CONFIG_DYNAMIC_FTRACE 503 static void update_traceon_count(struct ftrace_probe_ops *ops, 504 unsigned long ip, 505 struct trace_array *tr, bool on, 506 void *data) 507 { 508 struct ftrace_func_mapper *mapper = data; 509 long *count; 510 long old_count; 511 512 /* 513 * Tracing gets disabled (or enabled) once per count. 514 * This function can be called at the same time on multiple CPUs. 515 * It is fine if both disable (or enable) tracing, as disabling 516 * (or enabling) the second time doesn't do anything as the 517 * state of the tracer is already disabled (or enabled). 518 * What needs to be synchronized in this case is that the count 519 * only gets decremented once, even if the tracer is disabled 520 * (or enabled) twice, as the second one is really a nop. 521 * 522 * The memory barriers guarantee that we only decrement the 523 * counter once. First the count is read to a local variable 524 * and a read barrier is used to make sure that it is loaded 525 * before checking if the tracer is in the state we want. 526 * If the tracer is not in the state we want, then the count 527 * is guaranteed to be the old count. 528 * 529 * Next the tracer is set to the state we want (disabled or enabled) 530 * then a write memory barrier is used to make sure that 531 * the new state is visible before changing the counter by 532 * one minus the old counter. This guarantees that another CPU 533 * executing this code will see the new state before seeing 534 * the new counter value, and would not do anything if the new 535 * counter is seen. 536 * 537 * Note, there is no synchronization between this and a user 538 * setting the tracing_on file. But we currently don't care 539 * about that. 540 */ 541 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 542 old_count = *count; 543 544 if (old_count <= 0) 545 return; 546 547 /* Make sure we see count before checking tracing state */ 548 smp_rmb(); 549 550 if (on == !!tracer_tracing_is_on(tr)) 551 return; 552 553 if (on) 554 tracer_tracing_on(tr); 555 else 556 tracer_tracing_off(tr); 557 558 /* Make sure tracing state is visible before updating count */ 559 smp_wmb(); 560 561 *count = old_count - 1; 562 } 563 564 static void 565 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, 566 struct trace_array *tr, struct ftrace_probe_ops *ops, 567 void *data) 568 { 569 update_traceon_count(ops, ip, tr, 1, data); 570 } 571 572 static void 573 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, 574 struct trace_array *tr, struct ftrace_probe_ops *ops, 575 void *data) 576 { 577 update_traceon_count(ops, ip, tr, 0, data); 578 } 579 580 static void 581 ftrace_traceon(unsigned long ip, unsigned long parent_ip, 582 struct trace_array *tr, struct ftrace_probe_ops *ops, 583 void *data) 584 { 585 if (tracer_tracing_is_on(tr)) 586 return; 587 588 tracer_tracing_on(tr); 589 } 590 591 static void 592 ftrace_traceoff(unsigned long ip, unsigned long parent_ip, 593 struct trace_array *tr, struct ftrace_probe_ops *ops, 594 void *data) 595 { 596 if (!tracer_tracing_is_on(tr)) 597 return; 598 599 tracer_tracing_off(tr); 600 } 601 602 #ifdef CONFIG_UNWINDER_ORC 603 /* 604 * Skip 3: 605 * 606 * function_trace_probe_call() 607 * ftrace_ops_assist_func() 608 * ftrace_call() 609 */ 610 #define FTRACE_STACK_SKIP 3 611 #else 612 /* 613 * Skip 5: 614 * 615 * __trace_stack() 616 * ftrace_stacktrace() 617 * function_trace_probe_call() 618 * ftrace_ops_assist_func() 619 * ftrace_call() 620 */ 621 #define FTRACE_STACK_SKIP 5 622 #endif 623 624 static __always_inline void trace_stack(struct trace_array *tr) 625 { 626 __trace_stack(tr, tracing_gen_ctx_dec(), FTRACE_STACK_SKIP); 627 } 628 629 static void 630 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, 631 struct trace_array *tr, struct ftrace_probe_ops *ops, 632 void *data) 633 { 634 trace_stack(tr); 635 } 636 637 static void 638 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, 639 struct trace_array *tr, struct ftrace_probe_ops *ops, 640 void *data) 641 { 642 struct ftrace_func_mapper *mapper = data; 643 long *count; 644 long old_count; 645 long new_count; 646 647 if (!tracing_is_on()) 648 return; 649 650 /* unlimited? */ 651 if (!mapper) { 652 trace_stack(tr); 653 return; 654 } 655 656 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 657 658 /* 659 * Stack traces should only execute the number of times the 660 * user specified in the counter. 661 */ 662 do { 663 old_count = *count; 664 665 if (!old_count) 666 return; 667 668 new_count = old_count - 1; 669 new_count = cmpxchg(count, old_count, new_count); 670 if (new_count == old_count) 671 trace_stack(tr); 672 673 if (!tracing_is_on()) 674 return; 675 676 } while (new_count != old_count); 677 } 678 679 static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, 680 void *data) 681 { 682 struct ftrace_func_mapper *mapper = data; 683 long *count = NULL; 684 685 if (mapper) 686 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 687 688 if (count) { 689 if (*count <= 0) 690 return 0; 691 (*count)--; 692 } 693 694 return 1; 695 } 696 697 static void 698 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, 699 struct trace_array *tr, struct ftrace_probe_ops *ops, 700 void *data) 701 { 702 if (update_count(ops, ip, data)) 703 ftrace_dump(DUMP_ALL); 704 } 705 706 /* Only dump the current CPU buffer. */ 707 static void 708 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, 709 struct trace_array *tr, struct ftrace_probe_ops *ops, 710 void *data) 711 { 712 if (update_count(ops, ip, data)) 713 ftrace_dump(DUMP_ORIG); 714 } 715 716 static int 717 ftrace_probe_print(const char *name, struct seq_file *m, 718 unsigned long ip, struct ftrace_probe_ops *ops, 719 void *data) 720 { 721 struct ftrace_func_mapper *mapper = data; 722 long *count = NULL; 723 724 seq_printf(m, "%ps:%s", (void *)ip, name); 725 726 if (mapper) 727 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 728 729 if (count) 730 seq_printf(m, ":count=%ld\n", *count); 731 else 732 seq_puts(m, ":unlimited\n"); 733 734 return 0; 735 } 736 737 static int 738 ftrace_traceon_print(struct seq_file *m, unsigned long ip, 739 struct ftrace_probe_ops *ops, 740 void *data) 741 { 742 return ftrace_probe_print("traceon", m, ip, ops, data); 743 } 744 745 static int 746 ftrace_traceoff_print(struct seq_file *m, unsigned long ip, 747 struct ftrace_probe_ops *ops, void *data) 748 { 749 return ftrace_probe_print("traceoff", m, ip, ops, data); 750 } 751 752 static int 753 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, 754 struct ftrace_probe_ops *ops, void *data) 755 { 756 return ftrace_probe_print("stacktrace", m, ip, ops, data); 757 } 758 759 static int 760 ftrace_dump_print(struct seq_file *m, unsigned long ip, 761 struct ftrace_probe_ops *ops, void *data) 762 { 763 return ftrace_probe_print("dump", m, ip, ops, data); 764 } 765 766 static int 767 ftrace_cpudump_print(struct seq_file *m, unsigned long ip, 768 struct ftrace_probe_ops *ops, void *data) 769 { 770 return ftrace_probe_print("cpudump", m, ip, ops, data); 771 } 772 773 774 static int 775 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 776 unsigned long ip, void *init_data, void **data) 777 { 778 struct ftrace_func_mapper *mapper = *data; 779 780 if (!mapper) { 781 mapper = allocate_ftrace_func_mapper(); 782 if (!mapper) 783 return -ENOMEM; 784 *data = mapper; 785 } 786 787 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 788 } 789 790 static void 791 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 792 unsigned long ip, void *data) 793 { 794 struct ftrace_func_mapper *mapper = data; 795 796 if (!ip) { 797 free_ftrace_func_mapper(mapper, NULL); 798 return; 799 } 800 801 ftrace_func_mapper_remove_ip(mapper, ip); 802 } 803 804 static struct ftrace_probe_ops traceon_count_probe_ops = { 805 .func = ftrace_traceon_count, 806 .print = ftrace_traceon_print, 807 .init = ftrace_count_init, 808 .free = ftrace_count_free, 809 }; 810 811 static struct ftrace_probe_ops traceoff_count_probe_ops = { 812 .func = ftrace_traceoff_count, 813 .print = ftrace_traceoff_print, 814 .init = ftrace_count_init, 815 .free = ftrace_count_free, 816 }; 817 818 static struct ftrace_probe_ops stacktrace_count_probe_ops = { 819 .func = ftrace_stacktrace_count, 820 .print = ftrace_stacktrace_print, 821 .init = ftrace_count_init, 822 .free = ftrace_count_free, 823 }; 824 825 static struct ftrace_probe_ops dump_probe_ops = { 826 .func = ftrace_dump_probe, 827 .print = ftrace_dump_print, 828 .init = ftrace_count_init, 829 .free = ftrace_count_free, 830 }; 831 832 static struct ftrace_probe_ops cpudump_probe_ops = { 833 .func = ftrace_cpudump_probe, 834 .print = ftrace_cpudump_print, 835 }; 836 837 static struct ftrace_probe_ops traceon_probe_ops = { 838 .func = ftrace_traceon, 839 .print = ftrace_traceon_print, 840 }; 841 842 static struct ftrace_probe_ops traceoff_probe_ops = { 843 .func = ftrace_traceoff, 844 .print = ftrace_traceoff_print, 845 }; 846 847 static struct ftrace_probe_ops stacktrace_probe_ops = { 848 .func = ftrace_stacktrace, 849 .print = ftrace_stacktrace_print, 850 }; 851 852 static int 853 ftrace_trace_probe_callback(struct trace_array *tr, 854 struct ftrace_probe_ops *ops, 855 struct ftrace_hash *hash, char *glob, 856 char *cmd, char *param, int enable) 857 { 858 void *count = (void *)-1; 859 char *number; 860 int ret; 861 862 /* hash funcs only work with set_ftrace_filter */ 863 if (!enable) 864 return -EINVAL; 865 866 if (glob[0] == '!') 867 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 868 869 if (!param) 870 goto out_reg; 871 872 number = strsep(¶m, ":"); 873 874 if (!strlen(number)) 875 goto out_reg; 876 877 /* 878 * We use the callback data field (which is a pointer) 879 * as our counter. 880 */ 881 ret = kstrtoul(number, 0, (unsigned long *)&count); 882 if (ret) 883 return ret; 884 885 out_reg: 886 ret = register_ftrace_function_probe(glob, tr, ops, count); 887 888 return ret < 0 ? ret : 0; 889 } 890 891 static int 892 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, 893 char *glob, char *cmd, char *param, int enable) 894 { 895 struct ftrace_probe_ops *ops; 896 897 if (!tr) 898 return -ENODEV; 899 900 /* we register both traceon and traceoff to this callback */ 901 if (strcmp(cmd, "traceon") == 0) 902 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; 903 else 904 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; 905 906 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 907 param, enable); 908 } 909 910 static int 911 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, 912 char *glob, char *cmd, char *param, int enable) 913 { 914 struct ftrace_probe_ops *ops; 915 916 if (!tr) 917 return -ENODEV; 918 919 ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; 920 921 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 922 param, enable); 923 } 924 925 static int 926 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, 927 char *glob, char *cmd, char *param, int enable) 928 { 929 struct ftrace_probe_ops *ops; 930 931 if (!tr) 932 return -ENODEV; 933 934 ops = &dump_probe_ops; 935 936 /* Only dump once. */ 937 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 938 "1", enable); 939 } 940 941 static int 942 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, 943 char *glob, char *cmd, char *param, int enable) 944 { 945 struct ftrace_probe_ops *ops; 946 947 if (!tr) 948 return -ENODEV; 949 950 ops = &cpudump_probe_ops; 951 952 /* Only dump once. */ 953 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 954 "1", enable); 955 } 956 957 static struct ftrace_func_command ftrace_traceon_cmd = { 958 .name = "traceon", 959 .func = ftrace_trace_onoff_callback, 960 }; 961 962 static struct ftrace_func_command ftrace_traceoff_cmd = { 963 .name = "traceoff", 964 .func = ftrace_trace_onoff_callback, 965 }; 966 967 static struct ftrace_func_command ftrace_stacktrace_cmd = { 968 .name = "stacktrace", 969 .func = ftrace_stacktrace_callback, 970 }; 971 972 static struct ftrace_func_command ftrace_dump_cmd = { 973 .name = "dump", 974 .func = ftrace_dump_callback, 975 }; 976 977 static struct ftrace_func_command ftrace_cpudump_cmd = { 978 .name = "cpudump", 979 .func = ftrace_cpudump_callback, 980 }; 981 982 static int __init init_func_cmd_traceon(void) 983 { 984 int ret; 985 986 ret = register_ftrace_command(&ftrace_traceoff_cmd); 987 if (ret) 988 return ret; 989 990 ret = register_ftrace_command(&ftrace_traceon_cmd); 991 if (ret) 992 goto out_free_traceoff; 993 994 ret = register_ftrace_command(&ftrace_stacktrace_cmd); 995 if (ret) 996 goto out_free_traceon; 997 998 ret = register_ftrace_command(&ftrace_dump_cmd); 999 if (ret) 1000 goto out_free_stacktrace; 1001 1002 ret = register_ftrace_command(&ftrace_cpudump_cmd); 1003 if (ret) 1004 goto out_free_dump; 1005 1006 return 0; 1007 1008 out_free_dump: 1009 unregister_ftrace_command(&ftrace_dump_cmd); 1010 out_free_stacktrace: 1011 unregister_ftrace_command(&ftrace_stacktrace_cmd); 1012 out_free_traceon: 1013 unregister_ftrace_command(&ftrace_traceon_cmd); 1014 out_free_traceoff: 1015 unregister_ftrace_command(&ftrace_traceoff_cmd); 1016 1017 return ret; 1018 } 1019 #else 1020 static inline int init_func_cmd_traceon(void) 1021 { 1022 return 0; 1023 } 1024 #endif /* CONFIG_DYNAMIC_FTRACE */ 1025 1026 __init int init_function_trace(void) 1027 { 1028 init_func_cmd_traceon(); 1029 return register_tracer(&function_trace); 1030 } 1031