Lines Matching +full:ftrace +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0
3 * Infrastructure for profiling code inserted by 'gcc -pg'.
5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
8 * Originally ported from the -rt patch by:
13 * Copyright (C) 2004-2006 Ingo Molnar
29 #include <linux/ftrace.h>
93 /* ftrace_enabled is a method to turn ftrace on or off */
106 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) in ftrace_pids_enabled()
109 tr = ops->private; in ftrace_pids_enabled()
111 return tr->function_pids != NULL || tr->function_no_pids != NULL; in ftrace_pids_enabled()
162 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { in ftrace_ops_init()
163 mutex_init(&ops->local_hash.regex_lock); in ftrace_ops_init()
164 ops->func_hash = &ops->local_hash; in ftrace_ops_init()
165 ops->flags |= FTRACE_OPS_FL_INITIALIZED; in ftrace_ops_init()
173 struct trace_array *tr = op->private; in ftrace_pid_func()
177 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid); in ftrace_pid_func()
181 pid != current->pid) in ftrace_pid_func()
185 op->saved_func(ip, parent_ip, op, fregs); in ftrace_pid_func()
200 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) || in ftrace_ops_get_list_func()
228 } else if (rcu_dereference_protected(ftrace_ops_list->next, in update_ftrace_function()
289 rcu_assign_pointer(ops->next, *list); in add_ftrace_ops()
294 * the ops->next pointer is valid before another CPU sees in add_ftrace_ops()
311 rcu_dereference_protected(ops->next, in remove_ftrace_ops()
317 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) in remove_ftrace_ops()
322 return -1; in remove_ftrace_ops()
324 *p = (*p)->next; in remove_ftrace_ops()
332 if (ops->flags & FTRACE_OPS_FL_DELETED) in __register_ftrace_function()
333 return -EINVAL; in __register_ftrace_function()
335 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) in __register_ftrace_function()
336 return -EBUSY; in __register_ftrace_function()
344 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && in __register_ftrace_function()
345 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) in __register_ftrace_function()
346 return -EINVAL; in __register_ftrace_function()
348 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) in __register_ftrace_function()
349 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; in __register_ftrace_function()
351 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT)) in __register_ftrace_function()
352 return -EBUSY; in __register_ftrace_function()
355 ops->flags |= FTRACE_OPS_FL_DYNAMIC; in __register_ftrace_function()
360 ops->saved_func = ops->func; in __register_ftrace_function()
363 ops->func = ftrace_pid_func; in __register_ftrace_function()
377 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) in __unregister_ftrace_function()
378 return -EBUSY; in __unregister_ftrace_function()
388 ops->func = ops->saved_func; in __unregister_ftrace_function()
402 if (op->flags & FTRACE_OPS_FL_PID) { in ftrace_update_pid_func()
403 op->func = ftrace_pids_enabled(op) ? in ftrace_update_pid_func()
404 ftrace_pid_func : op->saved_func; in ftrace_update_pid_func()
438 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
445 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
465 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
466 pg = pg->next; in function_stat_next()
469 rec = &pg->records[0]; in function_stat_next()
470 if (!rec->counter) in function_stat_next()
482 if (!stat || !stat->start) in function_stat_start()
485 return function_stat_next(&stat->start->records[0], 0); in function_stat_start()
495 if (a->time < b->time) in function_stat_cmp()
496 return -1; in function_stat_cmp()
497 if (a->time > b->time) in function_stat_cmp()
509 if (a->counter < b->counter) in function_stat_cmp()
510 return -1; in function_stat_cmp()
511 if (a->counter > b->counter) in function_stat_cmp()
523 " -------- " in function_stat_headers()
524 "--- ---- --- ---\n"); in function_stat_headers()
527 " -------- ---\n"); in function_stat_headers()
545 if (unlikely(rec->counter == 0)) { in function_stat_show()
546 ret = -EBUSY; in function_stat_show()
551 avg = div64_ul(rec->time, rec->counter); in function_stat_show()
556 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); in function_stat_show()
557 seq_printf(m, " %-30.30s %10lu", str, rec->counter); in function_stat_show()
563 if (rec->counter <= 1) in function_stat_show()
568 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) in function_stat_show()
570 stddev = rec->counter * rec->time_squared - in function_stat_show()
571 rec->time * rec->time; in function_stat_show()
574 * Divide only 1000 for ns^2 -> us^2 conversion. in function_stat_show()
578 rec->counter * (rec->counter - 1) * 1000); in function_stat_show()
582 trace_print_graph_duration(rec->time, &s); in function_stat_show()
600 pg = stat->pages = stat->start; in ftrace_profile_reset()
603 memset(pg->records, 0, PROFILE_RECORDS_SIZE); in ftrace_profile_reset()
604 pg->index = 0; in ftrace_profile_reset()
605 pg = pg->next; in ftrace_profile_reset()
608 memset(stat->hash, 0, in ftrace_profile_reset()
620 if (stat->pages) in ftrace_profile_pages_init()
623 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
624 if (!stat->pages) in ftrace_profile_pages_init()
625 return -ENOMEM; in ftrace_profile_pages_init()
640 pg = stat->start = stat->pages; in ftrace_profile_pages_init()
645 pg->next = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
646 if (!pg->next) in ftrace_profile_pages_init()
648 pg = pg->next; in ftrace_profile_pages_init()
654 pg = stat->start; in ftrace_profile_pages_init()
658 pg = pg->next; in ftrace_profile_pages_init()
662 stat->pages = NULL; in ftrace_profile_pages_init()
663 stat->start = NULL; in ftrace_profile_pages_init()
665 return -ENOMEM; in ftrace_profile_pages_init()
671 int size; in ftrace_profile_init_cpu() local
675 if (stat->hash) { in ftrace_profile_init_cpu()
685 size = FTRACE_PROFILE_HASH_SIZE; in ftrace_profile_init_cpu()
687 stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL); in ftrace_profile_init_cpu()
689 if (!stat->hash) in ftrace_profile_init_cpu()
690 return -ENOMEM; in ftrace_profile_init_cpu()
694 kfree(stat->hash); in ftrace_profile_init_cpu()
695 stat->hash = NULL; in ftrace_profile_init_cpu()
696 return -ENOMEM; in ftrace_profile_init_cpu()
725 hhd = &stat->hash[key]; in ftrace_find_profiled_func()
731 if (rec->ip == ip) in ftrace_find_profiled_func()
743 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); in ftrace_add_profile()
744 hlist_add_head_rcu(&rec->node, &stat->hash[key]); in ftrace_add_profile()
756 if (atomic_inc_return(&stat->disabled) != 1) in ftrace_profile_alloc()
767 if (stat->pages->index == PROFILES_PER_PAGE) { in ftrace_profile_alloc()
768 if (!stat->pages->next) in ftrace_profile_alloc()
770 stat->pages = stat->pages->next; in ftrace_profile_alloc()
773 rec = &stat->pages->records[stat->pages->index++]; in ftrace_profile_alloc()
774 rec->ip = ip; in ftrace_profile_alloc()
778 atomic_dec(&stat->disabled); in ftrace_profile_alloc()
797 if (!stat->hash || !ftrace_profile_enabled) in function_profile_call()
807 rec->counter++; in function_profile_call()
824 function_profile_call(trace->func, 0, NULL, NULL); in profile_graph_entry()
827 if (!current->ret_stack) in profile_graph_entry()
832 ret_stack->subtime = 0; in profile_graph_entry()
847 if (!stat->hash || !ftrace_profile_enabled) in profile_graph_return()
851 if (!trace->calltime) in profile_graph_return()
854 calltime = trace->rettime - trace->calltime; in profile_graph_return()
861 ret_stack->subtime += calltime; in profile_graph_return()
864 if (ret_stack && ret_stack->subtime < calltime) in profile_graph_return()
865 calltime -= ret_stack->subtime; in profile_graph_return()
870 rec = ftrace_find_profiled_func(stat, trace->func); in profile_graph_return()
872 rec->time += calltime; in profile_graph_return()
873 rec->time_squared += calltime * calltime; in profile_graph_return()
1006 stat->stat = function_stats; in ftrace_profile_tracefs()
1007 stat->stat.name = name; in ftrace_profile_tracefs()
1008 ret = register_stat_tracer(&stat->stat); in ftrace_profile_tracefs()
1040 # error Dynamic ftrace depends on MCOUNT_RECORD
1074 * Used by the stack unwinder to know about dynamic ftrace trampolines.
1092 if (op->trampoline && op->trampoline_size) in ftrace_ops_trampoline()
1093 if (addr >= op->trampoline && in ftrace_ops_trampoline()
1094 addr < op->trampoline + op->trampoline_size) { in ftrace_ops_trampoline()
1131 if (hash->size_bits > 0) in ftrace_hash_key()
1132 return hash_long(ip, hash->size_bits); in ftrace_hash_key()
1146 hhd = &hash->buckets[key]; in __ftrace_lookup_ip()
1149 if (entry->ip == ip) in __ftrace_lookup_ip()
1156 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1180 key = ftrace_hash_key(hash, entry->ip); in __add_hash_entry()
1181 hhd = &hash->buckets[key]; in __add_hash_entry()
1182 hlist_add_head(&entry->hlist, hhd); in __add_hash_entry()
1183 hash->count++; in __add_hash_entry()
1195 entry->ip = ip; in add_hash_entry()
1205 hlist_del(&entry->hlist); in free_hash_entry()
1207 hash->count--; in free_hash_entry()
1214 hlist_del_rcu(&entry->hlist); in remove_hash_entry()
1215 hash->count--; in remove_hash_entry()
1223 int size = 1 << hash->size_bits; in ftrace_hash_clear() local
1226 if (!hash->count) in ftrace_hash_clear()
1229 for (i = 0; i < size; i++) { in ftrace_hash_clear()
1230 hhd = &hash->buckets[i]; in ftrace_hash_clear()
1234 FTRACE_WARN_ON(hash->count); in ftrace_hash_clear()
1239 list_del(&ftrace_mod->list); in free_ftrace_mod()
1240 kfree(ftrace_mod->module); in free_ftrace_mod()
1241 kfree(ftrace_mod->func); in free_ftrace_mod()
1264 kfree(hash->buckets); in free_ftrace_hash()
1280 call_rcu(&hash->rcu, __free_ftrace_hash_rcu); in free_ftrace_hash_rcu()
1284 * ftrace_free_filter - remove all filters for an ftrace_ops
1285 * @ops - the ops to remove the filters from
1290 free_ftrace_hash(ops->func_hash->filter_hash); in ftrace_free_filter()
1291 free_ftrace_hash(ops->func_hash->notrace_hash); in ftrace_free_filter()
1298 int size; in alloc_ftrace_hash() local
1304 size = 1 << size_bits; in alloc_ftrace_hash()
1305 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); in alloc_ftrace_hash()
1307 if (!hash->buckets) { in alloc_ftrace_hash()
1312 hash->size_bits = size_bits; in alloc_ftrace_hash()
1323 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace; in ftrace_add_mod()
1327 return -ENOMEM; in ftrace_add_mod()
1329 INIT_LIST_HEAD(&ftrace_mod->list); in ftrace_add_mod()
1330 ftrace_mod->func = kstrdup(func, GFP_KERNEL); in ftrace_add_mod()
1331 ftrace_mod->module = kstrdup(module, GFP_KERNEL); in ftrace_add_mod()
1332 ftrace_mod->enable = enable; in ftrace_add_mod()
1334 if (!ftrace_mod->func || !ftrace_mod->module) in ftrace_add_mod()
1337 list_add(&ftrace_mod->list, mod_head); in ftrace_add_mod()
1344 return -ENOMEM; in ftrace_add_mod()
1352 int size; in alloc_and_copy_ftrace_hash() local
1360 new_hash->flags = hash->flags; in alloc_and_copy_ftrace_hash()
1366 size = 1 << hash->size_bits; in alloc_and_copy_ftrace_hash()
1367 for (i = 0; i < size; i++) { in alloc_and_copy_ftrace_hash()
1368 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in alloc_and_copy_ftrace_hash()
1369 if (add_hash_entry(new_hash, entry->ip) == NULL) in alloc_and_copy_ftrace_hash()
1374 FTRACE_WARN_ON(new_hash->count != hash->count); in alloc_and_copy_ftrace_hash()
1391 static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size) in dup_hash() argument
1401 * Use around half the size (max bit of it), but in dup_hash()
1402 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits). in dup_hash()
1404 bits = fls(size / 2); in dup_hash()
1414 new_hash->flags = src->flags; in dup_hash()
1416 size = 1 << src->size_bits; in dup_hash()
1417 for (i = 0; i < size; i++) { in dup_hash()
1418 hhd = &src->buckets[i]; in dup_hash()
1430 int size = src->count; in __ftrace_hash_move() local
1438 return dup_hash(src, size); in __ftrace_hash_move()
1449 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) in ftrace_hash_move()
1450 return -EINVAL; in ftrace_hash_move()
1454 return -ENOMEM; in ftrace_hash_move()
1488 return (ftrace_hash_empty(hash->filter_hash) || in hash_contains_ip()
1489 __ftrace_lookup_ip(hash->filter_hash, ip)) && in hash_contains_ip()
1490 (ftrace_hash_empty(hash->notrace_hash) || in hash_contains_ip()
1491 !__ftrace_lookup_ip(hash->notrace_hash, ip)); in hash_contains_ip()
1496 * the ops->func or not.
1498 * It's a match if the ip is in the ops->filter_hash or
1501 * the ip is not in the ops->notrace_hash.
1514 * There's a small race when adding ops that the ftrace handler in ftrace_ops_test()
1518 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) in ftrace_ops_test()
1522 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); in ftrace_ops_test()
1523 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); in ftrace_ops_test()
1538 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1540 for (_____i = 0; _____i < pg->index; _____i++) { \
1541 rec = &pg->records[_____i];
1553 if (key->flags < rec->ip) in ftrace_cmp_recs()
1554 return -1; in ftrace_cmp_recs()
1555 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) in ftrace_cmp_recs()
1569 for (pg = ftrace_pages_start; pg; pg = pg->next) { in lookup_rec()
1570 if (pg->index == 0 || in lookup_rec()
1571 end < pg->records[0].ip || in lookup_rec()
1572 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in lookup_rec()
1574 rec = bsearch(&key, pg->records, pg->index, in lookup_rec()
1584 * ftrace_location_range - return the first address of a traced location
1590 * Returns rec->ip if the related ftrace location is a least partly within
1592 * that is either a NOP or call to the function tracer. It checks the ftrace
1601 return rec->ip; in ftrace_location_range()
1607 * ftrace_location - return the ftrace location
1610 * If @ip matches the ftrace location, return @ip.
1611 * If @ip matches sym+0, return sym's ftrace location.
1618 unsigned long size; in ftrace_location() local
1622 if (!kallsyms_lookup_size_offset(ip, &size, &offset)) in ftrace_location()
1627 rec = lookup_rec(ip, ip + size - 1); in ftrace_location()
1631 return rec->ip; in ftrace_location()
1638 * ftrace_text_reserved - return true if range contains an ftrace location
1642 * Returns 1 if @start and @end contains a ftrace location.
1644 * the function tracer. It checks the ftrace internal tables to
1664 ops != &ftrace_list_end; ops = ops->next) { in test_rec_ops_needs_regs()
1665 /* pass rec in as regs to have non-NULL val */ in test_rec_ops_needs_regs()
1666 if (ftrace_ops_test(ops, rec->ip, rec)) { in test_rec_ops_needs_regs()
1667 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in test_rec_ops_needs_regs()
1692 return rec->flags & FTRACE_FL_DISABLED && in skip_record()
1693 !(rec->flags & FTRACE_FL_ENABLED); in skip_record()
1709 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_rec_update()
1724 hash = ops->func_hash->filter_hash; in __ftrace_hash_rec_update()
1725 other_hash = ops->func_hash->notrace_hash; in __ftrace_hash_rec_update()
1730 hash = ops->func_hash->notrace_hash; in __ftrace_hash_rec_update()
1731 other_hash = ops->func_hash->filter_hash; in __ftrace_hash_rec_update()
1753 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) in __ftrace_hash_rec_update()
1756 in_hash = !!ftrace_lookup_ip(hash, rec->ip); in __ftrace_hash_rec_update()
1757 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); in __ftrace_hash_rec_update()
1779 rec->flags++; in __ftrace_hash_rec_update()
1783 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1784 rec->flags |= FTRACE_FL_DIRECT; in __ftrace_hash_rec_update()
1791 if (ftrace_rec_count(rec) == 1 && ops->trampoline) in __ftrace_hash_rec_update()
1792 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1800 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1806 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in __ftrace_hash_rec_update()
1807 rec->flags |= FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1811 rec->flags--; in __ftrace_hash_rec_update()
1819 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1820 rec->flags &= ~FTRACE_FL_DIRECT; in __ftrace_hash_rec_update()
1829 rec->flags & FTRACE_FL_REGS && in __ftrace_hash_rec_update()
1830 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in __ftrace_hash_rec_update()
1832 rec->flags &= ~FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1844 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1846 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1855 * If the rec has a single associated ops, and ops->func can be in __ftrace_hash_rec_update()
1860 ftrace_ops_get_func(ops) == ops->func) in __ftrace_hash_rec_update()
1861 rec->flags |= FTRACE_FL_CALL_OPS; in __ftrace_hash_rec_update()
1863 rec->flags &= ~FTRACE_FL_CALL_OPS; in __ftrace_hash_rec_update()
1871 if (!all && count == hash->count) in __ftrace_hash_rec_update()
1897 if (ops->func_hash != &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1908 if (op->func_hash == &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1927 * or no-needed to update, -EBUSY if it detects a conflict of the flag
1928 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1930 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1931 * - If the hash is EMPTY_HASH, it hits nothing
1932 * - Anything else hits the recs which match the hash entries.
1937 * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate
1951 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_update_ipmodify()
1954 is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1955 is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; in __ftrace_hash_update_ipmodify()
1970 return -EINVAL; in __ftrace_hash_update_ipmodify()
1972 /* Update rec->flags */ in __ftrace_hash_update_ipmodify()
1975 if (rec->flags & FTRACE_FL_DISABLED) in __ftrace_hash_update_ipmodify()
1979 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1980 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1985 if (rec->flags & FTRACE_FL_IPMODIFY) { in __ftrace_hash_update_ipmodify()
1992 FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT); in __ftrace_hash_update_ipmodify()
2000 if (!ops->ops_func) in __ftrace_hash_update_ipmodify()
2001 return -EBUSY; in __ftrace_hash_update_ipmodify()
2002 ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); in __ftrace_hash_update_ipmodify()
2006 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2009 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2021 if (rec->flags & FTRACE_FL_DISABLED) in __ftrace_hash_update_ipmodify()
2027 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2028 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2033 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2035 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2039 return -EBUSY; in __ftrace_hash_update_ipmodify()
2044 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_enable()
2055 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_disable()
2066 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_update()
2099 pr_info("Initializing ftrace call sites\n"); in print_bug_type()
2102 pr_info("Setting ftrace call site to NOP\n"); in print_bug_type()
2105 pr_info("Setting ftrace call site to call ftrace function\n"); in print_bug_type()
2108 pr_info("Updating ftrace call site to call a different ftrace function\n"); in print_bug_type()
2114 * ftrace_bug - report and shutdown function tracer
2121 * EFAULT - if the problem happens on reading the @ip address
2122 * EINVAL - if what is read at @ip is not what was expected
2123 * EPERM - if the problem happens on writing to the @ip address
2127 unsigned long ip = rec ? rec->ip : 0; in ftrace_bug()
2129 pr_info("------------[ ftrace bug ]------------\n"); in ftrace_bug()
2132 case -EFAULT: in ftrace_bug()
2133 pr_info("ftrace faulted on modifying "); in ftrace_bug()
2136 case -EINVAL: in ftrace_bug()
2137 pr_info("ftrace failed to modify "); in ftrace_bug()
2146 case -EPERM: in ftrace_bug()
2147 pr_info("ftrace faulted on writing "); in ftrace_bug()
2151 pr_info("ftrace faulted on unknown error "); in ftrace_bug()
2158 pr_info("ftrace record flags: %lx\n", rec->flags); in ftrace_bug()
2160 rec->flags & FTRACE_FL_REGS ? " R" : " ", in ftrace_bug()
2161 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " "); in ftrace_bug()
2162 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_bug()
2167 (void *)ops->trampoline, in ftrace_bug()
2168 (void *)ops->func); in ftrace_bug()
2212 if (!(rec->flags & FTRACE_FL_REGS) != in ftrace_check_record()
2213 !(rec->flags & FTRACE_FL_REGS_EN)) in ftrace_check_record()
2216 if (!(rec->flags & FTRACE_FL_TRAMP) != in ftrace_check_record()
2217 !(rec->flags & FTRACE_FL_TRAMP_EN)) in ftrace_check_record()
2232 if (!(rec->flags & FTRACE_FL_DIRECT) != in ftrace_check_record()
2233 !(rec->flags & FTRACE_FL_DIRECT_EN)) in ftrace_check_record()
2235 } else if (rec->flags & FTRACE_FL_DIRECT_EN) { in ftrace_check_record()
2245 if (!(rec->flags & FTRACE_FL_CALL_OPS) != in ftrace_check_record()
2246 !(rec->flags & FTRACE_FL_CALL_OPS_EN)) in ftrace_check_record()
2248 } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) { in ftrace_check_record()
2254 if ((rec->flags & FTRACE_FL_ENABLED) == flag) in ftrace_check_record()
2259 flag ^= rec->flags & FTRACE_FL_ENABLED; in ftrace_check_record()
2262 rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED; in ftrace_check_record()
2264 if (rec->flags & FTRACE_FL_REGS) in ftrace_check_record()
2265 rec->flags |= FTRACE_FL_REGS_EN; in ftrace_check_record()
2267 rec->flags &= ~FTRACE_FL_REGS_EN; in ftrace_check_record()
2270 if (rec->flags & FTRACE_FL_TRAMP) in ftrace_check_record()
2271 rec->flags |= FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2273 rec->flags &= ~FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2277 if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY)) in ftrace_check_record()
2278 rec->flags |= FTRACE_FL_MODIFIED; in ftrace_check_record()
2284 * directly (no ftrace trampoline). in ftrace_check_record()
2287 if (rec->flags & FTRACE_FL_DIRECT) in ftrace_check_record()
2288 rec->flags |= FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2290 rec->flags &= ~FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2296 rec->flags &= ~FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2302 if (rec->flags & FTRACE_FL_CALL_OPS) in ftrace_check_record()
2303 rec->flags |= FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2305 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2311 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2321 * from the save regs, to a non-save regs function or in ftrace_check_record()
2336 rec->flags &= FTRACE_NOCLEAR_FLAGS; in ftrace_check_record()
2342 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | in ftrace_check_record()
2352 * ftrace_update_record - set a record that now is tracing or not
2365 * ftrace_test_record - check if the record has been enabled or not
2382 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any()
2386 if (!op->trampoline) in ftrace_find_tramp_ops_any()
2389 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_any()
2400 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any_other()
2404 if (op == op_exclude || !op->trampoline) in ftrace_find_tramp_ops_any_other()
2407 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_any_other()
2418 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_next()
2422 if (!op->trampoline) in ftrace_find_tramp_ops_next()
2425 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_next()
2436 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_curr()
2445 if (hash_contains_ip(ip, &removed_ops->old_hash)) in ftrace_find_tramp_ops_curr()
2469 if (!op->trampoline) in ftrace_find_tramp_ops_curr()
2476 if (op->flags & FTRACE_OPS_FL_ADDING) in ftrace_find_tramp_ops_curr()
2485 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && in ftrace_find_tramp_ops_curr()
2486 hash_contains_ip(ip, &op->old_hash)) in ftrace_find_tramp_ops_curr()
2493 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && in ftrace_find_tramp_ops_curr()
2494 hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_curr()
2506 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_new()
2509 /* pass rec in as regs to have non-NULL val */ in ftrace_find_tramp_ops_new()
2510 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_new()
2521 unsigned long ip = rec->ip; in ftrace_find_unique_ops()
2525 if (hash_contains_ip(ip, op->func_hash)) { in ftrace_find_unique_ops()
2554 return entry->direct; in ftrace_find_rec_direct()
2560 unsigned long addr = READ_ONCE(ops->direct_call); in call_direct_funcs()
2570 * ftrace_get_addr_new - Get the call address to set to
2571 * @rec: The ftrace record descriptor
2584 if ((rec->flags & FTRACE_FL_DIRECT) && in ftrace_get_addr_new()
2586 addr = ftrace_find_rec_direct(rec->ip); in ftrace_get_addr_new()
2593 if (rec->flags & FTRACE_FL_TRAMP) { in ftrace_get_addr_new()
2595 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { in ftrace_get_addr_new()
2597 (void *)rec->ip, (void *)rec->ip, rec->flags); in ftrace_get_addr_new()
2598 /* Ftrace is shutting down, return anything */ in ftrace_get_addr_new()
2601 return ops->trampoline; in ftrace_get_addr_new()
2604 if (rec->flags & FTRACE_FL_REGS) in ftrace_get_addr_new()
2611 * ftrace_get_addr_curr - Get the call address that is already there
2612 * @rec: The ftrace record descriptor
2626 if (rec->flags & FTRACE_FL_DIRECT_EN) { in ftrace_get_addr_curr()
2627 addr = ftrace_find_rec_direct(rec->ip); in ftrace_get_addr_curr()
2634 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_get_addr_curr()
2638 (void *)rec->ip, (void *)rec->ip); in ftrace_get_addr_curr()
2639 /* Ftrace is shutting down, return anything */ in ftrace_get_addr_curr()
2642 return ops->trampoline; in ftrace_get_addr_curr()
2645 if (rec->flags & FTRACE_FL_REGS_EN) in ftrace_get_addr_curr()
2684 return -1; /* unknown ftrace bug */ in __ftrace_replace_code()
2720 * ftrace_rec_iter_start - start up iterating over traced functions
2737 iter->pg = ftrace_pages_start; in ftrace_rec_iter_start()
2738 iter->index = 0; in ftrace_rec_iter_start()
2741 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_start()
2742 iter->pg = iter->pg->next; in ftrace_rec_iter_start()
2744 if (!iter->pg) in ftrace_rec_iter_start()
2751 * ftrace_rec_iter_next - get the next record to process.
2758 iter->index++; in ftrace_rec_iter_next()
2760 if (iter->index >= iter->pg->index) { in ftrace_rec_iter_next()
2761 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2762 iter->index = 0; in ftrace_rec_iter_next()
2765 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_next()
2766 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2769 if (!iter->pg) in ftrace_rec_iter_next()
2776 * ftrace_rec_iter_record - get the record at the iterator location
2783 return &iter->pg->records[iter->index]; in ftrace_rec_iter_record()
2890 * ftrace_run_stop_machine - go back to the stop machine method
2891 * @command: The command to tell ftrace what to do
2902 * arch_ftrace_update_code - modify the code to trace or not trace
2931 ops->flags |= FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2932 ops->old_hash.filter_hash = old_hash->filter_hash; in ftrace_run_modify_code()
2933 ops->old_hash.notrace_hash = old_hash->notrace_hash; in ftrace_run_modify_code()
2935 ops->old_hash.filter_hash = NULL; in ftrace_run_modify_code()
2936 ops->old_hash.notrace_hash = NULL; in ftrace_run_modify_code()
2937 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2953 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list); in ftrace_add_trampoline_to_kallsyms()
2959 list_del_rcu(&ops->list); in ftrace_remove_trampoline_from_kallsyms()
2965 * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
2973 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) && in ftrace_trampoline_free()
2974 ops->trampoline) { in ftrace_trampoline_free()
2979 perf_event_text_poke((void *)ops->trampoline, in ftrace_trampoline_free()
2980 (void *)ops->trampoline, in ftrace_trampoline_free()
2981 ops->trampoline_size, NULL, 0); in ftrace_trampoline_free()
2983 ops->trampoline, ops->trampoline_size, in ftrace_trampoline_free()
3017 return -ENODEV; in ftrace_startup()
3026 * Note that ftrace probes uses this to start up in ftrace_startup()
3033 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; in ftrace_startup()
3039 ftrace_start_up--; in ftrace_startup()
3040 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_startup()
3041 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) in ftrace_startup()
3052 * If ftrace is in an undefined state, we just remove ops from list in ftrace_startup()
3058 return -ENODEV; in ftrace_startup()
3061 ops->flags &= ~FTRACE_OPS_FL_ADDING; in ftrace_startup()
3071 return -ENODEV; in ftrace_shutdown()
3077 ftrace_start_up--; in ftrace_shutdown()
3079 * Just warn in case of unbalance, no need to kill ftrace, it's not in ftrace_shutdown()
3081 * further ftrace uses. in ftrace_shutdown()
3091 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_shutdown()
3105 ops->flags |= FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3109 ops->old_hash.filter_hash = ops->func_hash->filter_hash; in ftrace_shutdown()
3110 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; in ftrace_shutdown()
3115 * If there's no more ops registered with ftrace, run a in ftrace_shutdown()
3124 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS)) in ftrace_shutdown()
3126 (void *)rec->ip, rec->flags); in ftrace_shutdown()
3130 ops->old_hash.filter_hash = NULL; in ftrace_shutdown()
3131 ops->old_hash.notrace_hash = NULL; in ftrace_shutdown()
3134 ops->flags &= ~FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3141 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) { in ftrace_shutdown()
3154 * while on a ftrace trampoline. Just scheduling a task on in ftrace_shutdown()
3179 return ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_traces_mod()
3180 ftrace_hash_empty(ops->func_hash->notrace_hash); in ops_traces_mod()
3198 * an entry in the ftrace data. Now, if ftrace is activated in ftrace_update_code()
3200 * read-only, the modification of enabling ftrace can fail if in ftrace_update_code()
3201 * the read-only is done while ftrace is converting the calls. in ftrace_update_code()
3204 * to read-only. in ftrace_update_code()
3209 for (pg = new_pgs; pg; pg = pg->next) { in ftrace_update_code()
3211 for (i = 0; i < pg->index; i++) { in ftrace_update_code()
3215 return -1; in ftrace_update_code()
3217 p = &pg->records[i]; in ftrace_update_code()
3218 p->flags = rec_flags; in ftrace_update_code()
3232 ftrace_update_time = stop - start; in ftrace_update_code()
3245 return -EINVAL; in ftrace_allocate_records()
3249 order = fls(pages) - 1; in ftrace_allocate_records()
3252 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); in ftrace_allocate_records()
3254 if (!pg->records) { in ftrace_allocate_records()
3255 /* if we can't allocate this size, try something smaller */ in ftrace_allocate_records()
3257 return -ENOMEM; in ftrace_allocate_records()
3258 order--; in ftrace_allocate_records()
3266 pg->order = order; in ftrace_allocate_records()
3279 if (pg->records) { in ftrace_free_pages()
3280 free_pages((unsigned long)pg->records, pg->order); in ftrace_free_pages()
3281 ftrace_number_of_pages -= 1 << pg->order; in ftrace_free_pages()
3283 pages = pg->next; in ftrace_free_pages()
3286 ftrace_number_of_groups--; in ftrace_free_pages()
3314 num_to_init -= cnt; in ftrace_allocate_pages()
3318 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3319 if (!pg->next) in ftrace_allocate_pages()
3322 pg = pg->next; in ftrace_allocate_pages()
3329 pr_info("ftrace: FAILED to allocate memory for functions\n"); in ftrace_allocate_pages()
3356 struct ftrace_iterator *iter = m->private; in t_probe_next()
3357 struct trace_array *tr = iter->ops->private; in t_probe_next()
3363 int size; in t_probe_next() local
3366 iter->pos = *pos; in t_probe_next()
3371 func_probes = &tr->func_probes; in t_probe_next()
3375 if (!iter->probe) { in t_probe_next()
3376 next = func_probes->next; in t_probe_next()
3377 iter->probe = list_entry(next, struct ftrace_func_probe, list); in t_probe_next()
3380 if (iter->probe_entry) in t_probe_next()
3381 hnd = &iter->probe_entry->hlist; in t_probe_next()
3383 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3392 size = 1 << hash->size_bits; in t_probe_next()
3395 if (iter->pidx >= size) { in t_probe_next()
3396 if (iter->probe->list.next == func_probes) in t_probe_next()
3398 next = iter->probe->list.next; in t_probe_next()
3399 iter->probe = list_entry(next, struct ftrace_func_probe, list); in t_probe_next()
3400 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3401 size = 1 << hash->size_bits; in t_probe_next()
3402 iter->pidx = 0; in t_probe_next()
3405 hhd = &hash->buckets[iter->pidx]; in t_probe_next()
3408 iter->pidx++; in t_probe_next()
3414 hnd = hhd->first; in t_probe_next()
3416 hnd = hnd->next; in t_probe_next()
3418 iter->pidx++; in t_probe_next()
3426 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist); in t_probe_next()
3433 struct ftrace_iterator *iter = m->private; in t_probe_start()
3437 if (!(iter->flags & FTRACE_ITER_DO_PROBES)) in t_probe_start()
3440 if (iter->mod_pos > *pos) in t_probe_start()
3443 iter->probe = NULL; in t_probe_start()
3444 iter->probe_entry = NULL; in t_probe_start()
3445 iter->pidx = 0; in t_probe_start()
3446 for (l = 0; l <= (*pos - iter->mod_pos); ) { in t_probe_start()
3455 iter->flags |= FTRACE_ITER_PROBE; in t_probe_start()
3467 probe = iter->probe; in t_probe_show()
3468 probe_entry = iter->probe_entry; in t_probe_show()
3471 return -EIO; in t_probe_show()
3473 probe_ops = probe->probe_ops; in t_probe_show()
3475 if (probe_ops->print) in t_probe_show()
3476 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); in t_probe_show()
3478 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, in t_probe_show()
3479 (void *)probe_ops->func); in t_probe_show()
3487 struct ftrace_iterator *iter = m->private; in t_mod_next()
3488 struct trace_array *tr = iter->tr; in t_mod_next()
3491 iter->pos = *pos; in t_mod_next()
3493 iter->mod_list = iter->mod_list->next; in t_mod_next()
3495 if (iter->mod_list == &tr->mod_trace || in t_mod_next()
3496 iter->mod_list == &tr->mod_notrace) { in t_mod_next()
3497 iter->flags &= ~FTRACE_ITER_MOD; in t_mod_next()
3501 iter->mod_pos = *pos; in t_mod_next()
3508 struct ftrace_iterator *iter = m->private; in t_mod_start()
3512 if (iter->func_pos > *pos) in t_mod_start()
3515 iter->mod_pos = iter->func_pos; in t_mod_start()
3518 if (!iter->tr) in t_mod_start()
3521 for (l = 0; l <= (*pos - iter->func_pos); ) { in t_mod_start()
3527 iter->flags &= ~FTRACE_ITER_MOD; in t_mod_start()
3532 iter->flags |= FTRACE_ITER_MOD; in t_mod_start()
3541 struct trace_array *tr = iter->tr; in t_mod_show()
3543 if (WARN_ON_ONCE(!iter->mod_list) || in t_mod_show()
3544 iter->mod_list == &tr->mod_trace || in t_mod_show()
3545 iter->mod_list == &tr->mod_notrace) in t_mod_show()
3546 return -EIO; in t_mod_show()
3548 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list); in t_mod_show()
3550 if (ftrace_mod->func) in t_mod_show()
3551 seq_printf(m, "%s", ftrace_mod->func); in t_mod_show()
3555 seq_printf(m, ":mod:%s\n", ftrace_mod->module); in t_mod_show()
3563 struct ftrace_iterator *iter = m->private; in t_func_next()
3569 if (iter->idx >= iter->pg->index) { in t_func_next()
3570 if (iter->pg->next) { in t_func_next()
3571 iter->pg = iter->pg->next; in t_func_next()
3572 iter->idx = 0; in t_func_next()
3576 rec = &iter->pg->records[iter->idx++]; in t_func_next()
3577 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && in t_func_next()
3578 !ftrace_lookup_ip(iter->hash, rec->ip)) || in t_func_next()
3580 ((iter->flags & FTRACE_ITER_ENABLED) && in t_func_next()
3581 !(rec->flags & FTRACE_FL_ENABLED)) || in t_func_next()
3583 ((iter->flags & FTRACE_ITER_TOUCHED) && in t_func_next()
3584 !(rec->flags & FTRACE_FL_TOUCHED))) { in t_func_next()
3594 iter->pos = iter->func_pos = *pos; in t_func_next()
3595 iter->func = rec; in t_func_next()
3603 struct ftrace_iterator *iter = m->private; in t_next()
3610 if (iter->flags & FTRACE_ITER_PROBE) in t_next()
3613 if (iter->flags & FTRACE_ITER_MOD) in t_next()
3616 if (iter->flags & FTRACE_ITER_PRINTALL) { in t_next()
3632 iter->pos = 0; in reset_iter_read()
3633 iter->func_pos = 0; in reset_iter_read()
3634 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD); in reset_iter_read()
3639 struct ftrace_iterator *iter = m->private; in t_start()
3651 if (*pos < iter->pos) in t_start()
3659 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && in t_start()
3660 ftrace_hash_empty(iter->hash)) { in t_start()
3661 iter->func_pos = 1; /* Account for the message */ in t_start()
3664 iter->flags |= FTRACE_ITER_PRINTALL; in t_start()
3666 iter->flags &= ~FTRACE_ITER_PROBE; in t_start()
3670 if (iter->flags & FTRACE_ITER_MOD) in t_start()
3678 iter->pg = ftrace_pages_start; in t_start()
3679 iter->idx = 0; in t_start()
3710 seq_printf(m, " ->%pS", ptr); in add_trampoline_func()
3730 ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str); in test_for_valid_rec()
3734 rec->flags |= FTRACE_FL_DISABLED; in test_for_valid_rec()
3797 return ret == NULL ? -1 : 0; in print_rec()
3814 struct ftrace_iterator *iter = m->private; in t_show()
3817 if (iter->flags & FTRACE_ITER_PROBE) in t_show()
3820 if (iter->flags & FTRACE_ITER_MOD) in t_show()
3823 if (iter->flags & FTRACE_ITER_PRINTALL) { in t_show()
3824 if (iter->flags & FTRACE_ITER_NOTRACE) in t_show()
3831 rec = iter->func; in t_show()
3836 if (iter->flags & FTRACE_ITER_ADDRS) in t_show()
3837 seq_printf(m, "%lx ", rec->ip); in t_show()
3839 if (print_rec(m, rec->ip)) { in t_show()
3841 WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED)); in t_show()
3846 if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) { in t_show()
3851 rec->flags & FTRACE_FL_REGS ? " R" : " ", in t_show()
3852 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ", in t_show()
3853 rec->flags & FTRACE_FL_DIRECT ? " D" : " ", in t_show()
3854 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ", in t_show()
3855 rec->flags & FTRACE_FL_MODIFIED ? " M " : " "); in t_show()
3856 if (rec->flags & FTRACE_FL_TRAMP_EN) { in t_show()
3861 (void *)ops->trampoline, in t_show()
3862 (void *)ops->func); in t_show()
3871 if (rec->flags & FTRACE_FL_CALL_OPS_EN) { in t_show()
3875 ops, ops->func); in t_show()
3880 if (rec->flags & FTRACE_FL_DIRECT) { in t_show()
3883 direct = ftrace_find_rec_direct(rec->ip); in t_show()
3885 seq_printf(m, "\n\tdirect-->%pS", (void *)direct); in t_show()
3912 return -ENODEV; in ftrace_avail_open()
3916 return -ENOMEM; in ftrace_avail_open()
3918 iter->pg = ftrace_pages_start; in ftrace_avail_open()
3919 iter->ops = &global_ops; in ftrace_avail_open()
3940 return -ENOMEM; in ftrace_enabled_open()
3942 iter->pg = ftrace_pages_start; in ftrace_enabled_open()
3943 iter->flags = FTRACE_ITER_ENABLED; in ftrace_enabled_open()
3944 iter->ops = &global_ops; in ftrace_enabled_open()
3965 return -ENOMEM; in ftrace_touched_open()
3967 iter->pg = ftrace_pages_start; in ftrace_touched_open()
3968 iter->flags = FTRACE_ITER_TOUCHED; in ftrace_touched_open()
3969 iter->ops = &global_ops; in ftrace_touched_open()
3985 return -ENODEV; in ftrace_avail_addrs_open()
3989 return -ENOMEM; in ftrace_avail_addrs_open()
3991 iter->pg = ftrace_pages_start; in ftrace_avail_addrs_open()
3992 iter->flags = FTRACE_ITER_ADDRS; in ftrace_avail_addrs_open()
3993 iter->ops = &global_ops; in ftrace_avail_addrs_open()
3999 * ftrace_regex_open - initialize function tracer filter files
4021 struct trace_array *tr = ops->private; in ftrace_regex_open()
4022 int ret = -ENOMEM; in ftrace_regex_open()
4027 return -ENODEV; in ftrace_regex_open()
4030 return -ENODEV; in ftrace_regex_open()
4036 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) in ftrace_regex_open()
4039 iter->ops = ops; in ftrace_regex_open()
4040 iter->flags = flag; in ftrace_regex_open()
4041 iter->tr = tr; in ftrace_regex_open()
4043 mutex_lock(&ops->func_hash->regex_lock); in ftrace_regex_open()
4046 hash = ops->func_hash->notrace_hash; in ftrace_regex_open()
4047 mod_head = tr ? &tr->mod_notrace : NULL; in ftrace_regex_open()
4049 hash = ops->func_hash->filter_hash; in ftrace_regex_open()
4050 mod_head = tr ? &tr->mod_trace : NULL; in ftrace_regex_open()
4053 iter->mod_list = mod_head; in ftrace_regex_open()
4055 if (file->f_mode & FMODE_WRITE) { in ftrace_regex_open()
4058 if (file->f_flags & O_TRUNC) { in ftrace_regex_open()
4059 iter->hash = alloc_ftrace_hash(size_bits); in ftrace_regex_open()
4062 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); in ftrace_regex_open()
4065 if (!iter->hash) { in ftrace_regex_open()
4066 trace_parser_put(&iter->parser); in ftrace_regex_open()
4070 iter->hash = hash; in ftrace_regex_open()
4074 if (file->f_mode & FMODE_READ) { in ftrace_regex_open()
4075 iter->pg = ftrace_pages_start; in ftrace_regex_open()
4079 struct seq_file *m = file->private_data; in ftrace_regex_open()
4080 m->private = iter; in ftrace_regex_open()
4083 free_ftrace_hash(iter->hash); in ftrace_regex_open()
4084 trace_parser_put(&iter->parser); in ftrace_regex_open()
4087 file->private_data = iter; in ftrace_regex_open()
4090 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_regex_open()
4105 struct ftrace_ops *ops = inode->i_private; in ftrace_filter_open()
4116 struct ftrace_ops *ops = inode->i_private; in ftrace_notrace_open()
4123 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
4131 * If symbols in an architecture don't correspond exactly to the user-visible
4145 str = arch_ftrace_match_adjust(str, g->search); in ftrace_match()
4147 switch (g->type) { in ftrace_match()
4149 if (strcmp(str, g->search) == 0) in ftrace_match()
4153 if (strncmp(str, g->search, g->len) == 0) in ftrace_match()
4157 if (strstr(str, g->search)) in ftrace_match()
4162 if (slen >= g->len && in ftrace_match()
4163 memcmp(str + slen - g->len, g->search, g->len) == 0) in ftrace_match()
4167 if (glob_match(g->search, str)) in ftrace_match()
4181 entry = ftrace_lookup_ip(hash, rec->ip); in enter_record()
4192 if (add_hash_entry(hash, rec->ip) == NULL) in enter_record()
4193 ret = -ENOMEM; in enter_record()
4202 long index = simple_strtoul(func_g->search, NULL, 0); in add_rec_by_index()
4207 if (--index < 0) in add_rec_by_index()
4211 if (pg->index <= index) { in add_rec_by_index()
4212 index -= pg->index; in add_rec_by_index()
4216 rec = &pg->records[index]; in add_rec_by_index()
4230 return -1; in lookup_ip()
4248 if (lookup_ip(rec->ip, &modname, str)) { in ftrace_match_record()
4251 !(rec->flags & FTRACE_FL_DISABLED)); in ftrace_match_record()
4259 if (!mod_g->len) { in ftrace_match_record()
4277 if (!func_g->len) in ftrace_match_record()
4321 if (rec->flags & FTRACE_FL_DISABLED) in match_records()
4354 if (ops->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4364 if (ops->func_hash != &global_ops.local_hash) in ftrace_ops_update_code()
4368 if (op->func_hash == &global_ops.local_hash && in ftrace_ops_update_code()
4369 op->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4387 old_hash_ops.filter_hash = ops->func_hash->filter_hash; in ftrace_hash_move_and_update_ops()
4388 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; in ftrace_hash_move_and_update_ops()
4407 if (n > sizeof(modname) - 1) in module_exists()
4418 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace; in cache_mod()
4426 ret = -EINVAL; in cache_mod()
4430 if (strcmp(ftrace_mod->module, module) != 0) in cache_mod()
4435 (ftrace_mod->func && in cache_mod()
4436 strcmp(ftrace_mod->func, func) == 0)) { in cache_mod()
4445 ret = -EINVAL; in cache_mod()
4471 mutex_lock(&ops->func_hash->regex_lock); in process_mod_list()
4474 orig_hash = &ops->func_hash->filter_hash; in process_mod_list()
4476 orig_hash = &ops->func_hash->notrace_hash; in process_mod_list()
4487 if (strcmp(ftrace_mod->module, mod) != 0) in process_mod_list()
4490 if (ftrace_mod->func) in process_mod_list()
4491 func = kstrdup(ftrace_mod->func, GFP_KERNEL); in process_mod_list()
4498 list_move(&ftrace_mod->list, &process_mods); in process_mod_list()
4501 kfree(ftrace_mod->func); in process_mod_list()
4502 ftrace_mod->func = func; in process_mod_list()
4509 func = ftrace_mod->func; in process_mod_list()
4517 new_hash->flags &= ~FTRACE_HASH_FL_MOD; in process_mod_list()
4526 mutex_unlock(&ops->func_hash->regex_lock); in process_mod_list()
4542 if (!list_empty(&tr->mod_trace)) in process_cached_mods()
4543 process_mod_list(&tr->mod_trace, tr->ops, mod, true); in process_cached_mods()
4544 if (!list_empty(&tr->mod_notrace)) in process_cached_mods()
4545 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); in process_cached_mods()
4568 return -ENOMEM; in ftrace_mod_callback()
4605 probe_ops = probe->probe_ops; in function_trace_probe_call()
4613 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); in function_trace_probe_call()
4627 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
4645 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
4661 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_find_ip()
4666 return &map->data; in ftrace_func_mapper_find_ip()
4670 * ftrace_func_mapper_add_ip - Map some data to an ip
4683 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_add_ip()
4685 return -EBUSY; in ftrace_func_mapper_add_ip()
4689 return -ENOMEM; in ftrace_func_mapper_add_ip()
4691 map->entry.ip = ip; in ftrace_func_mapper_add_ip()
4692 map->data = data; in ftrace_func_mapper_add_ip()
4694 __add_hash_entry(&mapper->hash, &map->entry); in ftrace_func_mapper_add_ip()
4700 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
4716 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_remove_ip()
4721 data = map->data; in ftrace_func_mapper_remove_ip()
4723 remove_hash_entry(&mapper->hash, entry); in ftrace_func_mapper_remove_ip()
4730 * free_ftrace_func_mapper - free a mapping of ips and data
4743 int size, i; in free_ftrace_func_mapper() local
4748 if (free_func && mapper->hash.count) { in free_ftrace_func_mapper()
4749 size = 1 << mapper->hash.size_bits; in free_ftrace_func_mapper()
4750 for (i = 0; i < size; i++) { in free_ftrace_func_mapper()
4751 hhd = &mapper->hash.buckets[i]; in free_ftrace_func_mapper()
4758 free_ftrace_hash(&mapper->hash); in free_ftrace_func_mapper()
4767 WARN_ON(probe->ref <= 0); in release_probe()
4770 probe->ref--; in release_probe()
4772 if (!probe->ref) { in release_probe()
4773 probe_ops = probe->probe_ops; in release_probe()
4776 * the probe->data itself in release_probe()
4778 if (probe_ops->free) in release_probe()
4779 probe_ops->free(probe_ops, probe->tr, 0, probe->data); in release_probe()
4780 list_del(&probe->list); in release_probe()
4792 probe->ref++; in acquire_probe_locked()
4806 int size; in register_ftrace_function_probe() local
4811 return -EINVAL; in register_ftrace_function_probe()
4815 return -EINVAL; in register_ftrace_function_probe()
4820 list_for_each_entry(iter, &tr->func_probes, list) { in register_ftrace_function_probe()
4821 if (iter->probe_ops == probe_ops) { in register_ftrace_function_probe()
4830 return -ENOMEM; in register_ftrace_function_probe()
4832 probe->probe_ops = probe_ops; in register_ftrace_function_probe()
4833 probe->ops.func = function_trace_probe_call; in register_ftrace_function_probe()
4834 probe->tr = tr; in register_ftrace_function_probe()
4835 ftrace_ops_init(&probe->ops); in register_ftrace_function_probe()
4836 list_add(&probe->list, &tr->func_probes); in register_ftrace_function_probe()
4844 * Note, there's a small window here that the func_hash->filter_hash in register_ftrace_function_probe()
4847 mutex_lock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
4849 orig_hash = &probe->ops.func_hash->filter_hash; in register_ftrace_function_probe()
4854 ret = -ENOMEM; in register_ftrace_function_probe()
4862 ret = -EINVAL; in register_ftrace_function_probe()
4867 size = 1 << hash->size_bits; in register_ftrace_function_probe()
4868 for (i = 0; i < size; i++) { in register_ftrace_function_probe()
4869 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_function_probe()
4870 if (ftrace_lookup_ip(old_hash, entry->ip)) in register_ftrace_function_probe()
4877 if (probe_ops->init) { in register_ftrace_function_probe()
4878 ret = probe_ops->init(probe_ops, tr, in register_ftrace_function_probe()
4879 entry->ip, data, in register_ftrace_function_probe()
4880 &probe->data); in register_ftrace_function_probe()
4882 if (probe_ops->free && count) in register_ftrace_function_probe()
4883 probe_ops->free(probe_ops, tr, in register_ftrace_function_probe()
4884 0, probe->data); in register_ftrace_function_probe()
4885 probe->data = NULL; in register_ftrace_function_probe()
4897 ret = -EINVAL; in register_ftrace_function_probe()
4901 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in register_ftrace_function_probe()
4907 probe->ref += count; in register_ftrace_function_probe()
4909 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) in register_ftrace_function_probe()
4910 ret = ftrace_startup(&probe->ops, 0); in register_ftrace_function_probe()
4918 mutex_unlock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
4926 if (!probe_ops->free || !count) in register_ftrace_function_probe()
4930 for (i = 0; i < size; i++) { in register_ftrace_function_probe()
4931 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_function_probe()
4932 if (ftrace_lookup_ip(old_hash, entry->ip)) in register_ftrace_function_probe()
4934 probe_ops->free(probe_ops, tr, entry->ip, probe->data); in register_ftrace_function_probe()
4955 int i, ret = -ENODEV; in unregister_ftrace_function_probe_func()
4956 int size; in unregister_ftrace_function_probe_func() local
4969 return -EINVAL; in unregister_ftrace_function_probe_func()
4974 list_for_each_entry(iter, &tr->func_probes, list) { in unregister_ftrace_function_probe_func()
4975 if (iter->probe_ops == probe_ops) { in unregister_ftrace_function_probe_func()
4983 ret = -EINVAL; in unregister_ftrace_function_probe_func()
4984 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) in unregister_ftrace_function_probe_func()
4991 mutex_lock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
4993 orig_hash = &probe->ops.func_hash->filter_hash; in unregister_ftrace_function_probe_func()
5003 ret = -ENOMEM; in unregister_ftrace_function_probe_func()
5010 size = 1 << hash->size_bits; in unregister_ftrace_function_probe_func()
5011 for (i = 0; i < size; i++) { in unregister_ftrace_function_probe_func()
5012 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { in unregister_ftrace_function_probe_func()
5015 kallsyms_lookup(entry->ip, NULL, NULL, in unregister_ftrace_function_probe_func()
5022 hlist_add_head(&entry->hlist, &hhd); in unregister_ftrace_function_probe_func()
5028 ret = -EINVAL; in unregister_ftrace_function_probe_func()
5034 WARN_ON(probe->ref < count); in unregister_ftrace_function_probe_func()
5036 probe->ref -= count; in unregister_ftrace_function_probe_func()
5039 ftrace_shutdown(&probe->ops, 0); in unregister_ftrace_function_probe_func()
5041 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in unregister_ftrace_function_probe_func()
5046 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, in unregister_ftrace_function_probe_func()
5051 hlist_del(&entry->hlist); in unregister_ftrace_function_probe_func()
5052 if (probe_ops->free) in unregister_ftrace_function_probe_func()
5053 probe_ops->free(probe_ops, tr, entry->ip, probe->data); in unregister_ftrace_function_probe_func()
5059 mutex_unlock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
5075 list_for_each_entry_safe(probe, n, &tr->func_probes, list) in clear_ftrace_function_probes()
5076 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops); in clear_ftrace_function_probes()
5083 * Currently we only register ftrace commands from __init, so mark this
5093 if (strcmp(cmd->name, p->name) == 0) { in register_ftrace_command()
5094 ret = -EBUSY; in register_ftrace_command()
5098 list_add(&cmd->list, &ftrace_commands); in register_ftrace_command()
5106 * Currently we only unregister ftrace commands from __init, so mark
5112 int ret = -ENODEV; in unregister_ftrace_command()
5116 if (strcmp(cmd->name, p->name) == 0) { in unregister_ftrace_command()
5118 list_del_init(&p->list); in unregister_ftrace_command()
5131 struct ftrace_hash *hash = iter->hash; in ftrace_process_regex()
5132 struct trace_array *tr = iter->ops->private; in ftrace_process_regex()
5135 int ret = -EINVAL; in ftrace_process_regex()
5142 ret = -EINVAL; in ftrace_process_regex()
5154 if (strcmp(p->name, command) == 0) { in ftrace_process_regex()
5155 ret = p->func(tr, hash, func, command, next, enable); in ftrace_process_regex()
5176 if (file->f_mode & FMODE_READ) { in ftrace_regex_write()
5177 struct seq_file *m = file->private_data; in ftrace_regex_write()
5178 iter = m->private; in ftrace_regex_write()
5180 iter = file->private_data; in ftrace_regex_write()
5183 return -ENODEV; in ftrace_regex_write()
5185 /* iter->hash is a local copy, so we don't need regex_lock */ in ftrace_regex_write()
5187 parser = &iter->parser; in ftrace_regex_write()
5192 ret = ftrace_process_regex(iter, parser->buffer, in ftrace_regex_write()
5193 parser->idx, enable); in ftrace_regex_write()
5225 return -EINVAL; in __ftrace_match_addr()
5230 return -ENOENT; in __ftrace_match_addr()
5236 return entry ? 0 : -ENOMEM; in __ftrace_match_addr()
5269 return -ENODEV; in ftrace_set_hash()
5271 mutex_lock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5274 orig_hash = &ops->func_hash->filter_hash; in ftrace_set_hash()
5276 orig_hash = &ops->func_hash->notrace_hash; in ftrace_set_hash()
5284 ret = -ENOMEM; in ftrace_set_hash()
5289 ret = -EINVAL; in ftrace_set_hash()
5303 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5342 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in check_direct_multi()
5343 return -EINVAL; in check_direct_multi()
5344 if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS) in check_direct_multi()
5345 return -EINVAL; in check_direct_multi()
5352 int size, i; in remove_direct_functions_hash() local
5354 size = 1 << hash->size_bits; in remove_direct_functions_hash()
5355 for (i = 0; i < size; i++) { in remove_direct_functions_hash()
5356 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in remove_direct_functions_hash()
5357 del = __ftrace_lookup_ip(direct_functions, entry->ip); in remove_direct_functions_hash()
5358 if (del && del->direct == addr) { in remove_direct_functions_hash()
5367 * register_ftrace_direct - Call a custom trampoline directly
5382 * -EINVAL - The @ops object was already registered with this call or
5384 * -EBUSY - Another direct function is already attached (there can be only one)
5385 * -ENODEV - @ip does not point to a ftrace nop location (or not supported)
5386 * -ENOMEM - There was an allocation failure.
5392 int err = -EBUSY, size, i; in register_ftrace_direct() local
5394 if (ops->func || ops->trampoline) in register_ftrace_direct()
5395 return -EINVAL; in register_ftrace_direct()
5396 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in register_ftrace_direct()
5397 return -EINVAL; in register_ftrace_direct()
5398 if (ops->flags & FTRACE_OPS_FL_ENABLED) in register_ftrace_direct()
5399 return -EINVAL; in register_ftrace_direct()
5401 hash = ops->func_hash->filter_hash; in register_ftrace_direct()
5403 return -EINVAL; in register_ftrace_direct()
5408 size = 1 << hash->size_bits; in register_ftrace_direct()
5409 for (i = 0; i < size; i++) { in register_ftrace_direct()
5410 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_direct()
5411 if (ftrace_find_rec_direct(entry->ip)) in register_ftrace_direct()
5416 err = -ENOMEM; in register_ftrace_direct()
5419 size = hash->count + direct_functions->count; in register_ftrace_direct()
5420 if (size > 32) in register_ftrace_direct()
5421 size = 32; in register_ftrace_direct()
5422 new_hash = alloc_ftrace_hash(fls(size)); in register_ftrace_direct()
5427 size = 1 << direct_functions->size_bits; in register_ftrace_direct()
5428 for (i = 0; i < size; i++) { in register_ftrace_direct()
5429 hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) { in register_ftrace_direct()
5430 new = add_hash_entry(new_hash, entry->ip); in register_ftrace_direct()
5433 new->direct = entry->direct; in register_ftrace_direct()
5438 size = 1 << hash->size_bits; in register_ftrace_direct()
5439 for (i = 0; i < size; i++) { in register_ftrace_direct()
5440 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_direct()
5441 new = add_hash_entry(new_hash, entry->ip); in register_ftrace_direct()
5445 new->direct = addr; in register_ftrace_direct()
5446 entry->direct = addr; in register_ftrace_direct()
5454 ops->func = call_direct_funcs; in register_ftrace_direct()
5455 ops->flags = MULTI_FLAGS; in register_ftrace_direct()
5456 ops->trampoline = FTRACE_REGS_ADDR; in register_ftrace_direct()
5457 ops->direct_call = addr; in register_ftrace_direct()
5477 * unregister_ftrace_direct - Remove calls to custom trampoline
5487 * -EINVAL - The @ops object was not properly registered.
5492 struct ftrace_hash *hash = ops->func_hash->filter_hash; in unregister_ftrace_direct()
5496 return -EINVAL; in unregister_ftrace_direct()
5497 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in unregister_ftrace_direct()
5498 return -EINVAL; in unregister_ftrace_direct()
5506 ops->func = NULL; in unregister_ftrace_direct()
5507 ops->trampoline = 0; in unregister_ftrace_direct()
5524 int i, size; in __modify_ftrace_direct() local
5531 tmp_ops.func_hash = ops->func_hash; in __modify_ftrace_direct()
5544 hash = ops->func_hash->filter_hash; in __modify_ftrace_direct()
5545 size = 1 << hash->size_bits; in __modify_ftrace_direct()
5546 for (i = 0; i < size; i++) { in __modify_ftrace_direct()
5547 hlist_for_each_entry(iter, &hash->buckets[i], hlist) { in __modify_ftrace_direct()
5548 entry = __ftrace_lookup_ip(direct_functions, iter->ip); in __modify_ftrace_direct()
5551 entry->direct = addr; in __modify_ftrace_direct()
5555 WRITE_ONCE(ops->direct_call, addr); in __modify_ftrace_direct()
5566 * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
5581 * -EINVAL - The @ops object was not properly registered.
5586 return -EINVAL; in modify_ftrace_direct_nolock()
5587 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct_nolock()
5588 return -EINVAL; in modify_ftrace_direct_nolock()
5595 * modify_ftrace_direct - Modify an existing direct 'multi' call
5607 * -EINVAL - The @ops object was not properly registered.
5614 return -EINVAL; in modify_ftrace_direct()
5615 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct()
5616 return -EINVAL; in modify_ftrace_direct()
5627 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
5628 * @ops - the ops to set the filter with
5629 * @ip - the address to add to or remove from the filter.
5630 * @remove - non zero to remove the ip from the filter
5631 * @reset - non zero to reset all filters before applying this filter.
5649 * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses
5650 * @ops - the ops to set the filter with
5651 * @ips - the array of addresses to add to or remove from the filter.
5652 * @cnt - the number of addresses in @ips
5653 * @remove - non zero to remove ips from the filter
5654 * @reset - non zero to reset all filters before applying this filter.
5672 * ftrace_ops_set_global_filter - setup ops to use global filters
5673 * @ops - the ops which will use the global filters
5675 * ftrace users who need global function trace filtering should call this.
5680 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) in ftrace_ops_set_global_filter()
5684 ops->func_hash = &global_ops.local_hash; in ftrace_ops_set_global_filter()
5696 * ftrace_set_filter - set a function to filter on in ftrace
5697 * @ops - the ops to set the filter with
5698 * @buf - the string that holds the function filter text.
5699 * @len - the length of the string.
5700 * @reset - non zero to reset all filters before applying this filter.
5718 * ftrace_set_notrace - set a function to not trace in ftrace
5719 * @ops - the ops to set the notrace filter with
5720 * @buf - the string that holds the function notrace text.
5721 * @len - the length of the string.
5722 * @reset - non zero to reset all filters before applying this filter.
5740 * ftrace_set_global_filter - set a function to filter on with global tracers
5741 * @buf - the string that holds the function filter text.
5742 * @len - the length of the string.
5743 * @reset - non zero to reset all filters before applying this filter.
5755 * ftrace_set_global_notrace - set a function to not trace with global tracers
5756 * @buf - the string that holds the function notrace text.
5757 * @len - the length of the string.
5758 * @reset - non zero to reset all filters before applying this filter.
5839 printk(KERN_DEBUG "ftrace: function %s not " in set_ftrace_early_graph()
5879 struct seq_file *m = (struct seq_file *)file->private_data; in ftrace_regex_release()
5885 if (file->f_mode & FMODE_READ) { in ftrace_regex_release()
5886 iter = m->private; in ftrace_regex_release()
5889 iter = file->private_data; in ftrace_regex_release()
5891 parser = &iter->parser; in ftrace_regex_release()
5893 int enable = !(iter->flags & FTRACE_ITER_NOTRACE); in ftrace_regex_release()
5895 ftrace_process_regex(iter, parser->buffer, in ftrace_regex_release()
5896 parser->idx, enable); in ftrace_regex_release()
5901 mutex_lock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
5903 if (file->f_mode & FMODE_WRITE) { in ftrace_regex_release()
5904 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); in ftrace_regex_release()
5907 orig_hash = &iter->ops->func_hash->filter_hash; in ftrace_regex_release()
5908 if (iter->tr) { in ftrace_regex_release()
5909 if (list_empty(&iter->tr->mod_trace)) in ftrace_regex_release()
5910 iter->hash->flags &= ~FTRACE_HASH_FL_MOD; in ftrace_regex_release()
5912 iter->hash->flags |= FTRACE_HASH_FL_MOD; in ftrace_regex_release()
5915 orig_hash = &iter->ops->func_hash->notrace_hash; in ftrace_regex_release()
5918 ftrace_hash_move_and_update_ops(iter->ops, orig_hash, in ftrace_regex_release()
5919 iter->hash, filter_hash); in ftrace_regex_release()
5923 iter->hash = NULL; in ftrace_regex_release()
5926 mutex_unlock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
5927 free_ftrace_hash(iter->hash); in ftrace_regex_release()
5928 if (iter->tr) in ftrace_regex_release()
5929 trace_array_put(iter->tr); in ftrace_regex_release()
6006 struct ftrace_graph_data *fgd = m->private; in __g_next()
6007 struct ftrace_func_entry *entry = fgd->entry; in __g_next()
6009 int i, idx = fgd->idx; in __g_next()
6011 if (*pos >= fgd->hash->count) in __g_next()
6016 fgd->entry = entry; in __g_next()
6023 for (i = idx; i < 1 << fgd->hash->size_bits; i++) { in __g_next()
6024 head = &fgd->hash->buckets[i]; in __g_next()
6026 fgd->entry = entry; in __g_next()
6027 fgd->idx = i; in __g_next()
6043 struct ftrace_graph_data *fgd = m->private; in g_start()
6047 if (fgd->type == GRAPH_FILTER_FUNCTION) in g_start()
6048 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, in g_start()
6051 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, in g_start()
6055 if (ftrace_hash_empty(fgd->hash) && !*pos) in g_start()
6058 fgd->idx = 0; in g_start()
6059 fgd->entry = NULL; in g_start()
6076 struct ftrace_graph_data *fgd = m->private; in g_show()
6078 if (fgd->type == GRAPH_FILTER_FUNCTION) in g_show()
6085 seq_printf(m, "%ps\n", (void *)entry->ip); in g_show()
6108 if (file->f_mode & FMODE_WRITE) { in __ftrace_graph_open()
6111 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX)) in __ftrace_graph_open()
6112 return -ENOMEM; in __ftrace_graph_open()
6114 if (file->f_flags & O_TRUNC) in __ftrace_graph_open()
6118 fgd->hash); in __ftrace_graph_open()
6120 ret = -ENOMEM; in __ftrace_graph_open()
6125 if (file->f_mode & FMODE_READ) { in __ftrace_graph_open()
6128 struct seq_file *m = file->private_data; in __ftrace_graph_open()
6129 m->private = fgd; in __ftrace_graph_open()
6136 file->private_data = fgd; in __ftrace_graph_open()
6139 if (ret < 0 && file->f_mode & FMODE_WRITE) in __ftrace_graph_open()
6140 trace_parser_put(&fgd->parser); in __ftrace_graph_open()
6142 fgd->new_hash = new_hash; in __ftrace_graph_open()
6145 * All uses of fgd->hash must be taken with the graph_lock in __ftrace_graph_open()
6147 * fgd->hash to be reinitialized when it is taken again. in __ftrace_graph_open()
6149 fgd->hash = NULL; in __ftrace_graph_open()
6161 return -ENODEV; in ftrace_graph_open()
6165 return -ENOMEM; in ftrace_graph_open()
6169 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, in ftrace_graph_open()
6171 fgd->type = GRAPH_FILTER_FUNCTION; in ftrace_graph_open()
6172 fgd->seq_ops = &ftrace_graph_seq_ops; in ftrace_graph_open()
6189 return -ENODEV; in ftrace_graph_notrace_open()
6193 return -ENOMEM; in ftrace_graph_notrace_open()
6197 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, in ftrace_graph_notrace_open()
6199 fgd->type = GRAPH_FILTER_NOTRACE; in ftrace_graph_notrace_open()
6200 fgd->seq_ops = &ftrace_graph_seq_ops; in ftrace_graph_notrace_open()
6218 if (file->f_mode & FMODE_READ) { in ftrace_graph_release()
6219 struct seq_file *m = file->private_data; in ftrace_graph_release()
6221 fgd = m->private; in ftrace_graph_release()
6224 fgd = file->private_data; in ftrace_graph_release()
6228 if (file->f_mode & FMODE_WRITE) { in ftrace_graph_release()
6230 parser = &fgd->parser; in ftrace_graph_release()
6233 ret = ftrace_graph_set_hash(fgd->new_hash, in ftrace_graph_release()
6234 parser->buffer); in ftrace_graph_release()
6239 new_hash = __ftrace_hash_move(fgd->new_hash); in ftrace_graph_release()
6241 ret = -ENOMEM; in ftrace_graph_release()
6247 if (fgd->type == GRAPH_FILTER_FUNCTION) { in ftrace_graph_release()
6274 free_ftrace_hash(fgd->new_hash); in ftrace_graph_release()
6300 return -ENODEV; in ftrace_graph_set_hash()
6305 if (rec->flags & FTRACE_FL_DISABLED) in ftrace_graph_set_hash()
6309 entry = ftrace_lookup_ip(hash, rec->ip); in ftrace_graph_set_hash()
6316 if (add_hash_entry(hash, rec->ip) == NULL) in ftrace_graph_set_hash()
6330 return -EINVAL; in ftrace_graph_set_hash()
6340 struct ftrace_graph_data *fgd = file->private_data; in ftrace_graph_write()
6347 if (file->f_mode & FMODE_READ) { in ftrace_graph_write()
6348 struct seq_file *m = file->private_data; in ftrace_graph_write()
6349 fgd = m->private; in ftrace_graph_write()
6352 parser = &fgd->parser; in ftrace_graph_write()
6359 ret = ftrace_graph_set_hash(fgd->new_hash, in ftrace_graph_write()
6360 parser->buffer); in ftrace_graph_write()
6411 if (ops->flags & FTRACE_OPS_FL_ENABLED) in ftrace_destroy_filter_files()
6413 ops->flags |= FTRACE_OPS_FL_DELETED; in ftrace_destroy_filter_files()
6455 return -1; in ftrace_cmp_ips()
6465 if (WARN(start[i - 1] > start[i], in test_is_sorted()
6467 (void *)start[i - 1], start[i - 1], in test_is_sorted()
6472 pr_info("ftrace section at %px sorted properly\n", start); in test_is_sorted()
6493 int ret = -ENOMEM; in ftrace_process_locs()
6495 count = end - start; in ftrace_process_locs()
6514 return -ENOMEM; in ftrace_process_locs()
6531 if (WARN_ON(ftrace_pages->next)) { in ftrace_process_locs()
6533 while (ftrace_pages->next) in ftrace_process_locs()
6534 ftrace_pages = ftrace_pages->next; in ftrace_process_locs()
6537 ftrace_pages->next = start_pg; in ftrace_process_locs()
6556 end_offset = (pg->index+1) * sizeof(pg->records[0]); in ftrace_process_locs()
6557 if (end_offset > PAGE_SIZE << pg->order) { in ftrace_process_locs()
6559 if (WARN_ON(!pg->next)) in ftrace_process_locs()
6561 pg = pg->next; in ftrace_process_locs()
6564 rec = &pg->records[pg->index++]; in ftrace_process_locs()
6565 rec->ip = addr; in ftrace_process_locs()
6568 if (pg->next) { in ftrace_process_locs()
6569 pg_unuse = pg->next; in ftrace_process_locs()
6570 pg->next = NULL; in ftrace_process_locs()
6605 unsigned int size; member
6626 if (!op->trampoline || symnum--) in ftrace_get_trampoline_kallsym()
6628 *value = op->trampoline; in ftrace_get_trampoline_kallsym()
6636 return -ERANGE; in ftrace_get_trampoline_kallsym()
6651 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in ops_references_ip()
6659 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_references_ip()
6660 !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) in ops_references_ip()
6664 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) in ops_references_ip()
6682 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { in referenced_filters()
6683 if (ops_references_ip(ops, rec->ip)) { in referenced_filters()
6684 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) in referenced_filters()
6686 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in referenced_filters()
6689 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in referenced_filters()
6690 rec->flags |= FTRACE_FL_REGS; in referenced_filters()
6691 if (cnt == 1 && ops->trampoline) in referenced_filters()
6692 rec->flags |= FTRACE_FL_TRAMP; in referenced_filters()
6694 rec->flags &= ~FTRACE_FL_TRAMP; in referenced_filters()
6711 for (i = 0; i < pg->index; i++) { in clear_mod_from_hash()
6712 rec = &pg->records[i]; in clear_mod_from_hash()
6713 entry = __ftrace_lookup_ip(hash, rec->ip); in clear_mod_from_hash()
6720 entry->ip = 0; in clear_mod_from_hash()
6731 if (!tr->ops || !tr->ops->func_hash) in clear_mod_from_hashes()
6733 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
6734 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); in clear_mod_from_hashes()
6735 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); in clear_mod_from_hashes()
6736 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
6748 list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) { in ftrace_free_mod_map()
6749 kfree(mod_func->name); in ftrace_free_mod_map()
6750 list_del(&mod_func->list); in ftrace_free_mod_map()
6772 if (mod_map->mod == mod) { in ftrace_release_mod()
6773 list_del_rcu(&mod_map->list); in ftrace_release_mod()
6774 call_rcu(&mod_map->rcu, ftrace_free_mod_map); in ftrace_release_mod()
6785 rec = &pg->records[0]; in ftrace_release_mod()
6786 if (within_module(rec->ip, mod)) { in ftrace_release_mod()
6798 ftrace_update_tot_cnt -= pg->index; in ftrace_release_mod()
6799 *last_pg = pg->next; in ftrace_release_mod()
6801 pg->next = tmp_page; in ftrace_release_mod()
6804 last_pg = &pg->next; in ftrace_release_mod()
6814 if (pg->records) { in ftrace_release_mod()
6815 free_pages((unsigned long)pg->records, pg->order); in ftrace_release_mod()
6816 ftrace_number_of_pages -= 1 << pg->order; in ftrace_release_mod()
6818 tmp_page = pg->next; in ftrace_release_mod()
6820 ftrace_number_of_groups--; in ftrace_release_mod()
6844 * text to read-only, as we now need to set it back to read-write in ftrace_module_enable()
6858 if (!within_module(rec->ip, mod)) in ftrace_module_enable()
6864 rec->flags = FTRACE_FL_DISABLED; in ftrace_module_enable()
6879 rec->flags &= ~FTRACE_FL_DISABLED; in ftrace_module_enable()
6880 rec->flags += cnt; in ftrace_module_enable()
6899 process_cached_mods(mod->name); in ftrace_module_enable()
6906 if (ftrace_disabled || !mod->num_ftrace_callsites) in ftrace_module_init()
6909 ret = ftrace_process_locs(mod, mod->ftrace_callsites, in ftrace_module_init()
6910 mod->ftrace_callsites + mod->num_ftrace_callsites); in ftrace_module_init()
6912 pr_warn("ftrace: failed to allocate entries for module '%s' functions\n", in ftrace_module_init()
6913 mod->name); in ftrace_module_init()
6926 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str); in save_ftrace_mod_rec()
6934 mod_func->name = kstrdup(str, GFP_KERNEL); in save_ftrace_mod_rec()
6935 if (!mod_func->name) { in save_ftrace_mod_rec()
6940 mod_func->ip = rec->ip - offset; in save_ftrace_mod_rec()
6941 mod_func->size = symsize; in save_ftrace_mod_rec()
6943 mod_map->num_funcs++; in save_ftrace_mod_rec()
6945 list_add_rcu(&mod_func->list, &mod_map->funcs); in save_ftrace_mod_rec()
6958 mod_map->mod = mod; in allocate_ftrace_mod_map()
6959 mod_map->start_addr = start; in allocate_ftrace_mod_map()
6960 mod_map->end_addr = end; in allocate_ftrace_mod_map()
6961 mod_map->num_funcs = 0; in allocate_ftrace_mod_map()
6963 INIT_LIST_HEAD_RCU(&mod_map->funcs); in allocate_ftrace_mod_map()
6965 list_add_rcu(&mod_map->list, &ftrace_mod_maps); in allocate_ftrace_mod_map()
6972 unsigned long addr, unsigned long *size, in ftrace_func_address_lookup() argument
6978 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { in ftrace_func_address_lookup()
6979 if (addr >= mod_func->ip && in ftrace_func_address_lookup()
6980 addr < mod_func->ip + mod_func->size) { in ftrace_func_address_lookup()
6987 if (size) in ftrace_func_address_lookup()
6988 *size = found_func->size; in ftrace_func_address_lookup()
6990 *off = addr - found_func->ip; in ftrace_func_address_lookup()
6992 strscpy(sym, found_func->name, KSYM_NAME_LEN); in ftrace_func_address_lookup()
6994 return found_func->name; in ftrace_func_address_lookup()
7001 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, in ftrace_mod_address_lookup() argument
7010 ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym); in ftrace_mod_address_lookup()
7013 *modname = mod_map->mod->name; in ftrace_mod_address_lookup()
7033 if (symnum >= mod_map->num_funcs) { in ftrace_mod_get_kallsym()
7034 symnum -= mod_map->num_funcs; in ftrace_mod_get_kallsym()
7038 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { in ftrace_mod_get_kallsym()
7040 symnum--; in ftrace_mod_get_kallsym()
7044 *value = mod_func->ip; in ftrace_mod_get_kallsym()
7046 strscpy(name, mod_func->name, KSYM_NAME_LEN); in ftrace_mod_get_kallsym()
7047 strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN); in ftrace_mod_get_kallsym()
7095 entry = ftrace_lookup_ip(hash, func->ip); in clear_func_from_hash()
7102 entry->ip = 0; in clear_func_from_hash()
7112 if (!tr->ops || !tr->ops->func_hash) in clear_func_from_hashes()
7114 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7115 clear_func_from_hash(func, tr->ops->func_hash->filter_hash); in clear_func_from_hashes()
7116 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash); in clear_func_from_hashes()
7117 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7129 MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n"); in add_to_clear_hash_list()
7133 func->ip = rec->ip; in add_to_clear_hash_list()
7134 list_add(&func->list, clear_list); in add_to_clear_hash_list()
7162 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { in ftrace_free_mem()
7163 if (end < pg->records[0].ip || in ftrace_free_mem()
7164 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in ftrace_free_mem()
7167 rec = bsearch(&key, pg->records, pg->index, in ftrace_free_mem()
7179 pg->index--; in ftrace_free_mem()
7180 ftrace_update_tot_cnt--; in ftrace_free_mem()
7181 if (!pg->index) { in ftrace_free_mem()
7182 *last_pg = pg->next; in ftrace_free_mem()
7183 if (pg->records) { in ftrace_free_mem()
7184 free_pages((unsigned long)pg->records, pg->order); in ftrace_free_mem()
7185 ftrace_number_of_pages -= 1 << pg->order; in ftrace_free_mem()
7187 ftrace_number_of_groups--; in ftrace_free_mem()
7195 (pg->index - (rec - pg->records)) * sizeof(*rec)); in ftrace_free_mem()
7235 count = __stop_mcount_loc - __start_mcount_loc; in ftrace_init()
7237 pr_info("ftrace: No functions to be traced?\n"); in ftrace_init()
7241 pr_info("ftrace: allocating %ld entries in %ld pages\n", in ftrace_init()
7248 pr_warn("ftrace: failed to allocate entries for functions\n"); in ftrace_init()
7252 pr_info("ftrace: allocated %ld pages with %ld groups\n", in ftrace_init()
7271 unsigned long trampoline = ops->trampoline; in ftrace_update_trampoline()
7274 if (ops->trampoline && ops->trampoline != trampoline && in ftrace_update_trampoline()
7275 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) { in ftrace_update_trampoline()
7279 ops->trampoline, ops->trampoline_size, false, in ftrace_update_trampoline()
7285 perf_event_text_poke((void *)ops->trampoline, NULL, 0, in ftrace_update_trampoline()
7286 (void *)ops->trampoline, in ftrace_update_trampoline()
7287 ops->trampoline_size); in ftrace_update_trampoline()
7293 INIT_LIST_HEAD(&tr->func_probes); in ftrace_init_trace_array()
7294 INIT_LIST_HEAD(&tr->mod_trace); in ftrace_init_trace_array()
7295 INIT_LIST_HEAD(&tr->mod_notrace); in ftrace_init_trace_array()
7323 tr->ops = &global_ops; in ftrace_init_global_array_ops()
7324 tr->ops->private = tr; in ftrace_init_global_array_ops()
7331 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { in ftrace_init_array_ops()
7332 if (WARN_ON(tr->ops->func != ftrace_stub)) in ftrace_init_array_ops()
7333 printk("ftrace ops had %pS for function\n", in ftrace_init_array_ops()
7334 tr->ops->func); in ftrace_init_array_ops()
7336 tr->ops->func = func; in ftrace_init_array_ops()
7337 tr->ops->private = tr; in ftrace_init_array_ops()
7342 tr->ops->func = ftrace_stub; in ftrace_reset_array_ops()
7364 if (op->flags & FTRACE_OPS_FL_STUB) in __ftrace_ops_list_func()
7371 * If any of the above fails then the op->func() is not executed. in __ftrace_ops_list_func()
7373 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && in __ftrace_ops_list_func()
7375 if (FTRACE_WARN_ON(!op->func)) { in __ftrace_ops_list_func()
7379 op->func(ip, parent_ip, op, fregs); in __ftrace_ops_list_func()
7430 if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) in ftrace_ops_assist_func()
7431 op->func(ip, parent_ip, op, fregs); in ftrace_ops_assist_func()
7438 * ftrace_ops_get_func - get the function a trampoline should call
7441 * Normally the mcount trampoline will call the ops->func, but there
7454 if (ops->flags & (FTRACE_OPS_FL_RECURSION | in ftrace_ops_get_func()
7458 return ops->func; in ftrace_ops_get_func()
7471 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_filter_pid_sched_switch_probe()
7472 no_pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_filter_pid_sched_switch_probe()
7475 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ftrace_filter_pid_sched_switch_probe()
7478 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ftrace_filter_pid_sched_switch_probe()
7479 next->pid); in ftrace_filter_pid_sched_switch_probe()
7490 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_pid_follow_sched_process_fork()
7493 pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_pid_follow_sched_process_fork()
7503 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_pid_follow_sched_process_exit()
7506 pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_pid_follow_sched_process_exit()
7531 pid_list = rcu_dereference_protected(tr->function_pids, in clear_ftrace_pids()
7533 no_pid_list = rcu_dereference_protected(tr->function_no_pids, in clear_ftrace_pids()
7544 per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE; in clear_ftrace_pids()
7548 rcu_assign_pointer(tr->function_pids, NULL); in clear_ftrace_pids()
7551 rcu_assign_pointer(tr->function_no_pids, NULL); in clear_ftrace_pids()
7590 struct trace_array *tr = m->private; in fpid_start()
7595 pid_list = rcu_dereference_sched(tr->function_pids); in fpid_start()
7605 struct trace_array *tr = m->private; in fpid_next()
7606 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); in fpid_next()
7643 struct trace_array *tr = m->private; in fnpid_start()
7648 pid_list = rcu_dereference_sched(tr->function_no_pids); in fnpid_start()
7658 struct trace_array *tr = m->private; in fnpid_next()
7659 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids); in fnpid_next()
7678 struct trace_array *tr = inode->i_private; in pid_open()
7686 if ((file->f_mode & FMODE_WRITE) && in pid_open()
7687 (file->f_flags & O_TRUNC)) in pid_open()
7700 return -EINVAL; in pid_open()
7707 m = file->private_data; in pid_open()
7709 m->private = tr; in pid_open()
7737 pid_list = rcu_dereference_protected(tr->function_pids, in ignore_task_cpu()
7739 no_pid_list = rcu_dereference_protected(tr->function_no_pids, in ignore_task_cpu()
7743 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ignore_task_cpu()
7746 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ignore_task_cpu()
7747 current->pid); in ignore_task_cpu()
7754 struct seq_file *m = filp->private_data; in pid_write()
7755 struct trace_array *tr = m->private; in pid_write()
7768 filtered_pids = rcu_dereference_protected(tr->function_pids, in pid_write()
7770 other_pids = rcu_dereference_protected(tr->function_no_pids, in pid_write()
7774 filtered_pids = rcu_dereference_protected(tr->function_no_pids, in pid_write()
7776 other_pids = rcu_dereference_protected(tr->function_pids, in pid_write()
7780 ret = -EINVAL; in pid_write()
7791 rcu_assign_pointer(tr->function_pids, pid_list); in pid_write()
7794 rcu_assign_pointer(tr->function_no_pids, pid_list); in pid_write()
7842 struct trace_array *tr = inode->i_private; in ftrace_pid_release()
7877 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); in ftrace_init_tracefs_toplevel()
7884 * ftrace_kill - kill ftrace
7886 * This function should be used by panic code. It stops ftrace
7887 * but in a not so nice way. If you need to simply kill ftrace
7888 * from a non-atomic section, use ftrace_kill.
7898 * ftrace_is_dead - Test if ftrace is dead or not.
7900 * Returns 1 if ftrace is "dead", zero otherwise.
7925 int size, i, ret; in prepare_direct_functions_for_ipmodify() local
7929 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in prepare_direct_functions_for_ipmodify()
7932 hash = ops->func_hash->filter_hash; in prepare_direct_functions_for_ipmodify()
7933 size = 1 << hash->size_bits; in prepare_direct_functions_for_ipmodify()
7934 for (i = 0; i < size; i++) { in prepare_direct_functions_for_ipmodify()
7935 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in prepare_direct_functions_for_ipmodify()
7936 unsigned long ip = entry->ip; in prepare_direct_functions_for_ipmodify()
7941 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) in prepare_direct_functions_for_ipmodify()
7951 if (!op->ops_func) in prepare_direct_functions_for_ipmodify()
7952 return -EBUSY; in prepare_direct_functions_for_ipmodify()
7954 ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); in prepare_direct_functions_for_ipmodify()
7974 int size, i; in cleanup_direct_functions_after_ipmodify() local
7976 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in cleanup_direct_functions_after_ipmodify()
7981 hash = ops->func_hash->filter_hash; in cleanup_direct_functions_after_ipmodify()
7982 size = 1 << hash->size_bits; in cleanup_direct_functions_after_ipmodify()
7983 for (i = 0; i < size; i++) { in cleanup_direct_functions_after_ipmodify()
7984 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in cleanup_direct_functions_after_ipmodify()
7985 unsigned long ip = entry->ip; in cleanup_direct_functions_after_ipmodify()
7990 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) in cleanup_direct_functions_after_ipmodify()
8000 if (found_op && op->ops_func) in cleanup_direct_functions_after_ipmodify()
8001 op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); in cleanup_direct_functions_after_ipmodify()
8045 * register_ftrace_function - register a function for profiling
8051 * Note: @ops->func and all the functions it calls must be labeled
8073 * unregister_ftrace_function - unregister a function for profiling.
8076 * Unregister a function that was added to be called by ftrace profiling.
8116 sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp); in kallsyms_callback()
8120 idx = sym - args->syms; in kallsyms_callback()
8121 if (args->addrs[idx]) in kallsyms_callback()
8127 args->addrs[idx] = addr; in kallsyms_callback()
8128 args->found++; in kallsyms_callback()
8129 return args->found == args->cnt ? 1 : 0; in kallsyms_callback()
8133 * ftrace_lookup_symbols - Lookup addresses for array of symbols
8146 * -ESRCH otherwise.
8163 return found_all ? 0 : -ESRCH; in ftrace_lookup_symbols()
8178 /* ftrace_start_up is true if we want ftrace running */ in ftrace_startup_sysctl()
8194 /* ftrace_start_up is true if ftrace is running */ in ftrace_shutdown_sysctl()
8212 if (op->flags & FTRACE_OPS_FL_PERMANENT) in is_permanent_ops_registered()
8223 int ret = -ENODEV; in ftrace_enable_sysctl()
8237 /* we are starting ftrace again */ in ftrace_enable_sysctl()
8247 ret = -EBUSY; in ftrace_enable_sysctl()
8251 /* stopping ftrace calls (just send to ftrace_stub) */ in ftrace_enable_sysctl()