Lines Matching +full:ftrace +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0
3 * Infrastructure for profiling code inserted by 'gcc -pg'.
5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
8 * Originally ported from the -rt patch by:
13 * Copyright (C) 2004-2006 Ingo Molnar
29 #include <linux/ftrace.h>
94 /* ftrace_enabled is a method to turn ftrace on or off */
107 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) in ftrace_pids_enabled()
110 tr = ops->private; in ftrace_pids_enabled()
112 return tr->function_pids != NULL || tr->function_no_pids != NULL; in ftrace_pids_enabled()
163 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { in ftrace_ops_init()
164 mutex_init(&ops->local_hash.regex_lock); in ftrace_ops_init()
165 INIT_LIST_HEAD(&ops->subop_list); in ftrace_ops_init()
166 ops->func_hash = &ops->local_hash; in ftrace_ops_init()
167 ops->flags |= FTRACE_OPS_FL_INITIALIZED; in ftrace_ops_init()
176 struct trace_array *tr = op->private; in ftrace_pid_func()
180 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid); in ftrace_pid_func()
184 pid != current->pid) in ftrace_pid_func()
188 op->saved_func(ip, parent_ip, op, fregs); in ftrace_pid_func()
203 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) || in ftrace_ops_get_list_func()
231 } else if (rcu_dereference_protected(ftrace_ops_list->next, in update_ftrace_function()
290 rcu_assign_pointer(ops->next, *list); in add_ftrace_ops()
295 * the ops->next pointer is valid before another CPU sees in add_ftrace_ops()
312 rcu_dereference_protected(ops->next, in remove_ftrace_ops()
318 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) in remove_ftrace_ops()
323 return -1; in remove_ftrace_ops()
325 *p = (*p)->next; in remove_ftrace_ops()
333 if (ops->flags & FTRACE_OPS_FL_DELETED) in __register_ftrace_function()
334 return -EINVAL; in __register_ftrace_function()
336 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) in __register_ftrace_function()
337 return -EBUSY; in __register_ftrace_function()
345 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && in __register_ftrace_function()
346 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) in __register_ftrace_function()
347 return -EINVAL; in __register_ftrace_function()
349 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) in __register_ftrace_function()
350 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; in __register_ftrace_function()
352 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT)) in __register_ftrace_function()
353 return -EBUSY; in __register_ftrace_function()
356 ops->flags |= FTRACE_OPS_FL_DYNAMIC; in __register_ftrace_function()
361 ops->saved_func = ops->func; in __register_ftrace_function()
364 ops->func = ftrace_pid_func; in __register_ftrace_function()
378 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) in __unregister_ftrace_function()
379 return -EBUSY; in __unregister_ftrace_function()
389 ops->func = ops->saved_func; in __unregister_ftrace_function()
403 if (op->flags & FTRACE_OPS_FL_PID) { in ftrace_update_pid_func()
404 op->func = ftrace_pids_enabled(op) ? in ftrace_update_pid_func()
405 ftrace_pid_func : op->saved_func; in ftrace_update_pid_func()
441 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
448 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
468 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
469 pg = pg->next; in function_stat_next()
472 rec = &pg->records[0]; in function_stat_next()
473 if (!rec->counter) in function_stat_next()
485 if (!stat || !stat->start) in function_stat_start()
488 return function_stat_next(&stat->start->records[0], 0); in function_stat_start()
498 if (a->time < b->time) in function_stat_cmp()
499 return -1; in function_stat_cmp()
500 if (a->time > b->time) in function_stat_cmp()
512 if (a->counter < b->counter) in function_stat_cmp()
513 return -1; in function_stat_cmp()
514 if (a->counter > b->counter) in function_stat_cmp()
526 " -------- " in function_stat_headers()
527 "--- ---- --- ---\n"); in function_stat_headers()
530 " -------- ---\n"); in function_stat_headers()
548 if (unlikely(rec->counter == 0)) in function_stat_show()
549 return -EBUSY; in function_stat_show()
552 avg = div64_ul(rec->time, rec->counter); in function_stat_show()
557 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); in function_stat_show()
558 seq_printf(m, " %-30.30s %10lu", str, rec->counter); in function_stat_show()
565 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) in function_stat_show()
567 * Divide only by 1000 for ns^2 -> us^2 conversion. in function_stat_show()
571 stddev_denom = rec->counter * (rec->counter - 1) * 1000; in function_stat_show()
573 stddev = rec->counter * rec->time_squared - in function_stat_show()
574 rec->time * rec->time; in function_stat_show()
579 trace_print_graph_duration(rec->time, &s); in function_stat_show()
595 pg = stat->pages = stat->start; in ftrace_profile_reset()
598 memset(pg->records, 0, PROFILE_RECORDS_SIZE); in ftrace_profile_reset()
599 pg->index = 0; in ftrace_profile_reset()
600 pg = pg->next; in ftrace_profile_reset()
603 memset(stat->hash, 0, in ftrace_profile_reset()
615 if (stat->pages) in ftrace_profile_pages_init()
618 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
619 if (!stat->pages) in ftrace_profile_pages_init()
620 return -ENOMEM; in ftrace_profile_pages_init()
635 pg = stat->start = stat->pages; in ftrace_profile_pages_init()
640 pg->next = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
641 if (!pg->next) in ftrace_profile_pages_init()
643 pg = pg->next; in ftrace_profile_pages_init()
649 pg = stat->start; in ftrace_profile_pages_init()
653 pg = pg->next; in ftrace_profile_pages_init()
657 stat->pages = NULL; in ftrace_profile_pages_init()
658 stat->start = NULL; in ftrace_profile_pages_init()
660 return -ENOMEM; in ftrace_profile_pages_init()
666 int size; in ftrace_profile_init_cpu() local
670 if (stat->hash) { in ftrace_profile_init_cpu()
680 size = FTRACE_PROFILE_HASH_SIZE; in ftrace_profile_init_cpu()
682 stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL); in ftrace_profile_init_cpu()
684 if (!stat->hash) in ftrace_profile_init_cpu()
685 return -ENOMEM; in ftrace_profile_init_cpu()
689 kfree(stat->hash); in ftrace_profile_init_cpu()
690 stat->hash = NULL; in ftrace_profile_init_cpu()
691 return -ENOMEM; in ftrace_profile_init_cpu()
720 hhd = &stat->hash[key]; in ftrace_find_profiled_func()
726 if (rec->ip == ip) in ftrace_find_profiled_func()
738 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); in ftrace_add_profile()
739 hlist_add_head_rcu(&rec->node, &stat->hash[key]); in ftrace_add_profile()
751 if (atomic_inc_return(&stat->disabled) != 1) in ftrace_profile_alloc()
762 if (stat->pages->index == PROFILES_PER_PAGE) { in ftrace_profile_alloc()
763 if (!stat->pages->next) in ftrace_profile_alloc()
765 stat->pages = stat->pages->next; in ftrace_profile_alloc()
768 rec = &stat->pages->records[stat->pages->index++]; in ftrace_profile_alloc()
769 rec->ip = ip; in ftrace_profile_alloc()
773 atomic_dec(&stat->disabled); in ftrace_profile_alloc()
791 if (!stat->hash || !ftrace_profile_enabled) in function_profile_call()
801 rec->counter++; in function_profile_call()
824 function_profile_call(trace->func, 0, NULL, NULL); in profile_graph_entry()
827 if (!current->ret_stack) in profile_graph_entry()
830 profile_data = fgraph_reserve_data(gops->idx, sizeof(*profile_data)); in profile_graph_entry()
834 profile_data->subtime = 0; in profile_graph_entry()
835 profile_data->sleeptime = current->ftrace_sleeptime; in profile_graph_entry()
836 profile_data->calltime = trace_clock_local(); in profile_graph_entry()
850 int size; in profile_graph_return() local
855 if (!stat->hash || !ftrace_profile_enabled) in profile_graph_return()
858 profile_data = fgraph_retrieve_data(gops->idx, &size); in profile_graph_return()
861 if (!profile_data || !profile_data->calltime) in profile_graph_return()
864 calltime = rettime - profile_data->calltime; in profile_graph_return()
867 if (current->ftrace_sleeptime) in profile_graph_return()
868 calltime -= current->ftrace_sleeptime - profile_data->sleeptime; in profile_graph_return()
875 parent_data = fgraph_retrieve_parent_data(gops->idx, &size, 1); in profile_graph_return()
877 parent_data->subtime += calltime; in profile_graph_return()
879 if (profile_data->subtime && profile_data->subtime < calltime) in profile_graph_return()
880 calltime -= profile_data->subtime; in profile_graph_return()
885 rec = ftrace_find_profiled_func(stat, trace->func); in profile_graph_return()
887 rec->time += calltime; in profile_graph_return()
888 rec->time_squared += calltime * calltime; in profile_graph_return()
1012 stat->stat = function_stats; in ftrace_profile_tracefs()
1013 stat->stat.name = name; in ftrace_profile_tracefs()
1014 ret = register_stat_tracer(&stat->stat); in ftrace_profile_tracefs()
1046 # error Dynamic ftrace depends on MCOUNT_RECORD
1080 * Used by the stack unwinder to know about dynamic ftrace trampolines.
1098 if (op->trampoline && op->trampoline_size) in ftrace_ops_trampoline()
1099 if (addr >= op->trampoline && in ftrace_ops_trampoline()
1100 addr < op->trampoline + op->trampoline_size) { in ftrace_ops_trampoline()
1137 if (hash->size_bits > 0) in ftrace_hash_key()
1138 return hash_long(ip, hash->size_bits); in ftrace_hash_key()
1152 hhd = &hash->buckets[key]; in __ftrace_lookup_ip()
1155 if (entry->ip == ip) in __ftrace_lookup_ip()
1162 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1186 key = ftrace_hash_key(hash, entry->ip); in __add_hash_entry()
1187 hhd = &hash->buckets[key]; in __add_hash_entry()
1188 hlist_add_head(&entry->hlist, hhd); in __add_hash_entry()
1189 hash->count++; in __add_hash_entry()
1201 entry->ip = ip; in add_hash_entry()
1211 hlist_del(&entry->hlist); in free_hash_entry()
1213 hash->count--; in free_hash_entry()
1220 hlist_del_rcu(&entry->hlist); in remove_hash_entry()
1221 hash->count--; in remove_hash_entry()
1229 int size = 1 << hash->size_bits; in ftrace_hash_clear() local
1232 if (!hash->count) in ftrace_hash_clear()
1235 for (i = 0; i < size; i++) { in ftrace_hash_clear()
1236 hhd = &hash->buckets[i]; in ftrace_hash_clear()
1240 FTRACE_WARN_ON(hash->count); in ftrace_hash_clear()
1245 list_del(&ftrace_mod->list); in free_ftrace_mod()
1246 kfree(ftrace_mod->module); in free_ftrace_mod()
1247 kfree(ftrace_mod->func); in free_ftrace_mod()
1270 kfree(hash->buckets); in free_ftrace_hash()
1286 call_rcu(&hash->rcu, __free_ftrace_hash_rcu); in free_ftrace_hash_rcu()
1290 * ftrace_free_filter - remove all filters for an ftrace_ops
1296 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) in ftrace_free_filter()
1298 free_ftrace_hash(ops->func_hash->filter_hash); in ftrace_free_filter()
1299 free_ftrace_hash(ops->func_hash->notrace_hash); in ftrace_free_filter()
1300 ops->func_hash->filter_hash = EMPTY_HASH; in ftrace_free_filter()
1301 ops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_free_filter()
1308 int size; in alloc_ftrace_hash() local
1314 size = 1 << size_bits; in alloc_ftrace_hash()
1315 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); in alloc_ftrace_hash()
1317 if (!hash->buckets) { in alloc_ftrace_hash()
1322 hash->size_bits = size_bits; in alloc_ftrace_hash()
1333 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace; in ftrace_add_mod()
1337 return -ENOMEM; in ftrace_add_mod()
1339 INIT_LIST_HEAD(&ftrace_mod->list); in ftrace_add_mod()
1340 ftrace_mod->func = kstrdup(func, GFP_KERNEL); in ftrace_add_mod()
1341 ftrace_mod->module = kstrdup(module, GFP_KERNEL); in ftrace_add_mod()
1342 ftrace_mod->enable = enable; in ftrace_add_mod()
1344 if (!ftrace_mod->func || !ftrace_mod->module) in ftrace_add_mod()
1347 list_add(&ftrace_mod->list, mod_head); in ftrace_add_mod()
1354 return -ENOMEM; in ftrace_add_mod()
1362 int size; in alloc_and_copy_ftrace_hash() local
1370 new_hash->flags = hash->flags; in alloc_and_copy_ftrace_hash()
1376 size = 1 << hash->size_bits; in alloc_and_copy_ftrace_hash()
1377 for (i = 0; i < size; i++) { in alloc_and_copy_ftrace_hash()
1378 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in alloc_and_copy_ftrace_hash()
1379 if (add_hash_entry(new_hash, entry->ip) == NULL) in alloc_and_copy_ftrace_hash()
1384 FTRACE_WARN_ON(new_hash->count != hash->count); in alloc_and_copy_ftrace_hash()
1403 static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) in __move_hash() argument
1413 * Use around half the size (max bit of it), but in __move_hash()
1414 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits). in __move_hash()
1416 bits = fls(size / 2); in __move_hash()
1426 new_hash->flags = src->flags; in __move_hash()
1428 size = 1 << src->size_bits; in __move_hash()
1429 for (i = 0; i < size; i++) { in __move_hash()
1430 hhd = &src->buckets[i]; in __move_hash()
1443 int size = src->count; in __ftrace_hash_move() local
1451 return __move_hash(src, size); in __ftrace_hash_move()
1455 * ftrace_hash_move - move a new hash to a filter and do updates
1482 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) in ftrace_hash_move()
1483 return -EINVAL; in ftrace_hash_move()
1487 return -ENOMEM; in ftrace_hash_move()
1521 return (ftrace_hash_empty(hash->filter_hash) || in hash_contains_ip()
1522 __ftrace_lookup_ip(hash->filter_hash, ip)) && in hash_contains_ip()
1523 (ftrace_hash_empty(hash->notrace_hash) || in hash_contains_ip()
1524 !__ftrace_lookup_ip(hash->notrace_hash, ip)); in hash_contains_ip()
1529 * the ops->func or not.
1531 * It's a match if the ip is in the ops->filter_hash or
1534 * the ip is not in the ops->notrace_hash.
1547 * There's a small race when adding ops that the ftrace handler in ftrace_ops_test()
1551 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) in ftrace_ops_test()
1555 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); in ftrace_ops_test()
1556 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); in ftrace_ops_test()
1571 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1573 for (_____i = 0; _____i < pg->index; _____i++) { \
1574 rec = &pg->records[_____i];
1586 if (key->flags < rec->ip) in ftrace_cmp_recs()
1587 return -1; in ftrace_cmp_recs()
1588 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) in ftrace_cmp_recs()
1602 for (pg = ftrace_pages_start; pg; pg = pg->next) { in lookup_rec()
1603 if (pg->index == 0 || in lookup_rec()
1604 end < pg->records[0].ip || in lookup_rec()
1605 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in lookup_rec()
1607 rec = bsearch(&key, pg->records, pg->index, in lookup_rec()
1617 * ftrace_location_range - return the first address of a traced location
1623 * Returns: rec->ip if the related ftrace location is a least partly within
1625 * that is either a NOP or call to the function tracer. It checks the ftrace
1636 ip = rec->ip; in ftrace_location_range()
1643 * ftrace_location - return the ftrace location
1647 * * If @ip matches the ftrace location, return @ip.
1648 * * If @ip matches sym+0, return sym's ftrace location.
1655 unsigned long size; in ftrace_location() local
1659 if (!kallsyms_lookup_size_offset(ip, &size, &offset)) in ftrace_location()
1664 loc = ftrace_location_range(ip, ip + size - 1); in ftrace_location()
1670 * ftrace_text_reserved - return true if range contains an ftrace location
1674 * Returns: 1 if @start and @end contains a ftrace location.
1676 * the function tracer. It checks the ftrace internal tables to
1696 ops != &ftrace_list_end; ops = ops->next) { in test_rec_ops_needs_regs()
1697 /* pass rec in as regs to have non-NULL val */ in test_rec_ops_needs_regs()
1698 if (ftrace_ops_test(ops, rec->ip, rec)) { in test_rec_ops_needs_regs()
1699 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in test_rec_ops_needs_regs()
1724 return rec->flags & FTRACE_FL_DISABLED && in skip_record()
1725 !(rec->flags & FTRACE_FL_ENABLED); in skip_record()
1729 * This is the main engine to the ftrace updates to the dyn_ftrace records.
1731 * It will iterate through all the available ftrace functions
1732 * (the ones that ftrace can have callbacks to) and set the flags
1750 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_rec_update()
1757 hash = ops->func_hash->filter_hash; in __ftrace_hash_rec_update()
1758 notrace_hash = ops->func_hash->notrace_hash; in __ftrace_hash_rec_update()
1775 if (!notrace_hash || !ftrace_lookup_ip(notrace_hash, rec->ip)) in __ftrace_hash_rec_update()
1778 in_hash = !!ftrace_lookup_ip(hash, rec->ip); in __ftrace_hash_rec_update()
1779 in_notrace_hash = !!ftrace_lookup_ip(notrace_hash, rec->ip); in __ftrace_hash_rec_update()
1792 rec->flags++; in __ftrace_hash_rec_update()
1796 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1797 rec->flags |= FTRACE_FL_DIRECT; in __ftrace_hash_rec_update()
1804 if (ftrace_rec_count(rec) == 1 && ops->trampoline) in __ftrace_hash_rec_update()
1805 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1813 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1819 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in __ftrace_hash_rec_update()
1820 rec->flags |= FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1824 rec->flags--; in __ftrace_hash_rec_update()
1832 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1833 rec->flags &= ~FTRACE_FL_DIRECT; in __ftrace_hash_rec_update()
1842 rec->flags & FTRACE_FL_REGS && in __ftrace_hash_rec_update()
1843 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in __ftrace_hash_rec_update()
1845 rec->flags &= ~FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1857 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1859 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1868 * If the rec has a single associated ops, and ops->func can be in __ftrace_hash_rec_update()
1873 ftrace_ops_get_func(ops) == ops->func) in __ftrace_hash_rec_update()
1874 rec->flags |= FTRACE_FL_CALL_OPS; in __ftrace_hash_rec_update()
1876 rec->flags &= ~FTRACE_FL_CALL_OPS; in __ftrace_hash_rec_update()
1884 if (!all && count == hash->count) in __ftrace_hash_rec_update()
1920 * ops->hash = new_hash
1934 if (ops->func_hash != &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1945 if (op->func_hash == &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1962 * or no-needed to update, -EBUSY if it detects a conflict of the flag
1963 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1965 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1966 * - If the hash is EMPTY_HASH, it hits nothing
1967 * - Anything else hits the recs which match the hash entries.
1972 * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate
1986 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_update_ipmodify()
1989 is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1990 is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; in __ftrace_hash_update_ipmodify()
2005 return -EINVAL; in __ftrace_hash_update_ipmodify()
2007 /* Update rec->flags */ in __ftrace_hash_update_ipmodify()
2010 if (rec->flags & FTRACE_FL_DISABLED) in __ftrace_hash_update_ipmodify()
2014 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2015 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2020 if (rec->flags & FTRACE_FL_IPMODIFY) { in __ftrace_hash_update_ipmodify()
2027 FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT); in __ftrace_hash_update_ipmodify()
2035 if (!ops->ops_func) in __ftrace_hash_update_ipmodify()
2036 return -EBUSY; in __ftrace_hash_update_ipmodify()
2037 ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); in __ftrace_hash_update_ipmodify()
2041 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2044 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2056 if (rec->flags & FTRACE_FL_DISABLED) in __ftrace_hash_update_ipmodify()
2060 return -EBUSY; in __ftrace_hash_update_ipmodify()
2062 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2063 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2068 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2070 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2073 return -EBUSY; in __ftrace_hash_update_ipmodify()
2078 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_enable()
2089 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_disable()
2100 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_update()
2133 pr_info("Initializing ftrace call sites\n"); in print_bug_type()
2136 pr_info("Setting ftrace call site to NOP\n"); in print_bug_type()
2139 pr_info("Setting ftrace call site to call ftrace function\n"); in print_bug_type()
2142 pr_info("Updating ftrace call site to call a different ftrace function\n"); in print_bug_type()
2148 * ftrace_bug - report and shutdown function tracer
2155 * EFAULT - if the problem happens on reading the @ip address
2156 * EINVAL - if what is read at @ip is not what was expected
2157 * EPERM - if the problem happens on writing to the @ip address
2161 unsigned long ip = rec ? rec->ip : 0; in ftrace_bug()
2163 pr_info("------------[ ftrace bug ]------------\n"); in ftrace_bug()
2166 case -EFAULT: in ftrace_bug()
2167 pr_info("ftrace faulted on modifying "); in ftrace_bug()
2170 case -EINVAL: in ftrace_bug()
2171 pr_info("ftrace failed to modify "); in ftrace_bug()
2180 case -EPERM: in ftrace_bug()
2181 pr_info("ftrace faulted on writing "); in ftrace_bug()
2185 pr_info("ftrace faulted on unknown error "); in ftrace_bug()
2192 pr_info("ftrace record flags: %lx\n", rec->flags); in ftrace_bug()
2194 rec->flags & FTRACE_FL_REGS ? " R" : " ", in ftrace_bug()
2195 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " "); in ftrace_bug()
2196 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_bug()
2201 (void *)ops->trampoline, in ftrace_bug()
2202 (void *)ops->func); in ftrace_bug()
2246 if (!(rec->flags & FTRACE_FL_REGS) != in ftrace_check_record()
2247 !(rec->flags & FTRACE_FL_REGS_EN)) in ftrace_check_record()
2250 if (!(rec->flags & FTRACE_FL_TRAMP) != in ftrace_check_record()
2251 !(rec->flags & FTRACE_FL_TRAMP_EN)) in ftrace_check_record()
2266 if (!(rec->flags & FTRACE_FL_DIRECT) != in ftrace_check_record()
2267 !(rec->flags & FTRACE_FL_DIRECT_EN)) in ftrace_check_record()
2269 } else if (rec->flags & FTRACE_FL_DIRECT_EN) { in ftrace_check_record()
2279 if (!(rec->flags & FTRACE_FL_CALL_OPS) != in ftrace_check_record()
2280 !(rec->flags & FTRACE_FL_CALL_OPS_EN)) in ftrace_check_record()
2282 } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) { in ftrace_check_record()
2288 if ((rec->flags & FTRACE_FL_ENABLED) == flag) in ftrace_check_record()
2293 flag ^= rec->flags & FTRACE_FL_ENABLED; in ftrace_check_record()
2296 rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED; in ftrace_check_record()
2298 if (rec->flags & FTRACE_FL_REGS) in ftrace_check_record()
2299 rec->flags |= FTRACE_FL_REGS_EN; in ftrace_check_record()
2301 rec->flags &= ~FTRACE_FL_REGS_EN; in ftrace_check_record()
2304 if (rec->flags & FTRACE_FL_TRAMP) in ftrace_check_record()
2305 rec->flags |= FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2307 rec->flags &= ~FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2311 if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY)) in ftrace_check_record()
2312 rec->flags |= FTRACE_FL_MODIFIED; in ftrace_check_record()
2318 * directly (no ftrace trampoline). in ftrace_check_record()
2321 if (rec->flags & FTRACE_FL_DIRECT) in ftrace_check_record()
2322 rec->flags |= FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2324 rec->flags &= ~FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2330 rec->flags &= ~FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2336 if (rec->flags & FTRACE_FL_CALL_OPS) in ftrace_check_record()
2337 rec->flags |= FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2339 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2345 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2355 * from the save regs, to a non-save regs function or in ftrace_check_record()
2370 rec->flags &= FTRACE_NOCLEAR_FLAGS; in ftrace_check_record()
2376 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | in ftrace_check_record()
2386 * ftrace_update_record - set a record that now is tracing or not
2399 * ftrace_test_record - check if the record has been enabled or not
2416 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any()
2420 if (!op->trampoline) in ftrace_find_tramp_ops_any()
2423 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_any()
2434 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any_other()
2438 if (op == op_exclude || !op->trampoline) in ftrace_find_tramp_ops_any_other()
2441 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_any_other()
2452 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_next()
2456 if (!op->trampoline) in ftrace_find_tramp_ops_next()
2459 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_next()
2470 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_curr()
2479 if (hash_contains_ip(ip, &removed_ops->old_hash)) in ftrace_find_tramp_ops_curr()
2503 if (!op->trampoline) in ftrace_find_tramp_ops_curr()
2510 if (op->flags & FTRACE_OPS_FL_ADDING) in ftrace_find_tramp_ops_curr()
2519 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && in ftrace_find_tramp_ops_curr()
2520 hash_contains_ip(ip, &op->old_hash)) in ftrace_find_tramp_ops_curr()
2527 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && in ftrace_find_tramp_ops_curr()
2528 hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_curr()
2540 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_new()
2543 /* pass rec in as regs to have non-NULL val */ in ftrace_find_tramp_ops_new()
2544 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_new()
2555 unsigned long ip = rec->ip; in ftrace_find_unique_ops()
2559 if (hash_contains_ip(ip, op->func_hash)) { in ftrace_find_unique_ops()
2587 return entry->direct; in ftrace_find_rec_direct()
2593 unsigned long addr = READ_ONCE(ops->direct_call); in call_direct_funcs()
2603 * ftrace_get_addr_new - Get the call address to set to
2604 * @rec: The ftrace record descriptor
2617 if ((rec->flags & FTRACE_FL_DIRECT) && in ftrace_get_addr_new()
2619 addr = ftrace_find_rec_direct(rec->ip); in ftrace_get_addr_new()
2626 if (rec->flags & FTRACE_FL_TRAMP) { in ftrace_get_addr_new()
2628 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { in ftrace_get_addr_new()
2630 (void *)rec->ip, (void *)rec->ip, rec->flags); in ftrace_get_addr_new()
2631 /* Ftrace is shutting down, return anything */ in ftrace_get_addr_new()
2634 return ops->trampoline; in ftrace_get_addr_new()
2637 if (rec->flags & FTRACE_FL_REGS) in ftrace_get_addr_new()
2644 * ftrace_get_addr_curr - Get the call address that is already there
2645 * @rec: The ftrace record descriptor
2659 if (rec->flags & FTRACE_FL_DIRECT_EN) { in ftrace_get_addr_curr()
2660 addr = ftrace_find_rec_direct(rec->ip); in ftrace_get_addr_curr()
2667 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_get_addr_curr()
2671 (void *)rec->ip, (void *)rec->ip); in ftrace_get_addr_curr()
2672 /* Ftrace is shutting down, return anything */ in ftrace_get_addr_curr()
2675 return ops->trampoline; in ftrace_get_addr_curr()
2678 if (rec->flags & FTRACE_FL_REGS_EN) in ftrace_get_addr_curr()
2717 return -1; /* unknown ftrace bug */ in __ftrace_replace_code()
2753 * ftrace_rec_iter_start - start up iterating over traced functions
2770 iter->pg = ftrace_pages_start; in ftrace_rec_iter_start()
2771 iter->index = 0; in ftrace_rec_iter_start()
2774 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_start()
2775 iter->pg = iter->pg->next; in ftrace_rec_iter_start()
2777 if (!iter->pg) in ftrace_rec_iter_start()
2784 * ftrace_rec_iter_next - get the next record to process.
2791 iter->index++; in ftrace_rec_iter_next()
2793 if (iter->index >= iter->pg->index) { in ftrace_rec_iter_next()
2794 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2795 iter->index = 0; in ftrace_rec_iter_next()
2798 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_next()
2799 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2802 if (!iter->pg) in ftrace_rec_iter_next()
2809 * ftrace_rec_iter_record - get the record at the iterator location
2816 return &iter->pg->records[iter->index]; in ftrace_rec_iter_record()
2923 * ftrace_run_stop_machine - go back to the stop machine method
2924 * @command: The command to tell ftrace what to do
2935 * arch_ftrace_update_code - modify the code to trace or not trace
2964 ops->flags |= FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2965 ops->old_hash.filter_hash = old_hash->filter_hash; in ftrace_run_modify_code()
2966 ops->old_hash.notrace_hash = old_hash->notrace_hash; in ftrace_run_modify_code()
2968 ops->old_hash.filter_hash = NULL; in ftrace_run_modify_code()
2969 ops->old_hash.notrace_hash = NULL; in ftrace_run_modify_code()
2970 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2986 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list); in ftrace_add_trampoline_to_kallsyms()
2992 list_del_rcu(&ops->list); in ftrace_remove_trampoline_from_kallsyms()
2998 * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
3006 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) && in ftrace_trampoline_free()
3007 ops->trampoline) { in ftrace_trampoline_free()
3012 perf_event_text_poke((void *)ops->trampoline, in ftrace_trampoline_free()
3013 (void *)ops->trampoline, in ftrace_trampoline_free()
3014 ops->trampoline_size, NULL, 0); in ftrace_trampoline_free()
3016 ops->trampoline, ops->trampoline_size, in ftrace_trampoline_free()
3050 return -ENODEV; in ftrace_startup()
3059 * Note that ftrace probes uses this to start up in ftrace_startup()
3066 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; in ftrace_startup()
3072 ftrace_start_up--; in ftrace_startup()
3073 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_startup()
3074 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) in ftrace_startup()
3085 * If ftrace is in an undefined state, we just remove ops from list in ftrace_startup()
3091 return -ENODEV; in ftrace_startup()
3094 ops->flags &= ~FTRACE_OPS_FL_ADDING; in ftrace_startup()
3104 return -ENODEV; in ftrace_shutdown()
3110 ftrace_start_up--; in ftrace_shutdown()
3112 * Just warn in case of unbalance, no need to kill ftrace, it's not in ftrace_shutdown()
3114 * further ftrace uses. in ftrace_shutdown()
3124 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_shutdown()
3138 ops->flags |= FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3142 ops->old_hash.filter_hash = ops->func_hash->filter_hash; in ftrace_shutdown()
3143 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; in ftrace_shutdown()
3148 * If there's no more ops registered with ftrace, run a in ftrace_shutdown()
3157 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS)) in ftrace_shutdown()
3159 (void *)rec->ip, rec->flags); in ftrace_shutdown()
3163 ops->old_hash.filter_hash = NULL; in ftrace_shutdown()
3164 ops->old_hash.notrace_hash = NULL; in ftrace_shutdown()
3167 ops->flags &= ~FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3174 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) { in ftrace_shutdown()
3187 * while on a ftrace trampoline. Just scheduling a task on in ftrace_shutdown()
3206 return alloc_and_copy_ftrace_hash(src->size_bits, src); in copy_hash()
3228 int size; in append_hash() local
3238 return -ENOMEM; in append_hash()
3248 size = 1 << new_hash->size_bits; in append_hash()
3249 for (i = 0; i < size; i++) { in append_hash()
3250 hlist_for_each_entry(entry, &new_hash->buckets[i], hlist) { in append_hash()
3252 if (!__ftrace_lookup_ip(*hash, entry->ip) && in append_hash()
3253 add_hash_entry(*hash, entry->ip) == NULL) in append_hash()
3254 return -ENOMEM; in append_hash()
3267 int size; in remove_hash() local
3274 size = 1 << hash->size_bits; in remove_hash()
3275 for (i = 0; i < size; i++) { in remove_hash()
3276 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { in remove_hash()
3277 if (!__ftrace_lookup_ip(notrace_hash, entry->ip)) in remove_hash()
3295 int size; in intersect_hash() local
3308 size = 1 << new_hash1->size_bits; in intersect_hash()
3309 for (i = 0; i < size; i++) { in intersect_hash()
3310 hlist_for_each_entry(entry, &new_hash1->buckets[i], hlist) { in intersect_hash()
3312 if (__ftrace_lookup_ip(new_hash2, entry->ip) && in intersect_hash()
3313 add_hash_entry(*hash, entry->ip) == NULL) in intersect_hash()
3314 return -ENOMEM; in intersect_hash()
3328 int size; in ops_equal() local
3337 if (A->count != B->count) in ops_equal()
3340 size = 1 << A->size_bits; in ops_equal()
3341 for (i = 0; i < size; i++) { in ops_equal()
3342 hlist_for_each_entry(entry, &A->buckets[i], hlist) { in ops_equal()
3343 if (!__ftrace_lookup_ip(B, entry->ip)) in ops_equal()
3364 old_hash_ops.filter_hash = ops->func_hash->filter_hash; in __ftrace_hash_move_and_update_ops()
3365 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; in __ftrace_hash_move_and_update_ops()
3379 if (!ops_equal(filter_hash, ops->func_hash->filter_hash)) { in ftrace_update_ops()
3380 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash, in ftrace_update_ops()
3386 if (!ops_equal(notrace_hash, ops->func_hash->notrace_hash)) { in ftrace_update_ops()
3387 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash, in ftrace_update_ops()
3400 if (!ftrace_hash_empty(func_hash->filter_hash)) { in add_first_hash()
3401 *filter_hash = copy_hash(func_hash->filter_hash); in add_first_hash()
3403 return -ENOMEM; in add_first_hash()
3404 remove_hash(*filter_hash, func_hash->notrace_hash); in add_first_hash()
3408 *notrace_hash = copy_hash(func_hash->notrace_hash); in add_first_hash()
3410 return -ENOMEM; in add_first_hash()
3423 if (ftrace_hash_empty(ops_hash->filter_hash) || in add_next_hash()
3424 ftrace_hash_empty(subops_hash->filter_hash)) { in add_next_hash()
3432 WARN_ON_ONCE(!ftrace_hash_empty(ops_hash->notrace_hash)); in add_next_hash()
3434 size_bits = max(ops_hash->filter_hash->size_bits, in add_next_hash()
3435 subops_hash->filter_hash->size_bits); in add_next_hash()
3438 *filter_hash = alloc_and_copy_ftrace_hash(size_bits, subops_hash->filter_hash); in add_next_hash()
3440 return -ENOMEM; in add_next_hash()
3442 remove_hash(*filter_hash, subops_hash->notrace_hash); in add_next_hash()
3444 ret = append_hash(filter_hash, ops_hash->filter_hash, in add_next_hash()
3468 size_bits = max(ops_hash->notrace_hash->size_bits, in add_next_hash()
3469 subops_hash->notrace_hash->size_bits); in add_next_hash()
3472 return -ENOMEM; in add_next_hash()
3474 ret = intersect_hash(notrace_hash, ops_hash->notrace_hash, in add_next_hash()
3475 subops_hash->notrace_hash); in add_next_hash()
3486 * ftrace_startup_subops - enable tracing for subops of an ops
3504 return -ENODEV; in ftrace_startup_subops()
3509 if (WARN_ON_ONCE(subops->flags & FTRACE_OPS_FL_ENABLED)) in ftrace_startup_subops()
3510 return -EBUSY; in ftrace_startup_subops()
3513 if (!ops->func_hash->filter_hash) in ftrace_startup_subops()
3514 ops->func_hash->filter_hash = EMPTY_HASH; in ftrace_startup_subops()
3515 if (!ops->func_hash->notrace_hash) in ftrace_startup_subops()
3516 ops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_startup_subops()
3517 if (!subops->func_hash->filter_hash) in ftrace_startup_subops()
3518 subops->func_hash->filter_hash = EMPTY_HASH; in ftrace_startup_subops()
3519 if (!subops->func_hash->notrace_hash) in ftrace_startup_subops()
3520 subops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_startup_subops()
3523 if (list_empty(&ops->subop_list)) { in ftrace_startup_subops()
3526 WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->filter_hash)); in ftrace_startup_subops()
3527 WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->notrace_hash)); in ftrace_startup_subops()
3529 ret = add_first_hash(&filter_hash, ¬race_hash, subops->func_hash); in ftrace_startup_subops()
3533 save_filter_hash = ops->func_hash->filter_hash; in ftrace_startup_subops()
3534 save_notrace_hash = ops->func_hash->notrace_hash; in ftrace_startup_subops()
3536 ops->func_hash->filter_hash = filter_hash; in ftrace_startup_subops()
3537 ops->func_hash->notrace_hash = notrace_hash; in ftrace_startup_subops()
3538 list_add(&subops->list, &ops->subop_list); in ftrace_startup_subops()
3541 list_del(&subops->list); in ftrace_startup_subops()
3542 ops->func_hash->filter_hash = save_filter_hash; in ftrace_startup_subops()
3543 ops->func_hash->notrace_hash = save_notrace_hash; in ftrace_startup_subops()
3549 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; in ftrace_startup_subops()
3550 subops->managed = ops; in ftrace_startup_subops()
3564 ret = add_next_hash(&filter_hash, ¬race_hash, ops->func_hash, subops->func_hash); in ftrace_startup_subops()
3568 list_add(&subops->list, &ops->subop_list); in ftrace_startup_subops()
3574 list_del(&subops->list); in ftrace_startup_subops()
3576 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; in ftrace_startup_subops()
3577 subops->managed = ops; in ftrace_startup_subops()
3593 list_for_each_entry(subops, &ops->subop_list, list) { in rebuild_hashes()
3598 ret = add_first_hash(filter_hash, notrace_hash, subops->func_hash); in rebuild_hashes()
3604 &temp_hash, subops->func_hash); in rebuild_hashes()
3622 * ftrace_shutdown_subops - Remove a subops from a manager ops
3640 return -ENODEV; in ftrace_shutdown_subops()
3642 if (WARN_ON_ONCE(!(subops->flags & FTRACE_OPS_FL_ENABLED))) in ftrace_shutdown_subops()
3643 return -EINVAL; in ftrace_shutdown_subops()
3645 list_del(&subops->list); in ftrace_shutdown_subops()
3647 if (list_empty(&ops->subop_list)) { in ftrace_shutdown_subops()
3652 list_add(&subops->list, &ops->subop_list); in ftrace_shutdown_subops()
3656 subops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_shutdown_subops()
3658 free_ftrace_hash(ops->func_hash->filter_hash); in ftrace_shutdown_subops()
3659 free_ftrace_hash(ops->func_hash->notrace_hash); in ftrace_shutdown_subops()
3660 ops->func_hash->filter_hash = EMPTY_HASH; in ftrace_shutdown_subops()
3661 ops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_shutdown_subops()
3662 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); in ftrace_shutdown_subops()
3663 subops->managed = NULL; in ftrace_shutdown_subops()
3675 list_add(&subops->list, &ops->subop_list); in ftrace_shutdown_subops()
3677 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); in ftrace_shutdown_subops()
3678 subops->managed = NULL; in ftrace_shutdown_subops()
3689 struct ftrace_ops *ops = subops->managed; in ftrace_hash_move_and_update_subops()
3697 if (WARN_ON_ONCE(!ops || ops->flags & FTRACE_OPS_FL_SUBOP)) in ftrace_hash_move_and_update_subops()
3698 return -EINVAL; in ftrace_hash_move_and_update_subops()
3705 return -ENOMEM; in ftrace_hash_move_and_update_subops()
3739 return ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_traces_mod()
3740 ftrace_hash_empty(ops->func_hash->notrace_hash); in ops_traces_mod()
3758 * an entry in the ftrace data. Now, if ftrace is activated in ftrace_update_code()
3760 * read-only, the modification of enabling ftrace can fail if in ftrace_update_code()
3761 * the read-only is done while ftrace is converting the calls. in ftrace_update_code()
3764 * to read-only. in ftrace_update_code()
3769 for (pg = new_pgs; pg; pg = pg->next) { in ftrace_update_code()
3771 for (i = 0; i < pg->index; i++) { in ftrace_update_code()
3775 return -1; in ftrace_update_code()
3777 p = &pg->records[i]; in ftrace_update_code()
3778 p->flags = rec_flags; in ftrace_update_code()
3792 update_time = stop - start; in ftrace_update_code()
3809 return -EINVAL; in ftrace_allocate_records()
3813 order = fls(pages) - 1; in ftrace_allocate_records()
3816 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); in ftrace_allocate_records()
3818 if (!pg->records) { in ftrace_allocate_records()
3819 /* if we can't allocate this size, try something smaller */ in ftrace_allocate_records()
3821 return -ENOMEM; in ftrace_allocate_records()
3822 order--; in ftrace_allocate_records()
3830 pg->order = order; in ftrace_allocate_records()
3843 if (pg->records) { in ftrace_free_pages()
3844 free_pages((unsigned long)pg->records, pg->order); in ftrace_free_pages()
3845 ftrace_number_of_pages -= 1 << pg->order; in ftrace_free_pages()
3847 pages = pg->next; in ftrace_free_pages()
3850 ftrace_number_of_groups--; in ftrace_free_pages()
3878 num_to_init -= cnt; in ftrace_allocate_pages()
3882 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3883 if (!pg->next) in ftrace_allocate_pages()
3886 pg = pg->next; in ftrace_allocate_pages()
3893 pr_info("ftrace: FAILED to allocate memory for functions\n"); in ftrace_allocate_pages()
3920 struct ftrace_iterator *iter = m->private; in t_probe_next()
3921 struct trace_array *tr = iter->ops->private; in t_probe_next()
3927 int size; in t_probe_next() local
3930 iter->pos = *pos; in t_probe_next()
3935 func_probes = &tr->func_probes; in t_probe_next()
3939 if (!iter->probe) { in t_probe_next()
3940 next = func_probes->next; in t_probe_next()
3941 iter->probe = list_entry(next, struct ftrace_func_probe, list); in t_probe_next()
3944 if (iter->probe_entry) in t_probe_next()
3945 hnd = &iter->probe_entry->hlist; in t_probe_next()
3947 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3956 size = 1 << hash->size_bits; in t_probe_next()
3959 if (iter->pidx >= size) { in t_probe_next()
3960 if (iter->probe->list.next == func_probes) in t_probe_next()
3962 next = iter->probe->list.next; in t_probe_next()
3963 iter->probe = list_entry(next, struct ftrace_func_probe, list); in t_probe_next()
3964 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3965 size = 1 << hash->size_bits; in t_probe_next()
3966 iter->pidx = 0; in t_probe_next()
3969 hhd = &hash->buckets[iter->pidx]; in t_probe_next()
3972 iter->pidx++; in t_probe_next()
3978 hnd = hhd->first; in t_probe_next()
3980 hnd = hnd->next; in t_probe_next()
3982 iter->pidx++; in t_probe_next()
3990 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist); in t_probe_next()
3997 struct ftrace_iterator *iter = m->private; in t_probe_start()
4001 if (!(iter->flags & FTRACE_ITER_DO_PROBES)) in t_probe_start()
4004 if (iter->mod_pos > *pos) in t_probe_start()
4007 iter->probe = NULL; in t_probe_start()
4008 iter->probe_entry = NULL; in t_probe_start()
4009 iter->pidx = 0; in t_probe_start()
4010 for (l = 0; l <= (*pos - iter->mod_pos); ) { in t_probe_start()
4019 iter->flags |= FTRACE_ITER_PROBE; in t_probe_start()
4031 probe = iter->probe; in t_probe_show()
4032 probe_entry = iter->probe_entry; in t_probe_show()
4035 return -EIO; in t_probe_show()
4037 probe_ops = probe->probe_ops; in t_probe_show()
4039 if (probe_ops->print) in t_probe_show()
4040 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); in t_probe_show()
4042 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, in t_probe_show()
4043 (void *)probe_ops->func); in t_probe_show()
4051 struct ftrace_iterator *iter = m->private; in t_mod_next()
4052 struct trace_array *tr = iter->tr; in t_mod_next()
4055 iter->pos = *pos; in t_mod_next()
4057 iter->mod_list = iter->mod_list->next; in t_mod_next()
4059 if (iter->mod_list == &tr->mod_trace || in t_mod_next()
4060 iter->mod_list == &tr->mod_notrace) { in t_mod_next()
4061 iter->flags &= ~FTRACE_ITER_MOD; in t_mod_next()
4065 iter->mod_pos = *pos; in t_mod_next()
4072 struct ftrace_iterator *iter = m->private; in t_mod_start()
4076 if (iter->func_pos > *pos) in t_mod_start()
4079 iter->mod_pos = iter->func_pos; in t_mod_start()
4082 if (!iter->tr) in t_mod_start()
4085 for (l = 0; l <= (*pos - iter->func_pos); ) { in t_mod_start()
4091 iter->flags &= ~FTRACE_ITER_MOD; in t_mod_start()
4096 iter->flags |= FTRACE_ITER_MOD; in t_mod_start()
4105 struct trace_array *tr = iter->tr; in t_mod_show()
4107 if (WARN_ON_ONCE(!iter->mod_list) || in t_mod_show()
4108 iter->mod_list == &tr->mod_trace || in t_mod_show()
4109 iter->mod_list == &tr->mod_notrace) in t_mod_show()
4110 return -EIO; in t_mod_show()
4112 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list); in t_mod_show()
4114 if (ftrace_mod->func) in t_mod_show()
4115 seq_printf(m, "%s", ftrace_mod->func); in t_mod_show()
4119 seq_printf(m, ":mod:%s\n", ftrace_mod->module); in t_mod_show()
4127 struct ftrace_iterator *iter = m->private; in t_func_next()
4133 if (iter->idx >= iter->pg->index) { in t_func_next()
4134 if (iter->pg->next) { in t_func_next()
4135 iter->pg = iter->pg->next; in t_func_next()
4136 iter->idx = 0; in t_func_next()
4140 rec = &iter->pg->records[iter->idx++]; in t_func_next()
4141 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && in t_func_next()
4142 !ftrace_lookup_ip(iter->hash, rec->ip)) || in t_func_next()
4144 ((iter->flags & FTRACE_ITER_ENABLED) && in t_func_next()
4145 !(rec->flags & FTRACE_FL_ENABLED)) || in t_func_next()
4147 ((iter->flags & FTRACE_ITER_TOUCHED) && in t_func_next()
4148 !(rec->flags & FTRACE_FL_TOUCHED))) { in t_func_next()
4158 iter->pos = iter->func_pos = *pos; in t_func_next()
4159 iter->func = rec; in t_func_next()
4167 struct ftrace_iterator *iter = m->private; in t_next()
4174 if (iter->flags & FTRACE_ITER_PROBE) in t_next()
4177 if (iter->flags & FTRACE_ITER_MOD) in t_next()
4180 if (iter->flags & FTRACE_ITER_PRINTALL) { in t_next()
4196 iter->pos = 0; in reset_iter_read()
4197 iter->func_pos = 0; in reset_iter_read()
4198 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD); in reset_iter_read()
4203 struct ftrace_iterator *iter = m->private; in t_start()
4215 if (*pos < iter->pos) in t_start()
4223 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && in t_start()
4224 ftrace_hash_empty(iter->hash)) { in t_start()
4225 iter->func_pos = 1; /* Account for the message */ in t_start()
4228 iter->flags |= FTRACE_ITER_PRINTALL; in t_start()
4230 iter->flags &= ~FTRACE_ITER_PROBE; in t_start()
4234 if (iter->flags & FTRACE_ITER_MOD) in t_start()
4242 iter->pg = ftrace_pages_start; in t_start()
4243 iter->idx = 0; in t_start()
4274 seq_printf(m, " ->%pS", ptr); in add_trampoline_func()
4294 ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str); in test_for_valid_rec()
4298 rec->flags |= FTRACE_FL_DISABLED; in test_for_valid_rec()
4361 return ret == NULL ? -1 : 0; in print_rec()
4378 struct ftrace_iterator *iter = m->private; in t_show()
4381 if (iter->flags & FTRACE_ITER_PROBE) in t_show()
4384 if (iter->flags & FTRACE_ITER_MOD) in t_show()
4387 if (iter->flags & FTRACE_ITER_PRINTALL) { in t_show()
4388 if (iter->flags & FTRACE_ITER_NOTRACE) in t_show()
4395 rec = iter->func; in t_show()
4400 if (iter->flags & FTRACE_ITER_ADDRS) in t_show()
4401 seq_printf(m, "%lx ", rec->ip); in t_show()
4403 if (print_rec(m, rec->ip)) { in t_show()
4405 WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED)); in t_show()
4410 if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) { in t_show()
4415 rec->flags & FTRACE_FL_REGS ? " R" : " ", in t_show()
4416 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ", in t_show()
4417 rec->flags & FTRACE_FL_DIRECT ? " D" : " ", in t_show()
4418 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ", in t_show()
4419 rec->flags & FTRACE_FL_MODIFIED ? " M " : " "); in t_show()
4420 if (rec->flags & FTRACE_FL_TRAMP_EN) { in t_show()
4425 (void *)ops->trampoline, in t_show()
4426 (void *)ops->func); in t_show()
4435 if (rec->flags & FTRACE_FL_CALL_OPS_EN) { in t_show()
4439 ops, ops->func); in t_show()
4444 if (rec->flags & FTRACE_FL_DIRECT) { in t_show()
4447 direct = ftrace_find_rec_direct(rec->ip); in t_show()
4449 seq_printf(m, "\n\tdirect-->%pS", (void *)direct); in t_show()
4476 return -ENODEV; in ftrace_avail_open()
4480 return -ENOMEM; in ftrace_avail_open()
4482 iter->pg = ftrace_pages_start; in ftrace_avail_open()
4483 iter->ops = &global_ops; in ftrace_avail_open()
4504 return -ENOMEM; in ftrace_enabled_open()
4506 iter->pg = ftrace_pages_start; in ftrace_enabled_open()
4507 iter->flags = FTRACE_ITER_ENABLED; in ftrace_enabled_open()
4508 iter->ops = &global_ops; in ftrace_enabled_open()
4529 return -ENOMEM; in ftrace_touched_open()
4531 iter->pg = ftrace_pages_start; in ftrace_touched_open()
4532 iter->flags = FTRACE_ITER_TOUCHED; in ftrace_touched_open()
4533 iter->ops = &global_ops; in ftrace_touched_open()
4549 return -ENODEV; in ftrace_avail_addrs_open()
4553 return -ENOMEM; in ftrace_avail_addrs_open()
4555 iter->pg = ftrace_pages_start; in ftrace_avail_addrs_open()
4556 iter->flags = FTRACE_ITER_ADDRS; in ftrace_avail_addrs_open()
4557 iter->ops = &global_ops; in ftrace_avail_addrs_open()
4563 * ftrace_regex_open - initialize function tracer filter files
4587 struct trace_array *tr = ops->private; in ftrace_regex_open()
4588 int ret = -ENOMEM; in ftrace_regex_open()
4593 return -ENODEV; in ftrace_regex_open()
4596 return -ENODEV; in ftrace_regex_open()
4602 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) in ftrace_regex_open()
4605 iter->ops = ops; in ftrace_regex_open()
4606 iter->flags = flag; in ftrace_regex_open()
4607 iter->tr = tr; in ftrace_regex_open()
4609 mutex_lock(&ops->func_hash->regex_lock); in ftrace_regex_open()
4612 hash = ops->func_hash->notrace_hash; in ftrace_regex_open()
4613 mod_head = tr ? &tr->mod_notrace : NULL; in ftrace_regex_open()
4615 hash = ops->func_hash->filter_hash; in ftrace_regex_open()
4616 mod_head = tr ? &tr->mod_trace : NULL; in ftrace_regex_open()
4619 iter->mod_list = mod_head; in ftrace_regex_open()
4621 if (file->f_mode & FMODE_WRITE) { in ftrace_regex_open()
4624 if (file->f_flags & O_TRUNC) { in ftrace_regex_open()
4625 iter->hash = alloc_ftrace_hash(size_bits); in ftrace_regex_open()
4628 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); in ftrace_regex_open()
4631 if (!iter->hash) { in ftrace_regex_open()
4632 trace_parser_put(&iter->parser); in ftrace_regex_open()
4636 iter->hash = hash; in ftrace_regex_open()
4640 if (file->f_mode & FMODE_READ) { in ftrace_regex_open()
4641 iter->pg = ftrace_pages_start; in ftrace_regex_open()
4645 struct seq_file *m = file->private_data; in ftrace_regex_open()
4646 m->private = iter; in ftrace_regex_open()
4649 free_ftrace_hash(iter->hash); in ftrace_regex_open()
4650 trace_parser_put(&iter->parser); in ftrace_regex_open()
4653 file->private_data = iter; in ftrace_regex_open()
4656 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_regex_open()
4671 struct ftrace_ops *ops = inode->i_private; in ftrace_filter_open()
4682 struct ftrace_ops *ops = inode->i_private; in ftrace_notrace_open()
4689 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
4697 * If symbols in an architecture don't correspond exactly to the user-visible
4711 str = arch_ftrace_match_adjust(str, g->search); in ftrace_match()
4713 switch (g->type) { in ftrace_match()
4715 if (strcmp(str, g->search) == 0) in ftrace_match()
4719 if (strncmp(str, g->search, g->len) == 0) in ftrace_match()
4723 if (strstr(str, g->search)) in ftrace_match()
4728 if (slen >= g->len && in ftrace_match()
4729 memcmp(str + slen - g->len, g->search, g->len) == 0) in ftrace_match()
4733 if (glob_match(g->search, str)) in ftrace_match()
4747 entry = ftrace_lookup_ip(hash, rec->ip); in enter_record()
4758 if (add_hash_entry(hash, rec->ip) == NULL) in enter_record()
4759 ret = -ENOMEM; in enter_record()
4773 if (kstrtoul(func_g->search, 0, &index) || --index < 0) in add_rec_by_index()
4777 if (pg->index <= index) { in add_rec_by_index()
4778 index -= pg->index; in add_rec_by_index()
4782 rec = &pg->records[index]; in add_rec_by_index()
4796 return -1; in lookup_ip()
4814 if (lookup_ip(rec->ip, &modname, str)) { in ftrace_match_record()
4817 !(rec->flags & FTRACE_FL_DISABLED)); in ftrace_match_record()
4825 if (!mod_g->len) { in ftrace_match_record()
4843 if (!func_g->len) in ftrace_match_record()
4885 if (rec->flags & FTRACE_FL_DISABLED) in match_records()
4914 if (ops->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4924 if (ops->func_hash != &global_ops.local_hash) in ftrace_ops_update_code()
4928 if (op->func_hash == &global_ops.local_hash && in ftrace_ops_update_code()
4929 op->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4942 if (ops->flags & FTRACE_OPS_FL_SUBOP) in ftrace_hash_move_and_update_ops()
4952 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) { in ftrace_hash_move_and_update_ops()
4959 list_for_each_entry(subops, &op->subop_list, list) { in ftrace_hash_move_and_update_ops()
4960 if ((subops->flags & FTRACE_OPS_FL_ENABLED) && in ftrace_hash_move_and_update_ops()
4961 subops->func_hash == ops->func_hash) { in ftrace_hash_move_and_update_ops()
4975 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace; in cache_mod()
4981 int ret = -EINVAL; in cache_mod()
4987 if (strcmp(ftrace_mod->module, module) != 0) in cache_mod()
4992 (ftrace_mod->func && in cache_mod()
4993 strcmp(ftrace_mod->func, func) == 0)) { in cache_mod()
5004 return -EINVAL; in cache_mod()
5019 mutex_lock(&ops->func_hash->regex_lock); in process_mod_list()
5022 orig_hash = &ops->func_hash->filter_hash; in process_mod_list()
5024 orig_hash = &ops->func_hash->notrace_hash; in process_mod_list()
5035 if (strcmp(ftrace_mod->module, mod) != 0) in process_mod_list()
5038 if (ftrace_mod->func) in process_mod_list()
5039 func = kstrdup(ftrace_mod->func, GFP_KERNEL); in process_mod_list()
5046 list_move(&ftrace_mod->list, &process_mods); in process_mod_list()
5049 kfree(ftrace_mod->func); in process_mod_list()
5050 ftrace_mod->func = func; in process_mod_list()
5057 func = ftrace_mod->func; in process_mod_list()
5065 new_hash->flags &= ~FTRACE_HASH_FL_MOD; in process_mod_list()
5074 mutex_unlock(&ops->func_hash->regex_lock); in process_mod_list()
5090 if (!list_empty(&tr->mod_trace)) in process_cached_mods()
5091 process_mod_list(&tr->mod_trace, tr->ops, mod, true); in process_cached_mods()
5092 if (!list_empty(&tr->mod_notrace)) in process_cached_mods()
5093 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); in process_cached_mods()
5114 return -ENODEV; in ftrace_mod_callback()
5119 return -ENOMEM; in ftrace_mod_callback()
5156 probe_ops = probe->probe_ops; in function_trace_probe_call()
5164 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); in function_trace_probe_call()
5178 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
5196 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
5212 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_find_ip()
5217 return &map->data; in ftrace_func_mapper_find_ip()
5221 * ftrace_func_mapper_add_ip - Map some data to an ip
5234 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_add_ip()
5236 return -EBUSY; in ftrace_func_mapper_add_ip()
5240 return -ENOMEM; in ftrace_func_mapper_add_ip()
5242 map->entry.ip = ip; in ftrace_func_mapper_add_ip()
5243 map->data = data; in ftrace_func_mapper_add_ip()
5245 __add_hash_entry(&mapper->hash, &map->entry); in ftrace_func_mapper_add_ip()
5251 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
5267 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_remove_ip()
5272 data = map->data; in ftrace_func_mapper_remove_ip()
5274 remove_hash_entry(&mapper->hash, entry); in ftrace_func_mapper_remove_ip()
5281 * free_ftrace_func_mapper - free a mapping of ips and data
5294 int size, i; in free_ftrace_func_mapper() local
5299 if (free_func && mapper->hash.count) { in free_ftrace_func_mapper()
5300 size = 1 << mapper->hash.size_bits; in free_ftrace_func_mapper()
5301 for (i = 0; i < size; i++) { in free_ftrace_func_mapper()
5302 hhd = &mapper->hash.buckets[i]; in free_ftrace_func_mapper()
5309 free_ftrace_hash(&mapper->hash); in free_ftrace_func_mapper()
5318 WARN_ON(probe->ref <= 0); in release_probe()
5321 probe->ref--; in release_probe()
5323 if (!probe->ref) { in release_probe()
5324 probe_ops = probe->probe_ops; in release_probe()
5327 * the probe->data itself in release_probe()
5329 if (probe_ops->free) in release_probe()
5330 probe_ops->free(probe_ops, probe->tr, 0, probe->data); in release_probe()
5331 list_del(&probe->list); in release_probe()
5342 probe->ref++; in acquire_probe_locked()
5356 int size; in register_ftrace_function_probe() local
5361 return -EINVAL; in register_ftrace_function_probe()
5365 return -EINVAL; in register_ftrace_function_probe()
5370 list_for_each_entry(iter, &tr->func_probes, list) { in register_ftrace_function_probe()
5371 if (iter->probe_ops == probe_ops) { in register_ftrace_function_probe()
5380 return -ENOMEM; in register_ftrace_function_probe()
5382 probe->probe_ops = probe_ops; in register_ftrace_function_probe()
5383 probe->ops.func = function_trace_probe_call; in register_ftrace_function_probe()
5384 probe->tr = tr; in register_ftrace_function_probe()
5385 ftrace_ops_init(&probe->ops); in register_ftrace_function_probe()
5386 list_add(&probe->list, &tr->func_probes); in register_ftrace_function_probe()
5394 * Note, there's a small window here that the func_hash->filter_hash in register_ftrace_function_probe()
5397 mutex_lock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
5399 orig_hash = &probe->ops.func_hash->filter_hash; in register_ftrace_function_probe()
5404 ret = -ENOMEM; in register_ftrace_function_probe()
5412 ret = -EINVAL; in register_ftrace_function_probe()
5417 size = 1 << hash->size_bits; in register_ftrace_function_probe()
5418 for (i = 0; i < size; i++) { in register_ftrace_function_probe()
5419 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_function_probe()
5420 if (ftrace_lookup_ip(old_hash, entry->ip)) in register_ftrace_function_probe()
5427 if (probe_ops->init) { in register_ftrace_function_probe()
5428 ret = probe_ops->init(probe_ops, tr, in register_ftrace_function_probe()
5429 entry->ip, data, in register_ftrace_function_probe()
5430 &probe->data); in register_ftrace_function_probe()
5432 if (probe_ops->free && count) in register_ftrace_function_probe()
5433 probe_ops->free(probe_ops, tr, in register_ftrace_function_probe()
5434 0, probe->data); in register_ftrace_function_probe()
5435 probe->data = NULL; in register_ftrace_function_probe()
5447 ret = -EINVAL; in register_ftrace_function_probe()
5451 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in register_ftrace_function_probe()
5457 probe->ref += count; in register_ftrace_function_probe()
5459 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) in register_ftrace_function_probe()
5460 ret = ftrace_startup(&probe->ops, 0); in register_ftrace_function_probe()
5468 mutex_unlock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
5476 if (!probe_ops->free || !count) in register_ftrace_function_probe()
5480 for (i = 0; i < size; i++) { in register_ftrace_function_probe()
5481 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_function_probe()
5482 if (ftrace_lookup_ip(old_hash, entry->ip)) in register_ftrace_function_probe()
5484 probe_ops->free(probe_ops, tr, entry->ip, probe->data); in register_ftrace_function_probe()
5505 int i, ret = -ENODEV; in unregister_ftrace_function_probe_func()
5506 int size; in unregister_ftrace_function_probe_func() local
5519 return -EINVAL; in unregister_ftrace_function_probe_func()
5524 list_for_each_entry(iter, &tr->func_probes, list) { in unregister_ftrace_function_probe_func()
5525 if (iter->probe_ops == probe_ops) { in unregister_ftrace_function_probe_func()
5533 ret = -EINVAL; in unregister_ftrace_function_probe_func()
5534 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) in unregister_ftrace_function_probe_func()
5541 mutex_lock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
5543 orig_hash = &probe->ops.func_hash->filter_hash; in unregister_ftrace_function_probe_func()
5553 ret = -ENOMEM; in unregister_ftrace_function_probe_func()
5560 size = 1 << hash->size_bits; in unregister_ftrace_function_probe_func()
5561 for (i = 0; i < size; i++) { in unregister_ftrace_function_probe_func()
5562 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { in unregister_ftrace_function_probe_func()
5565 kallsyms_lookup(entry->ip, NULL, NULL, in unregister_ftrace_function_probe_func()
5572 hlist_add_head(&entry->hlist, &hhd); in unregister_ftrace_function_probe_func()
5578 ret = -EINVAL; in unregister_ftrace_function_probe_func()
5584 WARN_ON(probe->ref < count); in unregister_ftrace_function_probe_func()
5586 probe->ref -= count; in unregister_ftrace_function_probe_func()
5589 ftrace_shutdown(&probe->ops, 0); in unregister_ftrace_function_probe_func()
5591 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in unregister_ftrace_function_probe_func()
5596 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, in unregister_ftrace_function_probe_func()
5601 hlist_del(&entry->hlist); in unregister_ftrace_function_probe_func()
5602 if (probe_ops->free) in unregister_ftrace_function_probe_func()
5603 probe_ops->free(probe_ops, tr, entry->ip, probe->data); in unregister_ftrace_function_probe_func()
5609 mutex_unlock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
5625 list_for_each_entry_safe(probe, n, &tr->func_probes, list) in clear_ftrace_function_probes()
5626 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops); in clear_ftrace_function_probes()
5633 * Currently we only register ftrace commands from __init, so mark this
5642 if (strcmp(cmd->name, p->name) == 0) in register_ftrace_command()
5643 return -EBUSY; in register_ftrace_command()
5645 list_add(&cmd->list, &ftrace_commands); in register_ftrace_command()
5651 * Currently we only unregister ftrace commands from __init, so mark
5661 if (strcmp(cmd->name, p->name) == 0) { in unregister_ftrace_command()
5662 list_del_init(&p->list); in unregister_ftrace_command()
5667 return -ENODEV; in unregister_ftrace_command()
5673 struct ftrace_hash *hash = iter->hash; in ftrace_process_regex()
5674 struct trace_array *tr = iter->ops->private; in ftrace_process_regex()
5684 ret = -EINVAL; in ftrace_process_regex()
5697 if (strcmp(p->name, command) == 0) in ftrace_process_regex()
5698 return p->func(tr, hash, func, command, next, enable); in ftrace_process_regex()
5701 return -EINVAL; in ftrace_process_regex()
5715 if (file->f_mode & FMODE_READ) { in ftrace_regex_write()
5716 struct seq_file *m = file->private_data; in ftrace_regex_write()
5717 iter = m->private; in ftrace_regex_write()
5719 iter = file->private_data; in ftrace_regex_write()
5722 return -ENODEV; in ftrace_regex_write()
5724 /* iter->hash is a local copy, so we don't need regex_lock */ in ftrace_regex_write()
5726 parser = &iter->parser; in ftrace_regex_write()
5731 ret = ftrace_process_regex(iter, parser->buffer, in ftrace_regex_write()
5732 parser->idx, enable); in ftrace_regex_write()
5762 return -EINVAL; in __ftrace_match_addr()
5767 return -ENOENT; in __ftrace_match_addr()
5776 return entry ? 0 : -ENOMEM; in __ftrace_match_addr()
5809 return -ENODEV; in ftrace_set_hash()
5811 mutex_lock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5814 orig_hash = &ops->func_hash->filter_hash; in ftrace_set_hash()
5816 orig_hash = &ops->func_hash->notrace_hash; in ftrace_set_hash()
5824 ret = -ENOMEM; in ftrace_set_hash()
5831 (*orig_hash)->flags |= FTRACE_HASH_FL_MOD; in ftrace_set_hash()
5837 ret = -EINVAL; in ftrace_set_hash()
5851 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5882 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in check_direct_multi()
5883 return -EINVAL; in check_direct_multi()
5884 if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS) in check_direct_multi()
5885 return -EINVAL; in check_direct_multi()
5892 int size, i; in remove_direct_functions_hash() local
5894 size = 1 << hash->size_bits; in remove_direct_functions_hash()
5895 for (i = 0; i < size; i++) { in remove_direct_functions_hash()
5896 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in remove_direct_functions_hash()
5897 del = __ftrace_lookup_ip(direct_functions, entry->ip); in remove_direct_functions_hash()
5898 if (del && del->direct == addr) { in remove_direct_functions_hash()
5914 * register_ftrace_direct - Call a custom trampoline directly
5929 * -EINVAL - The @ops object was already registered with this call or
5931 * -EBUSY - Another direct function is already attached (there can be only one)
5932 * -ENODEV - @ip does not point to a ftrace nop location (or not supported)
5933 * -ENOMEM - There was an allocation failure.
5939 int err = -EBUSY, size, i; in register_ftrace_direct() local
5941 if (ops->func || ops->trampoline) in register_ftrace_direct()
5942 return -EINVAL; in register_ftrace_direct()
5943 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in register_ftrace_direct()
5944 return -EINVAL; in register_ftrace_direct()
5945 if (ops->flags & FTRACE_OPS_FL_ENABLED) in register_ftrace_direct()
5946 return -EINVAL; in register_ftrace_direct()
5948 hash = ops->func_hash->filter_hash; in register_ftrace_direct()
5950 return -EINVAL; in register_ftrace_direct()
5955 size = 1 << hash->size_bits; in register_ftrace_direct()
5956 for (i = 0; i < size; i++) { in register_ftrace_direct()
5957 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_direct()
5958 if (ftrace_find_rec_direct(entry->ip)) in register_ftrace_direct()
5963 err = -ENOMEM; in register_ftrace_direct()
5966 size = hash->count + direct_functions->count; in register_ftrace_direct()
5967 size = fls(size); in register_ftrace_direct()
5968 if (size > FTRACE_HASH_MAX_BITS) in register_ftrace_direct()
5969 size = FTRACE_HASH_MAX_BITS; in register_ftrace_direct()
5970 new_hash = alloc_ftrace_hash(size); in register_ftrace_direct()
5975 size = 1 << direct_functions->size_bits; in register_ftrace_direct()
5976 for (i = 0; i < size; i++) { in register_ftrace_direct()
5977 hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) { in register_ftrace_direct()
5978 new = add_hash_entry(new_hash, entry->ip); in register_ftrace_direct()
5981 new->direct = entry->direct; in register_ftrace_direct()
5986 size = 1 << hash->size_bits; in register_ftrace_direct()
5987 for (i = 0; i < size; i++) { in register_ftrace_direct()
5988 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_direct()
5989 new = add_hash_entry(new_hash, entry->ip); in register_ftrace_direct()
5993 new->direct = addr; in register_ftrace_direct()
5994 entry->direct = addr; in register_ftrace_direct()
6002 ops->func = call_direct_funcs; in register_ftrace_direct()
6003 ops->flags = MULTI_FLAGS; in register_ftrace_direct()
6004 ops->trampoline = FTRACE_REGS_ADDR; in register_ftrace_direct()
6005 ops->direct_call = addr; in register_ftrace_direct()
6013 call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); in register_ftrace_direct()
6023 * unregister_ftrace_direct - Remove calls to custom trampoline
6035 * -EINVAL - The @ops object was not properly registered.
6040 struct ftrace_hash *hash = ops->func_hash->filter_hash; in unregister_ftrace_direct()
6044 return -EINVAL; in unregister_ftrace_direct()
6045 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in unregister_ftrace_direct()
6046 return -EINVAL; in unregister_ftrace_direct()
6054 ops->func = NULL; in unregister_ftrace_direct()
6055 ops->trampoline = 0; in unregister_ftrace_direct()
6072 int i, size; in __modify_ftrace_direct() local
6079 tmp_ops.func_hash = ops->func_hash; in __modify_ftrace_direct()
6092 hash = ops->func_hash->filter_hash; in __modify_ftrace_direct()
6093 size = 1 << hash->size_bits; in __modify_ftrace_direct()
6094 for (i = 0; i < size; i++) { in __modify_ftrace_direct()
6095 hlist_for_each_entry(iter, &hash->buckets[i], hlist) { in __modify_ftrace_direct()
6096 entry = __ftrace_lookup_ip(direct_functions, iter->ip); in __modify_ftrace_direct()
6099 entry->direct = addr; in __modify_ftrace_direct()
6103 WRITE_ONCE(ops->direct_call, addr); in __modify_ftrace_direct()
6114 * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
6129 * -EINVAL - The @ops object was not properly registered.
6134 return -EINVAL; in modify_ftrace_direct_nolock()
6135 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct_nolock()
6136 return -EINVAL; in modify_ftrace_direct_nolock()
6143 * modify_ftrace_direct - Modify an existing direct 'multi' call
6155 * -EINVAL - The @ops object was not properly registered.
6162 return -EINVAL; in modify_ftrace_direct()
6163 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct()
6164 return -EINVAL; in modify_ftrace_direct()
6175 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
6197 * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses
6220 * ftrace_ops_set_global_filter - setup ops to use global filters
6223 * ftrace users who need global function trace filtering should call this.
6228 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) in ftrace_ops_set_global_filter()
6232 ops->func_hash = &global_ops.local_hash; in ftrace_ops_set_global_filter()
6242 struct trace_array *tr = ops->private; in ftrace_set_regex()
6250 return -EINVAL; in ftrace_set_regex()
6254 return -EINVAL; in ftrace_set_regex()
6257 len = command - func; in ftrace_set_regex()
6267 return -ENOMEM; in ftrace_set_regex()
6275 * ftrace_set_filter - set a function to filter on in ftrace
6279 * @reset: non-zero to reset all filters before applying this filter.
6297 * ftrace_set_notrace - set a function to not trace in ftrace
6301 * @reset: non-zero to reset all filters before applying this filter.
6319 * ftrace_set_global_filter - set a function to filter on with global tracers
6322 * @reset: non-zero to reset all filters before applying this filter.
6334 * ftrace_set_global_notrace - set a function to not trace with global tracers
6337 * @reset: non-zero to reset all filters before applying this filter.
6417 printk(KERN_DEBUG "ftrace: function %s not " in set_ftrace_early_graph()
6436 if (!ops->private) { in ftrace_set_early_filter()
6439 ops->private = tr; in ftrace_set_early_filter()
6465 struct seq_file *m = (struct seq_file *)file->private_data; in ftrace_regex_release()
6471 if (file->f_mode & FMODE_READ) { in ftrace_regex_release()
6472 iter = m->private; in ftrace_regex_release()
6475 iter = file->private_data; in ftrace_regex_release()
6477 parser = &iter->parser; in ftrace_regex_release()
6479 int enable = !(iter->flags & FTRACE_ITER_NOTRACE); in ftrace_regex_release()
6481 ftrace_process_regex(iter, parser->buffer, in ftrace_regex_release()
6482 parser->idx, enable); in ftrace_regex_release()
6487 mutex_lock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
6489 if (file->f_mode & FMODE_WRITE) { in ftrace_regex_release()
6490 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); in ftrace_regex_release()
6493 orig_hash = &iter->ops->func_hash->filter_hash; in ftrace_regex_release()
6494 if (iter->tr) { in ftrace_regex_release()
6495 if (list_empty(&iter->tr->mod_trace)) in ftrace_regex_release()
6496 iter->hash->flags &= ~FTRACE_HASH_FL_MOD; in ftrace_regex_release()
6498 iter->hash->flags |= FTRACE_HASH_FL_MOD; in ftrace_regex_release()
6501 orig_hash = &iter->ops->func_hash->notrace_hash; in ftrace_regex_release()
6504 ftrace_hash_move_and_update_ops(iter->ops, orig_hash, in ftrace_regex_release()
6505 iter->hash, filter_hash); in ftrace_regex_release()
6509 iter->hash = NULL; in ftrace_regex_release()
6512 mutex_unlock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
6513 free_ftrace_hash(iter->hash); in ftrace_regex_release()
6514 if (iter->tr) in ftrace_regex_release()
6515 trace_array_put(iter->tr); in ftrace_regex_release()
6592 struct ftrace_graph_data *fgd = m->private; in __g_next()
6593 struct ftrace_func_entry *entry = fgd->entry; in __g_next()
6595 int i, idx = fgd->idx; in __g_next()
6597 if (*pos >= fgd->hash->count) in __g_next()
6602 fgd->entry = entry; in __g_next()
6609 for (i = idx; i < 1 << fgd->hash->size_bits; i++) { in __g_next()
6610 head = &fgd->hash->buckets[i]; in __g_next()
6612 fgd->entry = entry; in __g_next()
6613 fgd->idx = i; in __g_next()
6629 struct ftrace_graph_data *fgd = m->private; in g_start()
6633 if (fgd->type == GRAPH_FILTER_FUNCTION) in g_start()
6634 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, in g_start()
6637 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, in g_start()
6641 if (ftrace_hash_empty(fgd->hash) && !*pos) in g_start()
6644 fgd->idx = 0; in g_start()
6645 fgd->entry = NULL; in g_start()
6662 struct ftrace_graph_data *fgd = m->private; in g_show()
6664 if (fgd->type == GRAPH_FILTER_FUNCTION) in g_show()
6671 seq_printf(m, "%ps\n", (void *)entry->ip); in g_show()
6694 if (file->f_mode & FMODE_WRITE) { in __ftrace_graph_open()
6697 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX)) in __ftrace_graph_open()
6698 return -ENOMEM; in __ftrace_graph_open()
6700 if (file->f_flags & O_TRUNC) in __ftrace_graph_open()
6704 fgd->hash); in __ftrace_graph_open()
6706 ret = -ENOMEM; in __ftrace_graph_open()
6711 if (file->f_mode & FMODE_READ) { in __ftrace_graph_open()
6714 struct seq_file *m = file->private_data; in __ftrace_graph_open()
6715 m->private = fgd; in __ftrace_graph_open()
6722 file->private_data = fgd; in __ftrace_graph_open()
6725 if (ret < 0 && file->f_mode & FMODE_WRITE) in __ftrace_graph_open()
6726 trace_parser_put(&fgd->parser); in __ftrace_graph_open()
6728 fgd->new_hash = new_hash; in __ftrace_graph_open()
6731 * All uses of fgd->hash must be taken with the graph_lock in __ftrace_graph_open()
6733 * fgd->hash to be reinitialized when it is taken again. in __ftrace_graph_open()
6735 fgd->hash = NULL; in __ftrace_graph_open()
6747 return -ENODEV; in ftrace_graph_open()
6751 return -ENOMEM; in ftrace_graph_open()
6755 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, in ftrace_graph_open()
6757 fgd->type = GRAPH_FILTER_FUNCTION; in ftrace_graph_open()
6758 fgd->seq_ops = &ftrace_graph_seq_ops; in ftrace_graph_open()
6775 return -ENODEV; in ftrace_graph_notrace_open()
6779 return -ENOMEM; in ftrace_graph_notrace_open()
6783 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, in ftrace_graph_notrace_open()
6785 fgd->type = GRAPH_FILTER_NOTRACE; in ftrace_graph_notrace_open()
6786 fgd->seq_ops = &ftrace_graph_seq_ops; in ftrace_graph_notrace_open()
6804 if (file->f_mode & FMODE_READ) { in ftrace_graph_release()
6805 struct seq_file *m = file->private_data; in ftrace_graph_release()
6807 fgd = m->private; in ftrace_graph_release()
6810 fgd = file->private_data; in ftrace_graph_release()
6814 if (file->f_mode & FMODE_WRITE) { in ftrace_graph_release()
6816 parser = &fgd->parser; in ftrace_graph_release()
6819 ret = ftrace_graph_set_hash(fgd->new_hash, in ftrace_graph_release()
6820 parser->buffer); in ftrace_graph_release()
6825 new_hash = __ftrace_hash_move(fgd->new_hash); in ftrace_graph_release()
6827 ret = -ENOMEM; in ftrace_graph_release()
6833 if (fgd->type == GRAPH_FILTER_FUNCTION) { in ftrace_graph_release()
6860 free_ftrace_hash(fgd->new_hash); in ftrace_graph_release()
6885 return -ENODEV; in ftrace_graph_set_hash()
6889 if (rec->flags & FTRACE_FL_DISABLED) in ftrace_graph_set_hash()
6893 entry = ftrace_lookup_ip(hash, rec->ip); in ftrace_graph_set_hash()
6900 if (add_hash_entry(hash, rec->ip) == NULL) in ftrace_graph_set_hash()
6912 return fail ? -EINVAL : 0; in ftrace_graph_set_hash()
6920 struct ftrace_graph_data *fgd = file->private_data; in ftrace_graph_write()
6927 if (file->f_mode & FMODE_READ) { in ftrace_graph_write()
6928 struct seq_file *m = file->private_data; in ftrace_graph_write()
6929 fgd = m->private; in ftrace_graph_write()
6932 parser = &fgd->parser; in ftrace_graph_write()
6939 ret = ftrace_graph_set_hash(fgd->new_hash, in ftrace_graph_write()
6940 parser->buffer); in ftrace_graph_write()
6991 if (ops->flags & FTRACE_OPS_FL_ENABLED) in ftrace_destroy_filter_files()
6993 ops->flags |= FTRACE_OPS_FL_DELETED; in ftrace_destroy_filter_files()
7035 return -1; in ftrace_cmp_ips()
7045 if (WARN(start[i - 1] > start[i], in test_is_sorted()
7047 (void *)start[i - 1], start[i - 1], in test_is_sorted()
7052 pr_info("ftrace section at %px sorted properly\n", start); in test_is_sorted()
7074 int ret = -ENOMEM; in ftrace_process_locs()
7076 count = end - start; in ftrace_process_locs()
7097 return -ENOMEM; in ftrace_process_locs()
7114 if (WARN_ON(ftrace_pages->next)) { in ftrace_process_locs()
7116 while (ftrace_pages->next) in ftrace_process_locs()
7117 ftrace_pages = ftrace_pages->next; in ftrace_process_locs()
7120 ftrace_pages->next = start_pg; in ftrace_process_locs()
7154 end_offset = (pg->index+1) * sizeof(pg->records[0]); in ftrace_process_locs()
7155 if (end_offset > PAGE_SIZE << pg->order) { in ftrace_process_locs()
7157 if (WARN_ON(!pg->next)) in ftrace_process_locs()
7159 pg = pg->next; in ftrace_process_locs()
7162 rec = &pg->records[pg->index++]; in ftrace_process_locs()
7163 rec->ip = addr; in ftrace_process_locs()
7166 if (pg->next) { in ftrace_process_locs()
7167 pg_unuse = pg->next; in ftrace_process_locs()
7168 pg->next = NULL; in ftrace_process_locs()
7197 pg_remaining = (ENTRIES_PER_PAGE << pg->order) - pg->index; in ftrace_process_locs()
7199 if (!WARN(skipped < pg_remaining, "Extra allocated pages for ftrace")) { in ftrace_process_locs()
7201 skip = skipped - pg_remaining; in ftrace_process_locs()
7203 for (pg = pg_unuse; pg; pg = pg->next) in ftrace_process_locs()
7204 remaining += 1 << pg->order; in ftrace_process_locs()
7206 pages -= remaining; in ftrace_process_locs()
7214 WARN(skip != remaining, "Extra allocated pages for ftrace: %lu with %lu skipped", in ftrace_process_locs()
7223 count -= skipped; in ftrace_process_locs()
7224 pr_info("ftrace: allocating %ld entries in %ld pages\n", in ftrace_process_locs()
7235 unsigned int size; member
7256 if (!op->trampoline || symnum--) in ftrace_get_trampoline_kallsym()
7258 *value = op->trampoline; in ftrace_get_trampoline_kallsym()
7266 return -ERANGE; in ftrace_get_trampoline_kallsym()
7281 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in ops_references_ip()
7289 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_references_ip()
7290 !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) in ops_references_ip()
7294 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) in ops_references_ip()
7312 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { in referenced_filters()
7313 if (ops_references_ip(ops, rec->ip)) { in referenced_filters()
7314 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) in referenced_filters()
7316 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in referenced_filters()
7319 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in referenced_filters()
7320 rec->flags |= FTRACE_FL_REGS; in referenced_filters()
7321 if (cnt == 1 && ops->trampoline) in referenced_filters()
7322 rec->flags |= FTRACE_FL_TRAMP; in referenced_filters()
7324 rec->flags &= ~FTRACE_FL_TRAMP; in referenced_filters()
7341 for (i = 0; i < pg->index; i++) { in clear_mod_from_hash()
7342 rec = &pg->records[i]; in clear_mod_from_hash()
7343 entry = __ftrace_lookup_ip(hash, rec->ip); in clear_mod_from_hash()
7350 entry->ip = 0; in clear_mod_from_hash()
7361 if (!tr->ops || !tr->ops->func_hash) in clear_mod_from_hashes()
7363 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
7364 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); in clear_mod_from_hashes()
7365 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); in clear_mod_from_hashes()
7366 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
7378 list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) { in ftrace_free_mod_map()
7379 kfree(mod_func->name); in ftrace_free_mod_map()
7380 list_del(&mod_func->list); in ftrace_free_mod_map()
7402 if (mod_map->mod == mod) { in ftrace_release_mod()
7403 list_del_rcu(&mod_map->list); in ftrace_release_mod()
7404 call_rcu(&mod_map->rcu, ftrace_free_mod_map); in ftrace_release_mod()
7415 rec = &pg->records[0]; in ftrace_release_mod()
7416 if (within_module(rec->ip, mod)) { in ftrace_release_mod()
7428 ftrace_update_tot_cnt -= pg->index; in ftrace_release_mod()
7429 *last_pg = pg->next; in ftrace_release_mod()
7431 pg->next = tmp_page; in ftrace_release_mod()
7434 last_pg = &pg->next; in ftrace_release_mod()
7447 if (pg->records) { in ftrace_release_mod()
7448 free_pages((unsigned long)pg->records, pg->order); in ftrace_release_mod()
7449 ftrace_number_of_pages -= 1 << pg->order; in ftrace_release_mod()
7451 tmp_page = pg->next; in ftrace_release_mod()
7453 ftrace_number_of_groups--; in ftrace_release_mod()
7477 * text to read-only, as we now need to set it back to read-write in ftrace_module_enable()
7491 if (!within_module(rec->ip, mod)) in ftrace_module_enable()
7497 rec->flags = FTRACE_FL_DISABLED; in ftrace_module_enable()
7512 rec->flags &= ~FTRACE_FL_DISABLED; in ftrace_module_enable()
7513 rec->flags += cnt; in ftrace_module_enable()
7532 process_cached_mods(mod->name); in ftrace_module_enable()
7539 if (ftrace_disabled || !mod->num_ftrace_callsites) in ftrace_module_init()
7542 ret = ftrace_process_locs(mod, mod->ftrace_callsites, in ftrace_module_init()
7543 mod->ftrace_callsites + mod->num_ftrace_callsites); in ftrace_module_init()
7545 pr_warn("ftrace: failed to allocate entries for module '%s' functions\n", in ftrace_module_init()
7546 mod->name); in ftrace_module_init()
7559 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str); in save_ftrace_mod_rec()
7567 mod_func->name = kstrdup(str, GFP_KERNEL); in save_ftrace_mod_rec()
7568 if (!mod_func->name) { in save_ftrace_mod_rec()
7573 mod_func->ip = rec->ip - offset; in save_ftrace_mod_rec()
7574 mod_func->size = symsize; in save_ftrace_mod_rec()
7576 mod_map->num_funcs++; in save_ftrace_mod_rec()
7578 list_add_rcu(&mod_func->list, &mod_map->funcs); in save_ftrace_mod_rec()
7591 mod_map->mod = mod; in allocate_ftrace_mod_map()
7592 mod_map->start_addr = start; in allocate_ftrace_mod_map()
7593 mod_map->end_addr = end; in allocate_ftrace_mod_map()
7594 mod_map->num_funcs = 0; in allocate_ftrace_mod_map()
7596 INIT_LIST_HEAD_RCU(&mod_map->funcs); in allocate_ftrace_mod_map()
7598 list_add_rcu(&mod_map->list, &ftrace_mod_maps); in allocate_ftrace_mod_map()
7605 unsigned long addr, unsigned long *size, in ftrace_func_address_lookup() argument
7611 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { in ftrace_func_address_lookup()
7612 if (addr >= mod_func->ip && in ftrace_func_address_lookup()
7613 addr < mod_func->ip + mod_func->size) { in ftrace_func_address_lookup()
7620 if (size) in ftrace_func_address_lookup()
7621 *size = found_func->size; in ftrace_func_address_lookup()
7623 *off = addr - found_func->ip; in ftrace_func_address_lookup()
7624 return strscpy(sym, found_func->name, KSYM_NAME_LEN); in ftrace_func_address_lookup()
7631 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, in ftrace_mod_address_lookup() argument
7640 ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym); in ftrace_mod_address_lookup()
7643 *modname = mod_map->mod->name; in ftrace_mod_address_lookup()
7663 if (symnum >= mod_map->num_funcs) { in ftrace_mod_get_kallsym()
7664 symnum -= mod_map->num_funcs; in ftrace_mod_get_kallsym()
7668 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { in ftrace_mod_get_kallsym()
7670 symnum--; in ftrace_mod_get_kallsym()
7674 *value = mod_func->ip; in ftrace_mod_get_kallsym()
7676 strscpy(name, mod_func->name, KSYM_NAME_LEN); in ftrace_mod_get_kallsym()
7677 strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN); in ftrace_mod_get_kallsym()
7725 entry = ftrace_lookup_ip(hash, func->ip); in clear_func_from_hash()
7732 entry->ip = 0; in clear_func_from_hash()
7742 if (!tr->ops || !tr->ops->func_hash) in clear_func_from_hashes()
7744 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7745 clear_func_from_hash(func, tr->ops->func_hash->filter_hash); in clear_func_from_hashes()
7746 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash); in clear_func_from_hashes()
7747 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7759 MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n"); in add_to_clear_hash_list()
7763 func->ip = rec->ip; in add_to_clear_hash_list()
7764 list_add(&func->list, clear_list); in add_to_clear_hash_list()
7793 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { in ftrace_free_mem()
7794 if (end < pg->records[0].ip || in ftrace_free_mem()
7795 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in ftrace_free_mem()
7798 rec = bsearch(&key, pg->records, pg->index, in ftrace_free_mem()
7810 pg->index--; in ftrace_free_mem()
7811 ftrace_update_tot_cnt--; in ftrace_free_mem()
7812 if (!pg->index) { in ftrace_free_mem()
7813 *last_pg = pg->next; in ftrace_free_mem()
7814 pg->next = tmp_page; in ftrace_free_mem()
7822 (pg->index - (rec - pg->records)) * sizeof(*rec)); in ftrace_free_mem()
7867 count = __stop_mcount_loc - __start_mcount_loc; in ftrace_init()
7869 pr_info("ftrace: No functions to be traced?\n"); in ftrace_init()
7877 pr_warn("ftrace: failed to allocate entries for functions\n"); in ftrace_init()
7881 pr_info("ftrace: allocated %ld pages with %ld groups\n", in ftrace_init()
7900 unsigned long trampoline = ops->trampoline; in ftrace_update_trampoline()
7903 if (ops->trampoline && ops->trampoline != trampoline && in ftrace_update_trampoline()
7904 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) { in ftrace_update_trampoline()
7908 ops->trampoline, ops->trampoline_size, false, in ftrace_update_trampoline()
7914 perf_event_text_poke((void *)ops->trampoline, NULL, 0, in ftrace_update_trampoline()
7915 (void *)ops->trampoline, in ftrace_update_trampoline()
7916 ops->trampoline_size); in ftrace_update_trampoline()
7922 if (tr->flags & TRACE_ARRAY_FL_MOD_INIT) in ftrace_init_trace_array()
7925 INIT_LIST_HEAD(&tr->func_probes); in ftrace_init_trace_array()
7926 INIT_LIST_HEAD(&tr->mod_trace); in ftrace_init_trace_array()
7927 INIT_LIST_HEAD(&tr->mod_notrace); in ftrace_init_trace_array()
7929 tr->flags |= TRACE_ARRAY_FL_MOD_INIT; in ftrace_init_trace_array()
7957 tr->ops = &global_ops; in ftrace_init_global_array_ops()
7961 init_array_fgraph_ops(tr, tr->ops); in ftrace_init_global_array_ops()
7967 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { in ftrace_init_array_ops()
7968 if (WARN_ON(tr->ops->func != ftrace_stub)) in ftrace_init_array_ops()
7969 printk("ftrace ops had %pS for function\n", in ftrace_init_array_ops()
7970 tr->ops->func); in ftrace_init_array_ops()
7972 tr->ops->func = func; in ftrace_init_array_ops()
7973 tr->ops->private = tr; in ftrace_init_array_ops()
7978 tr->ops->func = ftrace_stub; in ftrace_reset_array_ops()
8000 if (op->flags & FTRACE_OPS_FL_STUB) in __ftrace_ops_list_func()
8007 * If any of the above fails then the op->func() is not executed. in __ftrace_ops_list_func()
8009 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && in __ftrace_ops_list_func()
8011 if (FTRACE_WARN_ON(!op->func)) { in __ftrace_ops_list_func()
8015 op->func(ip, parent_ip, op, fregs); in __ftrace_ops_list_func()
8067 if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) in ftrace_ops_assist_func()
8068 op->func(ip, parent_ip, op, fregs); in ftrace_ops_assist_func()
8075 * ftrace_ops_get_func - get the function a trampoline should call
8078 * Normally the mcount trampoline will call the ops->func, but there
8091 if (ops->flags & (FTRACE_OPS_FL_RECURSION | in ftrace_ops_get_func()
8095 return ops->func; in ftrace_ops_get_func()
8108 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_filter_pid_sched_switch_probe()
8109 no_pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_filter_pid_sched_switch_probe()
8112 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ftrace_filter_pid_sched_switch_probe()
8115 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ftrace_filter_pid_sched_switch_probe()
8116 next->pid); in ftrace_filter_pid_sched_switch_probe()
8127 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_pid_follow_sched_process_fork()
8130 pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_pid_follow_sched_process_fork()
8140 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_pid_follow_sched_process_exit()
8143 pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_pid_follow_sched_process_exit()
8168 pid_list = rcu_dereference_protected(tr->function_pids, in clear_ftrace_pids()
8170 no_pid_list = rcu_dereference_protected(tr->function_no_pids, in clear_ftrace_pids()
8181 per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE; in clear_ftrace_pids()
8185 rcu_assign_pointer(tr->function_pids, NULL); in clear_ftrace_pids()
8188 rcu_assign_pointer(tr->function_no_pids, NULL); in clear_ftrace_pids()
8227 struct trace_array *tr = m->private; in fpid_start()
8232 pid_list = rcu_dereference_sched(tr->function_pids); in fpid_start()
8242 struct trace_array *tr = m->private; in fpid_next()
8243 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); in fpid_next()
8280 struct trace_array *tr = m->private; in fnpid_start()
8285 pid_list = rcu_dereference_sched(tr->function_no_pids); in fnpid_start()
8295 struct trace_array *tr = m->private; in fnpid_next()
8296 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids); in fnpid_next()
8315 struct trace_array *tr = inode->i_private; in pid_open()
8323 if ((file->f_mode & FMODE_WRITE) && in pid_open()
8324 (file->f_flags & O_TRUNC)) in pid_open()
8337 return -EINVAL; in pid_open()
8344 m = file->private_data; in pid_open()
8346 m->private = tr; in pid_open()
8374 pid_list = rcu_dereference_protected(tr->function_pids, in ignore_task_cpu()
8376 no_pid_list = rcu_dereference_protected(tr->function_no_pids, in ignore_task_cpu()
8380 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ignore_task_cpu()
8383 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ignore_task_cpu()
8384 current->pid); in ignore_task_cpu()
8391 struct seq_file *m = filp->private_data; in pid_write()
8392 struct trace_array *tr = m->private; in pid_write()
8405 filtered_pids = rcu_dereference_protected(tr->function_pids, in pid_write()
8407 other_pids = rcu_dereference_protected(tr->function_no_pids, in pid_write()
8411 filtered_pids = rcu_dereference_protected(tr->function_no_pids, in pid_write()
8413 other_pids = rcu_dereference_protected(tr->function_pids, in pid_write()
8418 return -EINVAL; in pid_write()
8427 rcu_assign_pointer(tr->function_pids, pid_list); in pid_write()
8430 rcu_assign_pointer(tr->function_no_pids, pid_list); in pid_write()
8475 struct trace_array *tr = inode->i_private; in ftrace_pid_release()
8510 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); in ftrace_init_tracefs_toplevel()
8517 * ftrace_kill - kill ftrace
8519 * This function should be used by panic code. It stops ftrace
8520 * but in a not so nice way. If you need to simply kill ftrace
8521 * from a non-atomic section, use ftrace_kill.
8532 * ftrace_is_dead - Test if ftrace is dead or not.
8534 * Returns: 1 if ftrace is "dead", zero otherwise.
8559 int size, i, ret; in prepare_direct_functions_for_ipmodify() local
8563 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in prepare_direct_functions_for_ipmodify()
8566 hash = ops->func_hash->filter_hash; in prepare_direct_functions_for_ipmodify()
8567 size = 1 << hash->size_bits; in prepare_direct_functions_for_ipmodify()
8568 for (i = 0; i < size; i++) { in prepare_direct_functions_for_ipmodify()
8569 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in prepare_direct_functions_for_ipmodify()
8570 unsigned long ip = entry->ip; in prepare_direct_functions_for_ipmodify()
8575 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) in prepare_direct_functions_for_ipmodify()
8585 if (!op->ops_func) in prepare_direct_functions_for_ipmodify()
8586 return -EBUSY; in prepare_direct_functions_for_ipmodify()
8588 ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); in prepare_direct_functions_for_ipmodify()
8608 int size, i; in cleanup_direct_functions_after_ipmodify() local
8610 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in cleanup_direct_functions_after_ipmodify()
8615 hash = ops->func_hash->filter_hash; in cleanup_direct_functions_after_ipmodify()
8616 size = 1 << hash->size_bits; in cleanup_direct_functions_after_ipmodify()
8617 for (i = 0; i < size; i++) { in cleanup_direct_functions_after_ipmodify()
8618 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in cleanup_direct_functions_after_ipmodify()
8619 unsigned long ip = entry->ip; in cleanup_direct_functions_after_ipmodify()
8624 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) in cleanup_direct_functions_after_ipmodify()
8634 if (found_op && op->ops_func) in cleanup_direct_functions_after_ipmodify()
8635 op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); in cleanup_direct_functions_after_ipmodify()
8679 * register_ftrace_function - register a function for profiling
8685 * Note: @ops->func and all the functions it calls must be labeled
8707 * unregister_ftrace_function - unregister a function for profiling.
8710 * Unregister a function that was added to be called by ftrace profiling.
8750 sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp); in kallsyms_callback()
8754 idx = sym - args->syms; in kallsyms_callback()
8755 if (args->addrs[idx]) in kallsyms_callback()
8761 args->addrs[idx] = addr; in kallsyms_callback()
8762 args->found++; in kallsyms_callback()
8763 return args->found == args->cnt ? 1 : 0; in kallsyms_callback()
8767 * ftrace_lookup_symbols - Lookup addresses for array of symbols
8779 * Returns: 0 if all provided symbols are found, -ESRCH otherwise.
8796 return found_all ? 0 : -ESRCH; in ftrace_lookup_symbols()
8811 /* ftrace_start_up is true if we want ftrace running */ in ftrace_startup_sysctl()
8827 /* ftrace_start_up is true if ftrace is running */ in ftrace_shutdown_sysctl()
8845 if (op->flags & FTRACE_OPS_FL_PERMANENT) in is_permanent_ops_registered()
8861 return -ENODEV; in ftrace_enable_sysctl()
8870 /* we are starting ftrace again */ in ftrace_enable_sysctl()
8880 return -EBUSY; in ftrace_enable_sysctl()
8883 /* stopping ftrace calls (just send to ftrace_stub) */ in ftrace_enable_sysctl()