Lines Matching +full:reserved +full:- +full:ipi +full:- +full:vectors
1 // SPDX-License-Identifier: GPL-2.0
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
75 struct task_struct *p = tfc->p; in remote_function()
78 /* -EAGAIN */ in remote_function()
87 tfc->ret = -ESRCH; /* No such (running) process */ in remote_function()
92 tfc->ret = tfc->func(tfc->info); in remote_function()
96 * task_function_call - call a function on the cpu on which a task runs
106 * returns @func return value or -ESRCH or -ENXIO when the process isn't running
115 .ret = -EAGAIN, in task_function_call()
125 if (ret != -EAGAIN) in task_function_call()
135 * cpu_function_call - call a function on the cpu
142 * returns: @func return value or -ENXIO when the cpu is offline
150 .ret = -ENXIO, /* No such CPU */ in cpu_function_call()
161 raw_spin_lock(&cpuctx->ctx.lock); in perf_ctx_lock()
163 raw_spin_lock(&ctx->lock); in perf_ctx_lock()
170 raw_spin_unlock(&ctx->lock); in perf_ctx_unlock()
171 raw_spin_unlock(&cpuctx->ctx.lock); in perf_ctx_unlock()
174 #define TASK_TOMBSTONE ((void *)-1L)
178 return READ_ONCE(event->owner) == TASK_TOMBSTONE; in is_kernel_event()
186 return this_cpu_ptr(&perf_cpu_context)->task_ctx; in perf_cpu_task_ctx()
192 * When !ctx->nr_events a task context will not be scheduled. This means
198 * - removing the last event from a task ctx; this is relatively straight
201 * - adding the first event to a task ctx; this is tricky because we cannot
202 * rely on ctx->is_active and therefore cannot use event_function_call().
205 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
220 struct perf_event *event = efs->event; in event_function()
221 struct perf_event_context *ctx = event->ctx; in event_function()
223 struct perf_event_context *task_ctx = cpuctx->task_ctx; in event_function()
230 * Since we do the IPI call without holding ctx->lock things can have in event_function()
233 if (ctx->task) { in event_function()
234 if (ctx->task != current) { in event_function()
235 ret = -ESRCH; in event_function()
243 * above ctx->task != current test), therefore we must have in event_function()
244 * ctx->is_active here. in event_function()
246 WARN_ON_ONCE(!ctx->is_active); in event_function()
248 * And since we have ctx->is_active, cpuctx->task_ctx must in event_function()
253 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function()
256 efs->func(event, cpuctx, ctx, efs->data); in event_function()
265 struct perf_event_context *ctx = event->ctx; in event_function_call()
266 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ in event_function_call()
273 if (!event->parent) { in event_function_call()
276 * stabilize the event->ctx relation. See in event_function_call()
279 lockdep_assert_held(&ctx->mutex); in event_function_call()
283 cpu_function_call(event->cpu, event_function, &efs); in event_function_call()
294 raw_spin_lock_irq(&ctx->lock); in event_function_call()
299 task = ctx->task; in event_function_call()
301 raw_spin_unlock_irq(&ctx->lock); in event_function_call()
304 if (ctx->is_active) { in event_function_call()
305 raw_spin_unlock_irq(&ctx->lock); in event_function_call()
309 raw_spin_unlock_irq(&ctx->lock); in event_function_call()
318 struct perf_event_context *ctx = event->ctx; in event_function_local()
320 struct task_struct *task = READ_ONCE(ctx->task); in event_function_local()
334 task = ctx->task; in event_function_local()
341 * otherwise we're screwed, since we cannot IPI to somewhere in event_function_local()
344 if (ctx->is_active) { in event_function_local()
348 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) in event_function_local()
352 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function_local()
414 * -1 - not paranoid at all
415 * 0 - disallow raw tracepoint access for unpriv
416 * 1 - disallow cpu events for unpriv
417 * 2 - disallow kernel profiling for unpriv
462 return -EINVAL; in perf_event_max_sample_rate_handler()
532 running_len -= running_len/NR_ACCUMULATED_SAMPLES; in perf_sample_event_took()
586 return event->clock(); in perf_event_clock()
592 * The basic idea is to use event->state to determine which (if any) time
614 struct perf_event *leader = event->group_leader; in __perf_effective_state()
616 if (leader->state <= PERF_EVENT_STATE_OFF) in __perf_effective_state()
617 return leader->state; in __perf_effective_state()
619 return event->state; in __perf_effective_state()
626 u64 delta = now - event->tstamp; in __perf_update_times()
628 *enabled = event->total_time_enabled; in __perf_update_times()
632 *running = event->total_time_running; in __perf_update_times()
641 __perf_update_times(event, now, &event->total_time_enabled, in perf_event_update_time()
642 &event->total_time_running); in perf_event_update_time()
643 event->tstamp = now; in perf_event_update_time()
657 if (event->state == state) in perf_event_set_state()
665 if ((event->state < 0) ^ (state < 0)) in perf_event_set_state()
668 WRITE_ONCE(event->state, state); in perf_event_set_state()
672 * UP store-release, load-acquire
692 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in perf_ctx_disable()
693 if (cgroup && !pmu_ctx->nr_cgroups) in perf_ctx_disable()
695 perf_pmu_disable(pmu_ctx->pmu); in perf_ctx_disable()
703 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in perf_ctx_enable()
704 if (cgroup && !pmu_ctx->nr_cgroups) in perf_ctx_enable()
706 perf_pmu_enable(pmu_ctx->pmu); in perf_ctx_enable()
721 if (!event->cgrp) in perf_cgroup_match()
725 if (!cpuctx->cgrp) in perf_cgroup_match()
734 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, in perf_cgroup_match()
735 event->cgrp->css.cgroup); in perf_cgroup_match()
740 css_put(&event->cgrp->css); in perf_detach_cgroup()
741 event->cgrp = NULL; in perf_detach_cgroup()
746 return event->cgrp != NULL; in is_cgroup_event()
753 t = per_cpu_ptr(event->cgrp->info, event->cpu); in perf_cgroup_event_time()
754 return t->time; in perf_cgroup_event_time()
761 t = per_cpu_ptr(event->cgrp->info, event->cpu); in perf_cgroup_event_time_now()
762 if (!__load_acquire(&t->active)) in perf_cgroup_event_time_now()
763 return t->time; in perf_cgroup_event_time_now()
764 now += READ_ONCE(t->timeoffset); in perf_cgroup_event_time_now()
771 info->time += now - info->timestamp; in __update_cgrp_time()
772 info->timestamp = now; in __update_cgrp_time()
776 WRITE_ONCE(info->timeoffset, info->time - info->timestamp); in __update_cgrp_time()
781 struct perf_cgroup *cgrp = cpuctx->cgrp; in update_cgrp_time_from_cpuctx()
788 for (css = &cgrp->css; css; css = css->parent) { in update_cgrp_time_from_cpuctx()
790 info = this_cpu_ptr(cgrp->info); in update_cgrp_time_from_cpuctx()
794 __store_release(&info->active, 0); in update_cgrp_time_from_cpuctx()
810 info = this_cpu_ptr(event->cgrp->info); in update_cgrp_time_from_event()
814 if (info->active) in update_cgrp_time_from_event()
821 struct perf_event_context *ctx = &cpuctx->ctx; in perf_cgroup_set_timestamp()
822 struct perf_cgroup *cgrp = cpuctx->cgrp; in perf_cgroup_set_timestamp()
827 * ctx->lock held by caller in perf_cgroup_set_timestamp()
834 WARN_ON_ONCE(!ctx->nr_cgroups); in perf_cgroup_set_timestamp()
836 for (css = &cgrp->css; css; css = css->parent) { in perf_cgroup_set_timestamp()
838 info = this_cpu_ptr(cgrp->info); in perf_cgroup_set_timestamp()
839 __update_cgrp_time(info, ctx->timestamp, false); in perf_cgroup_set_timestamp()
840 __store_release(&info->active, 1); in perf_cgroup_set_timestamp()
853 * cpuctx->cgrp is set when the first cgroup event enabled, in perf_cgroup_switch()
856 if (READ_ONCE(cpuctx->cgrp) == NULL) in perf_cgroup_switch()
859 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); in perf_cgroup_switch()
862 if (READ_ONCE(cpuctx->cgrp) == cgrp) in perf_cgroup_switch()
865 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
866 perf_ctx_disable(&cpuctx->ctx, true); in perf_cgroup_switch()
868 ctx_sched_out(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP); in perf_cgroup_switch()
874 cpuctx->cgrp = cgrp; in perf_cgroup_switch()
880 ctx_sched_in(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP); in perf_cgroup_switch()
882 perf_ctx_enable(&cpuctx->ctx, true); in perf_cgroup_switch()
883 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
897 for (heap_size = 1; css; css = css->parent) in perf_cgroup_ensure_storage()
902 if (heap_size <= cpuctx->heap_size) in perf_cgroup_ensure_storage()
908 ret = -ENOMEM; in perf_cgroup_ensure_storage()
912 raw_spin_lock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
913 if (cpuctx->heap_size < heap_size) { in perf_cgroup_ensure_storage()
914 swap(cpuctx->heap, storage); in perf_cgroup_ensure_storage()
915 if (storage == cpuctx->heap_default) in perf_cgroup_ensure_storage()
917 cpuctx->heap_size = heap_size; in perf_cgroup_ensure_storage()
919 raw_spin_unlock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
937 return -EBADF; in perf_cgroup_connect()
939 css = css_tryget_online_from_dir(f.file->f_path.dentry, in perf_cgroup_connect()
951 event->cgrp = cgrp; in perf_cgroup_connect()
958 if (group_leader && group_leader->cgrp != cgrp) { in perf_cgroup_connect()
960 ret = -EINVAL; in perf_cgroup_connect()
975 event->pmu_ctx->nr_cgroups++; in perf_cgroup_event_enable()
978 * Because cgroup events are always per-cpu events, in perf_cgroup_event_enable()
979 * @ctx == &cpuctx->ctx. in perf_cgroup_event_enable()
983 if (ctx->nr_cgroups++) in perf_cgroup_event_enable()
986 cpuctx->cgrp = perf_cgroup_from_task(current, ctx); in perf_cgroup_event_enable()
997 event->pmu_ctx->nr_cgroups--; in perf_cgroup_event_disable()
1000 * Because cgroup events are always per-cpu events, in perf_cgroup_event_disable()
1001 * @ctx == &cpuctx->ctx. in perf_cgroup_event_disable()
1005 if (--ctx->nr_cgroups) in perf_cgroup_event_disable()
1008 cpuctx->cgrp = NULL; in perf_cgroup_event_disable()
1040 return -EINVAL; in perf_cgroup_connect()
1091 raw_spin_lock(&cpc->hrtimer_lock); in perf_mux_hrtimer_handler()
1093 hrtimer_forward_now(hr, cpc->hrtimer_interval); in perf_mux_hrtimer_handler()
1095 cpc->hrtimer_active = 0; in perf_mux_hrtimer_handler()
1096 raw_spin_unlock(&cpc->hrtimer_lock); in perf_mux_hrtimer_handler()
1103 struct hrtimer *timer = &cpc->hrtimer; in __perf_mux_hrtimer_init()
1104 struct pmu *pmu = cpc->epc.pmu; in __perf_mux_hrtimer_init()
1111 interval = pmu->hrtimer_interval_ms; in __perf_mux_hrtimer_init()
1113 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; in __perf_mux_hrtimer_init()
1115 cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); in __perf_mux_hrtimer_init()
1117 raw_spin_lock_init(&cpc->hrtimer_lock); in __perf_mux_hrtimer_init()
1119 timer->function = perf_mux_hrtimer_handler; in __perf_mux_hrtimer_init()
1124 struct hrtimer *timer = &cpc->hrtimer; in perf_mux_hrtimer_restart()
1127 raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags); in perf_mux_hrtimer_restart()
1128 if (!cpc->hrtimer_active) { in perf_mux_hrtimer_restart()
1129 cpc->hrtimer_active = 1; in perf_mux_hrtimer_restart()
1130 hrtimer_forward_now(timer, cpc->hrtimer_interval); in perf_mux_hrtimer_restart()
1133 raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags); in perf_mux_hrtimer_restart()
1145 int *count = this_cpu_ptr(pmu->pmu_disable_count); in perf_pmu_disable()
1147 pmu->pmu_disable(pmu); in perf_pmu_disable()
1152 int *count = this_cpu_ptr(pmu->pmu_disable_count); in perf_pmu_enable()
1153 if (!--(*count)) in perf_pmu_enable()
1154 pmu->pmu_enable(pmu); in perf_pmu_enable()
1159 WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) == 0); in perf_assert_pmu_disabled()
1164 refcount_inc(&ctx->refcount); in get_ctx()
1169 if (pmu->task_ctx_cache) in alloc_task_ctx_data()
1170 return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL); in alloc_task_ctx_data()
1177 if (pmu->task_ctx_cache && task_ctx_data) in free_task_ctx_data()
1178 kmem_cache_free(pmu->task_ctx_cache, task_ctx_data); in free_task_ctx_data()
1191 if (refcount_dec_and_test(&ctx->refcount)) { in put_ctx()
1192 if (ctx->parent_ctx) in put_ctx()
1193 put_ctx(ctx->parent_ctx); in put_ctx()
1194 if (ctx->task && ctx->task != TASK_TOMBSTONE) in put_ctx()
1195 put_task_struct(ctx->task); in put_ctx()
1196 call_rcu(&ctx->rcu_head, free_ctx); in put_ctx()
1210 * - perf_event_exit_task_context() [ child , 0 ]
1214 * - perf_event_init_context() [ parent, 0 ]
1222 * While it appears there is an obvious deadlock here -- the parent and child
1224 * life-time rules separate them. That is an exiting task cannot fork, and a
1227 * But remember that these are parent<->child context relations, and
1233 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1244 * means that only external vectors (perf_fops, prctl) can perturb the event
1248 * However; because event->ctx can change while we're waiting to acquire
1249 * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1264 * cpuctx->mutex / perf_event_context::mutex
1273 ctx = READ_ONCE(event->ctx); in perf_event_ctx_lock_nested()
1274 if (!refcount_inc_not_zero(&ctx->refcount)) { in perf_event_ctx_lock_nested()
1280 mutex_lock_nested(&ctx->mutex, nesting); in perf_event_ctx_lock_nested()
1281 if (event->ctx != ctx) { in perf_event_ctx_lock_nested()
1282 mutex_unlock(&ctx->mutex); in perf_event_ctx_lock_nested()
1299 mutex_unlock(&ctx->mutex); in perf_event_ctx_unlock()
1304 * This must be done under the ctx->lock, such as to serialize against
1306 * calling scheduler related locks and ctx->lock nests inside those.
1311 struct perf_event_context *parent_ctx = ctx->parent_ctx; in unclone_ctx()
1313 lockdep_assert_held(&ctx->lock); in unclone_ctx()
1316 ctx->parent_ctx = NULL; in unclone_ctx()
1317 ctx->generation++; in unclone_ctx()
1329 if (event->parent) in perf_event_pid_type()
1330 event = event->parent; in perf_event_pid_type()
1332 nr = __task_pid_nr_ns(p, type, event->ns); in perf_event_pid_type()
1333 /* avoid -1 if it is idle thread or runs in another ns */ in perf_event_pid_type()
1335 nr = -1; in perf_event_pid_type()
1355 u64 id = event->id; in primary_event_id()
1357 if (event->parent) in primary_event_id()
1358 id = event->parent->id; in primary_event_id()
1378 * part of the read side critical section was irqs-enabled -- see in perf_lock_task_context()
1381 * Since ctx->lock nests under rq->lock we must ensure the entire read in perf_lock_task_context()
1386 ctx = rcu_dereference(task->perf_event_ctxp); in perf_lock_task_context()
1398 raw_spin_lock(&ctx->lock); in perf_lock_task_context()
1399 if (ctx != rcu_dereference(task->perf_event_ctxp)) { in perf_lock_task_context()
1400 raw_spin_unlock(&ctx->lock); in perf_lock_task_context()
1406 if (ctx->task == TASK_TOMBSTONE || in perf_lock_task_context()
1407 !refcount_inc_not_zero(&ctx->refcount)) { in perf_lock_task_context()
1408 raw_spin_unlock(&ctx->lock); in perf_lock_task_context()
1411 WARN_ON_ONCE(ctx->task != task); in perf_lock_task_context()
1433 ++ctx->pin_count; in perf_pin_task_context()
1434 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_pin_task_context()
1443 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_unpin_context()
1444 --ctx->pin_count; in perf_unpin_context()
1445 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_unpin_context()
1455 lockdep_assert_held(&ctx->lock); in __update_context_time()
1458 ctx->time += now - ctx->timestamp; in __update_context_time()
1459 ctx->timestamp = now; in __update_context_time()
1462 * The above: time' = time + (now - timestamp), can be re-arranged in __update_context_time()
1463 * into: time` = now + (time - timestamp), which gives a single value in __update_context_time()
1467 * it's (obviously) not possible to acquire ctx->lock in order to read in __update_context_time()
1470 WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp); in __update_context_time()
1480 struct perf_event_context *ctx = event->ctx; in perf_event_time()
1488 return ctx->time; in perf_event_time()
1493 struct perf_event_context *ctx = event->ctx; in perf_event_time_now()
1501 if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) in perf_event_time_now()
1502 return ctx->time; in perf_event_time_now()
1504 now += READ_ONCE(ctx->timeoffset); in perf_event_time_now()
1510 struct perf_event_context *ctx = event->ctx; in get_event_type()
1513 lockdep_assert_held(&ctx->lock); in get_event_type()
1519 if (event->group_leader != event) in get_event_type()
1520 event = event->group_leader; in get_event_type()
1522 event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE; in get_event_type()
1523 if (!ctx->task) in get_event_type()
1534 RB_CLEAR_NODE(&event->group_node); in init_event_group()
1535 event->group_index = 0; in init_event_group()
1545 if (event->attr.pinned) in get_event_groups()
1546 return &ctx->pinned_groups; in get_event_groups()
1548 return &ctx->flexible_groups; in get_event_groups()
1556 groups->tree = RB_ROOT; in perf_event_groups_init()
1557 groups->index = 0; in perf_event_groups_init()
1565 if (event->cgrp) in event_cgroup()
1566 cgroup = event->cgrp->css.cgroup; in event_cgroup()
1583 if (left_cpu < right->cpu) in perf_event_groups_cmp()
1584 return -1; in perf_event_groups_cmp()
1585 if (left_cpu > right->cpu) in perf_event_groups_cmp()
1589 if (left_pmu < right->pmu_ctx->pmu) in perf_event_groups_cmp()
1590 return -1; in perf_event_groups_cmp()
1591 if (left_pmu > right->pmu_ctx->pmu) in perf_event_groups_cmp()
1605 return -1; in perf_event_groups_cmp()
1616 return -1; in perf_event_groups_cmp()
1623 if (left_group_index < right->group_index) in perf_event_groups_cmp()
1624 return -1; in perf_event_groups_cmp()
1625 if (left_group_index > right->group_index) in perf_event_groups_cmp()
1637 return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e), in __group_less()
1638 e->group_index, __node_2_pe(b)) < 0; in __group_less()
1653 return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b); in __group_cmp()
1663 return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b), in __group_cmp_ignore_cgroup()
1664 b->group_index, b); in __group_cmp_ignore_cgroup()
1669 * {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1676 event->group_index = ++groups->index; in perf_event_groups_insert()
1678 rb_add(&event->group_node, &groups->tree, __group_less); in perf_event_groups_insert()
1700 WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) || in perf_event_groups_delete()
1701 RB_EMPTY_ROOT(&groups->tree)); in perf_event_groups_delete()
1703 rb_erase(&event->group_node, &groups->tree); in perf_event_groups_delete()
1733 node = rb_find_first(&key, &groups->tree, __group_cmp); in perf_event_groups_first()
1744 .cpu = event->cpu, in perf_event_groups_next()
1750 next = rb_next_match(&key, &event->group_node, __group_cmp); in perf_event_groups_next()
1765 for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
1767 event = rb_entry_safe(rb_next(&event->group_node), \
1772 * Must be called with ctx->mutex and ctx->lock held.
1777 lockdep_assert_held(&ctx->lock); in list_add_event()
1779 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); in list_add_event()
1780 event->attach_state |= PERF_ATTACH_CONTEXT; in list_add_event()
1782 event->tstamp = perf_event_time(event); in list_add_event()
1789 if (event->group_leader == event) { in list_add_event()
1790 event->group_caps = event->event_caps; in list_add_event()
1794 list_add_rcu(&event->event_entry, &ctx->event_list); in list_add_event()
1795 ctx->nr_events++; in list_add_event()
1796 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) in list_add_event()
1797 ctx->nr_user++; in list_add_event()
1798 if (event->attr.inherit_stat) in list_add_event()
1799 ctx->nr_stat++; in list_add_event()
1801 if (event->state > PERF_EVENT_STATE_OFF) in list_add_event()
1804 ctx->generation++; in list_add_event()
1805 event->pmu_ctx->nr_events++; in list_add_event()
1813 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : in perf_event__state_init()
1853 size += sizeof(data->ip); in __perf_event_header_size()
1856 size += sizeof(data->addr); in __perf_event_header_size()
1859 size += sizeof(data->period); in __perf_event_header_size()
1862 size += sizeof(data->weight.full); in __perf_event_header_size()
1865 size += event->read_size; in __perf_event_header_size()
1868 size += sizeof(data->data_src.val); in __perf_event_header_size()
1871 size += sizeof(data->txn); in __perf_event_header_size()
1874 size += sizeof(data->phys_addr); in __perf_event_header_size()
1877 size += sizeof(data->cgroup); in __perf_event_header_size()
1880 size += sizeof(data->data_page_size); in __perf_event_header_size()
1883 size += sizeof(data->code_page_size); in __perf_event_header_size()
1885 event->header_size = size; in __perf_event_header_size()
1894 event->read_size = in perf_event__header_size()
1895 __perf_event_read_size(event->attr.read_format, in perf_event__header_size()
1896 event->group_leader->nr_siblings); in perf_event__header_size()
1897 __perf_event_header_size(event, event->attr.sample_type); in perf_event__header_size()
1903 u64 sample_type = event->attr.sample_type; in perf_event__id_header_size()
1907 size += sizeof(data->tid_entry); in perf_event__id_header_size()
1910 size += sizeof(data->time); in perf_event__id_header_size()
1913 size += sizeof(data->id); in perf_event__id_header_size()
1916 size += sizeof(data->id); in perf_event__id_header_size()
1919 size += sizeof(data->stream_id); in perf_event__id_header_size()
1922 size += sizeof(data->cpu_entry); in perf_event__id_header_size()
1924 event->id_header_size = size; in perf_event__id_header_size()
1933 * depends on per-event read_format, also (re)check the existing events.
1940 struct perf_event *sibling, *group_leader = event->group_leader; in perf_event_validate_size()
1942 if (__perf_event_read_size(event->attr.read_format, in perf_event_validate_size()
1943 group_leader->nr_siblings + 1) > 16*1024) in perf_event_validate_size()
1946 if (__perf_event_read_size(group_leader->attr.read_format, in perf_event_validate_size()
1947 group_leader->nr_siblings + 1) > 16*1024) in perf_event_validate_size()
1951 * When creating a new group leader, group_leader->ctx is initialized in perf_event_validate_size()
1953 * for_each_sibling_event() until group_leader->ctx is set. A new group in perf_event_validate_size()
1955 * the non-existent siblings. in perf_event_validate_size()
1961 if (__perf_event_read_size(sibling->attr.read_format, in perf_event_validate_size()
1962 group_leader->nr_siblings + 1) > 16*1024) in perf_event_validate_size()
1971 struct perf_event *group_leader = event->group_leader, *pos; in perf_group_attach()
1973 lockdep_assert_held(&event->ctx->lock); in perf_group_attach()
1979 if (event->attach_state & PERF_ATTACH_GROUP) in perf_group_attach()
1982 event->attach_state |= PERF_ATTACH_GROUP; in perf_group_attach()
1987 WARN_ON_ONCE(group_leader->ctx != event->ctx); in perf_group_attach()
1989 group_leader->group_caps &= event->event_caps; in perf_group_attach()
1991 list_add_tail(&event->sibling_list, &group_leader->sibling_list); in perf_group_attach()
1992 group_leader->nr_siblings++; in perf_group_attach()
1993 group_leader->group_generation++; in perf_group_attach()
2003 * Must be called with ctx->mutex and ctx->lock held.
2008 WARN_ON_ONCE(event->ctx != ctx); in list_del_event()
2009 lockdep_assert_held(&ctx->lock); in list_del_event()
2012 * We can have double detach due to exit/hot-unplug + close. in list_del_event()
2014 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) in list_del_event()
2017 event->attach_state &= ~PERF_ATTACH_CONTEXT; in list_del_event()
2019 ctx->nr_events--; in list_del_event()
2020 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) in list_del_event()
2021 ctx->nr_user--; in list_del_event()
2022 if (event->attr.inherit_stat) in list_del_event()
2023 ctx->nr_stat--; in list_del_event()
2025 list_del_rcu(&event->event_entry); in list_del_event()
2027 if (event->group_leader == event) in list_del_event()
2034 * of error state is by explicit re-enabling in list_del_event()
2037 if (event->state > PERF_EVENT_STATE_OFF) { in list_del_event()
2042 ctx->generation++; in list_del_event()
2043 event->pmu_ctx->nr_events--; in list_del_event()
2052 if (!event->pmu->aux_output_match) in perf_aux_output_match()
2055 return event->pmu->aux_output_match(aux_event); in perf_aux_output_match()
2064 struct perf_event_context *ctx = event->ctx; in perf_put_aux_event()
2070 if (event->aux_event) { in perf_put_aux_event()
2071 iter = event->aux_event; in perf_put_aux_event()
2072 event->aux_event = NULL; in perf_put_aux_event()
2081 for_each_sibling_event(iter, event->group_leader) { in perf_put_aux_event()
2082 if (iter->aux_event != event) in perf_put_aux_event()
2085 iter->aux_event = NULL; in perf_put_aux_event()
2100 return !!event->attr.aux_output || !!event->attr.aux_sample_size; in perf_need_aux_event()
2118 if (event->attr.aux_output && event->attr.aux_sample_size) in perf_get_aux_event()
2121 if (event->attr.aux_output && in perf_get_aux_event()
2125 if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux) in perf_get_aux_event()
2128 if (!atomic_long_inc_not_zero(&group_leader->refcount)) in perf_get_aux_event()
2137 event->aux_event = group_leader; in perf_get_aux_event()
2144 return event->attr.pinned ? &event->pmu_ctx->pinned_active : in get_event_list()
2145 &event->pmu_ctx->flexible_active; in get_event_list()
2156 event_sched_out(event, event->ctx); in perf_remove_sibling_event()
2162 struct perf_event *leader = event->group_leader; in perf_group_detach()
2164 struct perf_event_context *ctx = event->ctx; in perf_group_detach()
2166 lockdep_assert_held(&ctx->lock); in perf_group_detach()
2169 * We can have double detach due to exit/hot-unplug + close. in perf_group_detach()
2171 if (!(event->attach_state & PERF_ATTACH_GROUP)) in perf_group_detach()
2174 event->attach_state &= ~PERF_ATTACH_GROUP; in perf_group_detach()
2182 list_del_init(&event->sibling_list); in perf_group_detach()
2183 event->group_leader->nr_siblings--; in perf_group_detach()
2184 event->group_leader->group_generation++; in perf_group_detach()
2193 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { in perf_group_detach()
2195 if (sibling->event_caps & PERF_EV_CAP_SIBLING) in perf_group_detach()
2198 sibling->group_leader = sibling; in perf_group_detach()
2199 list_del_init(&sibling->sibling_list); in perf_group_detach()
2202 sibling->group_caps = event->group_caps; in perf_group_detach()
2204 if (sibling->attach_state & PERF_ATTACH_CONTEXT) { in perf_group_detach()
2205 add_event_to_groups(sibling, event->ctx); in perf_group_detach()
2207 if (sibling->state == PERF_EVENT_STATE_ACTIVE) in perf_group_detach()
2208 list_add_tail(&sibling->active_list, get_event_list(sibling)); in perf_group_detach()
2211 WARN_ON_ONCE(sibling->ctx != event->ctx); in perf_group_detach()
2225 struct perf_event *parent_event = event->parent; in perf_child_detach()
2227 if (!(event->attach_state & PERF_ATTACH_CHILD)) in perf_child_detach()
2230 event->attach_state &= ~PERF_ATTACH_CHILD; in perf_child_detach()
2235 lockdep_assert_held(&parent_event->child_mutex); in perf_child_detach()
2238 list_del_init(&event->child_list); in perf_child_detach()
2243 return event->state == PERF_EVENT_STATE_DEAD; in is_orphaned_event()
2249 return (event->cpu == -1 || event->cpu == smp_processor_id()) && in event_filter_match()
2256 struct perf_event_pmu_context *epc = event->pmu_ctx; in event_sched_out()
2257 struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context); in event_sched_out()
2260 // XXX cpc serialization, probably per-cpu IRQ disabled in event_sched_out()
2262 WARN_ON_ONCE(event->ctx != ctx); in event_sched_out()
2263 lockdep_assert_held(&ctx->lock); in event_sched_out()
2265 if (event->state != PERF_EVENT_STATE_ACTIVE) in event_sched_out()
2273 list_del_init(&event->active_list); in event_sched_out()
2275 perf_pmu_disable(event->pmu); in event_sched_out()
2277 event->pmu->del(event, 0); in event_sched_out()
2278 event->oncpu = -1; in event_sched_out()
2280 if (event->pending_disable) { in event_sched_out()
2281 event->pending_disable = 0; in event_sched_out()
2286 if (event->pending_sigtrap) { in event_sched_out()
2289 event->pending_sigtrap = 0; in event_sched_out()
2291 !event->pending_work) { in event_sched_out()
2292 event->pending_work = 1; in event_sched_out()
2294 WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount)); in event_sched_out()
2295 task_work_add(current, &event->pending_task, TWA_RESUME); in event_sched_out()
2298 local_dec(&event->ctx->nr_pending); in event_sched_out()
2304 cpc->active_oncpu--; in event_sched_out()
2305 if (event->attr.freq && event->attr.sample_freq) in event_sched_out()
2306 ctx->nr_freq--; in event_sched_out()
2307 if (event->attr.exclusive || !cpc->active_oncpu) in event_sched_out()
2308 cpc->exclusive = 0; in event_sched_out()
2310 perf_pmu_enable(event->pmu); in event_sched_out()
2318 if (group_event->state != PERF_EVENT_STATE_ACTIVE) in group_sched_out()
2321 perf_assert_pmu_disabled(group_event->pmu_ctx->pmu); in group_sched_out()
2348 struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx; in __perf_remove_from_context()
2351 if (ctx->is_active & EVENT_TIME) { in __perf_remove_from_context()
2361 event->pending_disable = 1; in __perf_remove_from_context()
2369 event->state = PERF_EVENT_STATE_DEAD; in __perf_remove_from_context()
2371 if (!pmu_ctx->nr_events) { in __perf_remove_from_context()
2372 pmu_ctx->rotate_necessary = 0; in __perf_remove_from_context()
2374 if (ctx->task && ctx->is_active) { in __perf_remove_from_context()
2377 cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context); in __perf_remove_from_context()
2378 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); in __perf_remove_from_context()
2379 cpc->task_epc = NULL; in __perf_remove_from_context()
2383 if (!ctx->nr_events && ctx->is_active) { in __perf_remove_from_context()
2384 if (ctx == &cpuctx->ctx) in __perf_remove_from_context()
2387 ctx->is_active = 0; in __perf_remove_from_context()
2388 if (ctx->task) { in __perf_remove_from_context()
2389 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in __perf_remove_from_context()
2390 cpuctx->task_ctx = NULL; in __perf_remove_from_context()
2398 * If event->ctx is a cloned context, callers must make sure that
2399 * every task struct that event->ctx->task could possibly point to
2401 * that only calls us on the top-level context, which can't be a clone.
2407 struct perf_event_context *ctx = event->ctx; in perf_remove_from_context()
2409 lockdep_assert_held(&ctx->mutex); in perf_remove_from_context()
2416 raw_spin_lock_irq(&ctx->lock); in perf_remove_from_context()
2417 if (!ctx->is_active) { in perf_remove_from_context()
2420 raw_spin_unlock_irq(&ctx->lock); in perf_remove_from_context()
2423 raw_spin_unlock_irq(&ctx->lock); in perf_remove_from_context()
2436 if (event->state < PERF_EVENT_STATE_INACTIVE) in __perf_event_disable()
2439 if (ctx->is_active & EVENT_TIME) { in __perf_event_disable()
2444 perf_pmu_disable(event->pmu_ctx->pmu); in __perf_event_disable()
2446 if (event == event->group_leader) in __perf_event_disable()
2454 perf_pmu_enable(event->pmu_ctx->pmu); in __perf_event_disable()
2460 * If event->ctx is a cloned context, callers must make sure that
2461 * every task struct that event->ctx->task could possibly point to
2464 * hold the top-level event's child_mutex, so any descendant that
2467 * When called from perf_pending_irq it's OK because event->ctx
2473 struct perf_event_context *ctx = event->ctx; in _perf_event_disable()
2475 raw_spin_lock_irq(&ctx->lock); in _perf_event_disable()
2476 if (event->state <= PERF_EVENT_STATE_OFF) { in _perf_event_disable()
2477 raw_spin_unlock_irq(&ctx->lock); in _perf_event_disable()
2480 raw_spin_unlock_irq(&ctx->lock); in _perf_event_disable()
2506 event->pending_disable = 1; in perf_event_disable_inatomic()
2507 irq_work_queue(&event->pending_irq); in perf_event_disable_inatomic()
2518 struct perf_event_pmu_context *epc = event->pmu_ctx; in event_sched_in()
2519 struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context); in event_sched_in()
2522 WARN_ON_ONCE(event->ctx != ctx); in event_sched_in()
2524 lockdep_assert_held(&ctx->lock); in event_sched_in()
2526 if (event->state <= PERF_EVENT_STATE_OFF) in event_sched_in()
2529 WRITE_ONCE(event->oncpu, smp_processor_id()); in event_sched_in()
2533 * ->oncpu if it sees ACTIVE. in event_sched_in()
2543 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { in event_sched_in()
2545 event->hw.interrupts = 0; in event_sched_in()
2548 perf_pmu_disable(event->pmu); in event_sched_in()
2552 if (event->pmu->add(event, PERF_EF_START)) { in event_sched_in()
2554 event->oncpu = -1; in event_sched_in()
2555 ret = -EAGAIN; in event_sched_in()
2560 cpc->active_oncpu++; in event_sched_in()
2561 if (event->attr.freq && event->attr.sample_freq) in event_sched_in()
2562 ctx->nr_freq++; in event_sched_in()
2564 if (event->attr.exclusive) in event_sched_in()
2565 cpc->exclusive = 1; in event_sched_in()
2568 perf_pmu_enable(event->pmu); in event_sched_in()
2577 struct pmu *pmu = group_event->pmu_ctx->pmu; in group_sched_in()
2579 if (group_event->state == PERF_EVENT_STATE_OFF) in group_sched_in()
2582 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); in group_sched_in()
2597 if (!pmu->commit_txn(pmu)) in group_sched_in()
2615 pmu->cancel_txn(pmu); in group_sched_in()
2616 return -EAGAIN; in group_sched_in()
2624 struct perf_event_pmu_context *epc = event->pmu_ctx; in group_can_go_on()
2625 struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context); in group_can_go_on()
2630 if (event->group_caps & PERF_EV_CAP_SOFTWARE) in group_can_go_on()
2636 if (cpc->exclusive) in group_can_go_on()
2642 if (event->attr.exclusive && !list_empty(get_event_list(event))) in group_can_go_on()
2663 if (!cpuctx->task_ctx) in task_ctx_sched_out()
2666 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) in task_ctx_sched_out()
2675 ctx_sched_in(&cpuctx->ctx, EVENT_PINNED); in perf_event_sched_in()
2678 ctx_sched_in(&cpuctx->ctx, EVENT_FLEXIBLE); in perf_event_sched_in()
2685 * - CPU pinned (EVENT_CPU | EVENT_PINNED)
2686 * - task pinned (EVENT_PINNED)
2687 * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2688 * - task flexible (EVENT_FLEXIBLE).
2718 perf_ctx_disable(&cpuctx->ctx, false); in ctx_resched()
2727 * - EVENT_CPU: schedule out corresponding groups; in ctx_resched()
2728 * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; in ctx_resched()
2729 * - otherwise, do nothing more. in ctx_resched()
2732 ctx_sched_out(&cpuctx->ctx, event_type); in ctx_resched()
2734 ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE); in ctx_resched()
2738 perf_ctx_enable(&cpuctx->ctx, false); in ctx_resched()
2746 struct perf_event_context *task_ctx = cpuctx->task_ctx; in perf_pmu_resched()
2757 * things like ctx->is_active and cpuctx->task_ctx are set.
2762 struct perf_event_context *ctx = event->ctx; in __perf_install_in_context()
2764 struct perf_event_context *task_ctx = cpuctx->task_ctx; in __perf_install_in_context()
2768 raw_spin_lock(&cpuctx->ctx.lock); in __perf_install_in_context()
2769 if (ctx->task) { in __perf_install_in_context()
2770 raw_spin_lock(&ctx->lock); in __perf_install_in_context()
2773 reprogram = (ctx->task == current); in __perf_install_in_context()
2779 * If its not running, we don't care, ctx->lock will in __perf_install_in_context()
2782 if (task_curr(ctx->task) && !reprogram) { in __perf_install_in_context()
2783 ret = -ESRCH; in __perf_install_in_context()
2787 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); in __perf_install_in_context()
2789 raw_spin_lock(&task_ctx->lock); in __perf_install_in_context()
2793 if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) { in __perf_install_in_context()
2799 reprogram = cgroup_is_descendant(cgrp->css.cgroup, in __perf_install_in_context()
2800 event->cgrp->css.cgroup); in __perf_install_in_context()
2831 struct task_struct *task = READ_ONCE(ctx->task); in perf_install_in_context()
2833 lockdep_assert_held(&ctx->mutex); in perf_install_in_context()
2837 if (event->cpu != -1) in perf_install_in_context()
2838 WARN_ON_ONCE(event->cpu != cpu); in perf_install_in_context()
2841 * Ensures that if we can observe event->ctx, both the event and ctx in perf_install_in_context()
2844 smp_store_release(&event->ctx, ctx); in perf_install_in_context()
2848 * without IPI. Except when this is the first event for the context, in in perf_install_in_context()
2849 * that case we need the magic of the IPI to set ctx->is_active. in perf_install_in_context()
2852 * event will issue the IPI and reprogram the hardware. in perf_install_in_context()
2855 ctx->nr_events && !is_cgroup_event(event)) { in perf_install_in_context()
2856 raw_spin_lock_irq(&ctx->lock); in perf_install_in_context()
2857 if (ctx->task == TASK_TOMBSTONE) { in perf_install_in_context()
2858 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2862 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2878 * Installing events is tricky because we cannot rely on ctx->is_active in perf_install_in_context()
2879 * to be set in case this is the nr_events 0 -> 1 transition. in perf_install_in_context()
2893 * our task->perf_event_ctxp[] store, such that it will in fact take in perf_install_in_context()
2896 * We do this by task_function_call(), if the IPI fails to hit the task in perf_install_in_context()
2902 * This smp_mb() orders the task->perf_event_ctxp[] store with the in perf_install_in_context()
2903 * task_cpu() load, such that if the IPI then does not find the task in perf_install_in_context()
2912 raw_spin_lock_irq(&ctx->lock); in perf_install_in_context()
2913 task = ctx->task; in perf_install_in_context()
2917 * cannot happen), and we hold ctx->mutex, which serializes us in perf_install_in_context()
2920 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2924 * If the task is not running, ctx->lock will avoid it becoming so, in perf_install_in_context()
2928 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2932 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2943 struct perf_event *leader = event->group_leader; in __perf_event_enable()
2946 if (event->state >= PERF_EVENT_STATE_INACTIVE || in __perf_event_enable()
2947 event->state <= PERF_EVENT_STATE_ERROR) in __perf_event_enable()
2950 if (ctx->is_active) in __perf_event_enable()
2956 if (!ctx->is_active) in __perf_event_enable()
2968 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) { in __perf_event_enable()
2973 task_ctx = cpuctx->task_ctx; in __perf_event_enable()
2974 if (ctx->task) in __perf_event_enable()
2983 * If event->ctx is a cloned context, callers must make sure that
2984 * every task struct that event->ctx->task could possibly point to
2991 struct perf_event_context *ctx = event->ctx; in _perf_event_enable()
2993 raw_spin_lock_irq(&ctx->lock); in _perf_event_enable()
2994 if (event->state >= PERF_EVENT_STATE_INACTIVE || in _perf_event_enable()
2995 event->state < PERF_EVENT_STATE_ERROR) { in _perf_event_enable()
2997 raw_spin_unlock_irq(&ctx->lock); in _perf_event_enable()
3006 * been scheduled away before the cross-call arrived. in _perf_event_enable()
3008 if (event->state == PERF_EVENT_STATE_ERROR) { in _perf_event_enable()
3012 if (event->event_caps & PERF_EV_CAP_SIBLING && in _perf_event_enable()
3013 event->group_leader == event) in _perf_event_enable()
3016 event->state = PERF_EVENT_STATE_OFF; in _perf_event_enable()
3018 raw_spin_unlock_irq(&ctx->lock); in _perf_event_enable()
3044 struct perf_event *event = sd->event; in __perf_event_stop()
3047 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) in __perf_event_stop()
3057 if (READ_ONCE(event->oncpu) != smp_processor_id()) in __perf_event_stop()
3058 return -EAGAIN; in __perf_event_stop()
3060 event->pmu->stop(event, PERF_EF_UPDATE); in __perf_event_stop()
3068 * Since this is happening on an event-local CPU, no trace is lost in __perf_event_stop()
3071 if (sd->restart) in __perf_event_stop()
3072 event->pmu->start(event, 0); in __perf_event_stop()
3086 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) in perf_event_stop()
3094 * inactive here (event->oncpu==-1), there's nothing more to do; in perf_event_stop()
3095 * fall through with ret==-ENXIO. in perf_event_stop()
3097 ret = cpu_function_call(READ_ONCE(event->oncpu), in perf_event_stop()
3099 } while (ret == -EAGAIN); in perf_event_stop()
3118 * pre-existing mappings, called once when new filters arrive via SET_FILTER
3133 raw_spin_lock(&ifh->lock); in perf_event_addr_filters_sync()
3134 if (event->addr_filters_gen != event->hw.addr_filters_gen) { in perf_event_addr_filters_sync()
3135 event->pmu->addr_filters_sync(event); in perf_event_addr_filters_sync()
3136 event->hw.addr_filters_gen = event->addr_filters_gen; in perf_event_addr_filters_sync()
3138 raw_spin_unlock(&ifh->lock); in perf_event_addr_filters_sync()
3147 if (event->attr.inherit || !is_sampling_event(event)) in _perf_event_refresh()
3148 return -EINVAL; in _perf_event_refresh()
3150 atomic_add(refresh, &event->event_limit); in _perf_event_refresh()
3181 if (!bp->attr.disabled) in perf_event_modify_breakpoint()
3188 * Copy event-type-independent attributes that may be modified.
3193 to->sig_data = from->sig_data; in perf_event_modify_copy_attr()
3203 if (event->attr.type != attr->type) in perf_event_modify_attr()
3204 return -EINVAL; in perf_event_modify_attr()
3206 switch (event->attr.type) { in perf_event_modify_attr()
3212 return -EOPNOTSUPP; in perf_event_modify_attr()
3215 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_event_modify_attr()
3217 mutex_lock(&event->child_mutex); in perf_event_modify_attr()
3219 * Event-type-independent attributes must be copied before event-type in perf_event_modify_attr()
3223 perf_event_modify_copy_attr(&event->attr, attr); in perf_event_modify_attr()
3227 list_for_each_entry(child, &event->child_list, child_list) { in perf_event_modify_attr()
3228 perf_event_modify_copy_attr(&child->attr, attr); in perf_event_modify_attr()
3234 mutex_unlock(&event->child_mutex); in perf_event_modify_attr()
3241 struct perf_event_context *ctx = pmu_ctx->ctx; in __pmu_ctx_sched_out()
3243 struct pmu *pmu = pmu_ctx->pmu; in __pmu_ctx_sched_out()
3245 if (ctx->task && !ctx->is_active) { in __pmu_ctx_sched_out()
3248 cpc = this_cpu_ptr(pmu->cpu_pmu_context); in __pmu_ctx_sched_out()
3249 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); in __pmu_ctx_sched_out()
3250 cpc->task_epc = NULL; in __pmu_ctx_sched_out()
3259 &pmu_ctx->pinned_active, in __pmu_ctx_sched_out()
3266 &pmu_ctx->flexible_active, in __pmu_ctx_sched_out()
3274 pmu_ctx->rotate_necessary = 0; in __pmu_ctx_sched_out()
3284 int is_active = ctx->is_active; in ctx_sched_out()
3289 lockdep_assert_held(&ctx->lock); in ctx_sched_out()
3291 if (likely(!ctx->nr_events)) { in ctx_sched_out()
3295 WARN_ON_ONCE(ctx->is_active); in ctx_sched_out()
3296 if (ctx->task) in ctx_sched_out()
3297 WARN_ON_ONCE(cpuctx->task_ctx); in ctx_sched_out()
3314 update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx); in ctx_sched_out()
3316 * CPU-release for the below ->is_active store, in ctx_sched_out()
3322 ctx->is_active &= ~event_type; in ctx_sched_out()
3323 if (!(ctx->is_active & EVENT_ALL)) in ctx_sched_out()
3324 ctx->is_active = 0; in ctx_sched_out()
3326 if (ctx->task) { in ctx_sched_out()
3327 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_out()
3328 if (!ctx->is_active) in ctx_sched_out()
3329 cpuctx->task_ctx = NULL; in ctx_sched_out()
3332 is_active ^= ctx->is_active; /* changed bits */ in ctx_sched_out()
3334 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in ctx_sched_out()
3335 if (cgroup && !pmu_ctx->nr_cgroups) in ctx_sched_out()
3352 lockdep_assert_held(&ctx1->lock); in context_equiv()
3353 lockdep_assert_held(&ctx2->lock); in context_equiv()
3356 if (ctx1->pin_count || ctx2->pin_count) in context_equiv()
3360 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) in context_equiv()
3364 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) in context_equiv()
3371 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && in context_equiv()
3372 ctx1->parent_gen == ctx2->parent_gen) in context_equiv()
3384 if (!event->attr.inherit_stat) in __perf_event_sync_stat()
3394 if (event->state == PERF_EVENT_STATE_ACTIVE) in __perf_event_sync_stat()
3395 event->pmu->read(event); in __perf_event_sync_stat()
3400 * In order to keep per-task stats reliable we need to flip the event in __perf_event_sync_stat()
3403 value = local64_read(&next_event->count); in __perf_event_sync_stat()
3404 value = local64_xchg(&event->count, value); in __perf_event_sync_stat()
3405 local64_set(&next_event->count, value); in __perf_event_sync_stat()
3407 swap(event->total_time_enabled, next_event->total_time_enabled); in __perf_event_sync_stat()
3408 swap(event->total_time_running, next_event->total_time_running); in __perf_event_sync_stat()
3422 if (!ctx->nr_stat) in perf_event_sync_stat()
3427 event = list_first_entry(&ctx->event_list, in perf_event_sync_stat()
3430 next_event = list_first_entry(&next_ctx->event_list, in perf_event_sync_stat()
3433 while (&event->event_entry != &ctx->event_list && in perf_event_sync_stat()
3434 &next_event->event_entry != &next_ctx->event_list) { in perf_event_sync_stat()
3456 if (!prev_ctx->nr_task_data) in perf_event_swap_task_ctx_data()
3460 &prev_ctx->pmu_ctx_list, &next_ctx->pmu_ctx_list, in perf_event_swap_task_ctx_data()
3463 if (WARN_ON_ONCE(prev_epc->pmu != next_epc->pmu)) in perf_event_swap_task_ctx_data()
3472 if (prev_epc->pmu->swap_task_ctx) in perf_event_swap_task_ctx_data()
3473 prev_epc->pmu->swap_task_ctx(prev_epc, next_epc); in perf_event_swap_task_ctx_data()
3475 swap(prev_epc->task_ctx_data, next_epc->task_ctx_data); in perf_event_swap_task_ctx_data()
3484 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in perf_ctx_sched_task_cb()
3485 cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context); in perf_ctx_sched_task_cb()
3487 if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) in perf_ctx_sched_task_cb()
3488 pmu_ctx->pmu->sched_task(pmu_ctx, sched_in); in perf_ctx_sched_task_cb()
3495 struct perf_event_context *ctx = task->perf_event_ctxp; in perf_event_context_sched_out()
3504 next_ctx = rcu_dereference(next->perf_event_ctxp); in perf_event_context_sched_out()
3508 parent = rcu_dereference(ctx->parent_ctx); in perf_event_context_sched_out()
3509 next_parent = rcu_dereference(next_ctx->parent_ctx); in perf_event_context_sched_out()
3520 * lock (including re-checking that neither has been in perf_event_context_sched_out()
3525 raw_spin_lock(&ctx->lock); in perf_event_context_sched_out()
3526 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); in perf_event_context_sched_out()
3531 /* PMIs are disabled; ctx->nr_pending is stable. */ in perf_event_context_sched_out()
3532 if (local_read(&ctx->nr_pending) || in perf_event_context_sched_out()
3533 local_read(&next_ctx->nr_pending)) { in perf_event_context_sched_out()
3536 * events that rely on the ctx->task relation. in perf_event_context_sched_out()
3538 raw_spin_unlock(&next_ctx->lock); in perf_event_context_sched_out()
3543 WRITE_ONCE(ctx->task, next); in perf_event_context_sched_out()
3544 WRITE_ONCE(next_ctx->task, task); in perf_event_context_sched_out()
3554 * ctx->task and ctx->task_ctx_data are immaterial in perf_event_context_sched_out()
3556 * ctx->lock which we're now holding. in perf_event_context_sched_out()
3558 RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx); in perf_event_context_sched_out()
3559 RCU_INIT_POINTER(next->perf_event_ctxp, ctx); in perf_event_context_sched_out()
3565 raw_spin_unlock(&next_ctx->lock); in perf_event_context_sched_out()
3566 raw_spin_unlock(&ctx->lock); in perf_event_context_sched_out()
3572 raw_spin_lock(&ctx->lock); in perf_event_context_sched_out()
3580 raw_spin_unlock(&ctx->lock); in perf_event_context_sched_out()
3589 struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context); in perf_sched_cb_dec()
3594 if (!--cpc->sched_cb_usage) in perf_sched_cb_dec()
3595 list_del(&cpc->sched_cb_entry); in perf_sched_cb_dec()
3601 struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context); in perf_sched_cb_inc()
3603 if (!cpc->sched_cb_usage++) in perf_sched_cb_inc()
3604 list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); in perf_sched_cb_inc()
3614 * This callback is relevant even to per-cpu events; for example multi event
3623 pmu = cpc->epc.pmu; in __perf_pmu_sched_task()
3626 if (WARN_ON_ONCE(!pmu->sched_task)) in __perf_pmu_sched_task()
3629 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3632 pmu->sched_task(cpc->task_epc, sched_in); in __perf_pmu_sched_task()
3635 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3645 /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ in perf_pmu_sched_task()
3646 if (prev == next || cpuctx->task_ctx) in perf_pmu_sched_task()
3660 * We stop each event and update the event value in event->count.
3681 * cgroup event are system-wide mode only in __perf_event_task_sched_out()
3691 return le->group_index < re->group_index; in perf_less_group_idx()
3709 struct perf_event **itrs = heap->data; in __heap_add()
3712 itrs[heap->nr] = event; in __heap_add()
3713 heap->nr++; in __heap_add()
3721 if (!pmu_ctx->ctx->task) in __link_epc()
3724 cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context); in __link_epc()
3725 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); in __link_epc()
3726 cpc->task_epc = pmu_ctx; in __link_epc()
3745 if (pmu->filter && pmu->filter(pmu, cpu)) in visit_groups_merge()
3748 if (!ctx->task) { in visit_groups_merge()
3751 .data = cpuctx->heap, in visit_groups_merge()
3753 .size = cpuctx->heap_size, in visit_groups_merge()
3756 lockdep_assert_held(&cpuctx->ctx.lock); in visit_groups_merge()
3759 if (cpuctx->cgrp) in visit_groups_merge()
3760 css = &cpuctx->cgrp->css; in visit_groups_merge()
3769 __heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL)); in visit_groups_merge()
3776 for (; css; css = css->parent) in visit_groups_merge()
3777 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup)); in visit_groups_merge()
3781 __link_epc((*evt)->pmu_ctx); in visit_groups_merge()
3782 perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu); in visit_groups_merge()
3803 * Because the userpage is strictly per-event (there is no concept of context,
3805 * when context time starts :-(
3811 if (likely(!atomic_read(&event->mmap_count))) in event_update_userpage()
3833 struct perf_event_context *ctx = event->ctx; in merge_sched_in()
3836 if (event->state <= PERF_EVENT_STATE_OFF) in merge_sched_in()
3844 list_add_tail(&event->active_list, get_event_list(event)); in merge_sched_in()
3847 if (event->state == PERF_EVENT_STATE_INACTIVE) { in merge_sched_in()
3849 if (event->attr.pinned) { in merge_sched_in()
3855 event->pmu_ctx->rotate_necessary = 1; in merge_sched_in()
3856 cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context); in merge_sched_in()
3880 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in ctx_groups_sched_in()
3881 if (cgroup && !pmu_ctx->nr_cgroups) in ctx_groups_sched_in()
3883 pmu_groups_sched_in(ctx, groups, pmu_ctx->pmu); in ctx_groups_sched_in()
3890 pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu); in __pmu_ctx_sched_in()
3897 int is_active = ctx->is_active; in ctx_sched_in()
3902 lockdep_assert_held(&ctx->lock); in ctx_sched_in()
3904 if (likely(!ctx->nr_events)) in ctx_sched_in()
3912 * CPU-release for the below ->is_active store, in ctx_sched_in()
3918 ctx->is_active |= (event_type | EVENT_TIME); in ctx_sched_in()
3919 if (ctx->task) { in ctx_sched_in()
3921 cpuctx->task_ctx = ctx; in ctx_sched_in()
3923 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_in()
3926 is_active ^= ctx->is_active; /* changed bits */ in ctx_sched_in()
3933 ctx_groups_sched_in(ctx, &ctx->pinned_groups, cgroup); in ctx_sched_in()
3937 ctx_groups_sched_in(ctx, &ctx->flexible_groups, cgroup); in ctx_sched_in()
3946 ctx = rcu_dereference(task->perf_event_ctxp); in perf_event_context_sched_in()
3950 if (cpuctx->task_ctx == ctx) { in perf_event_context_sched_in()
3963 * We must check ctx->nr_events while holding ctx->lock, such in perf_event_context_sched_in()
3966 if (!ctx->nr_events) in perf_event_context_sched_in()
3978 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) { in perf_event_context_sched_in()
3979 perf_ctx_disable(&cpuctx->ctx, false); in perf_event_context_sched_in()
3980 ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE); in perf_event_context_sched_in()
3985 perf_ctx_sched_task_cb(cpuctx->task_ctx, true); in perf_event_context_sched_in()
3987 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) in perf_event_context_sched_in()
3988 perf_ctx_enable(&cpuctx->ctx, false); in perf_event_context_sched_in()
4023 u64 frequency = event->attr.sample_freq; in perf_calculate_period()
4039 * period = ------------------- in perf_calculate_period()
4052 a##_fls--; \ in perf_calculate_period()
4055 b##_fls--; \ in perf_calculate_period()
4099 struct hw_perf_event *hwc = &event->hw; in perf_adjust_period()
4105 delta = (s64)(period - hwc->sample_period); in perf_adjust_period()
4108 sample_period = hwc->sample_period + delta; in perf_adjust_period()
4113 hwc->sample_period = sample_period; in perf_adjust_period()
4115 if (local64_read(&hwc->period_left) > 8*sample_period) { in perf_adjust_period()
4117 event->pmu->stop(event, PERF_EF_UPDATE); in perf_adjust_period()
4119 local64_set(&hwc->period_left, 0); in perf_adjust_period()
4122 event->pmu->start(event, PERF_EF_RELOAD); in perf_adjust_period()
4141 * - context have events in frequency mode (needs freq adjust) in perf_adjust_freq_unthr_context()
4142 * - there are events to unthrottle on this cpu in perf_adjust_freq_unthr_context()
4144 if (!(ctx->nr_freq || unthrottle)) in perf_adjust_freq_unthr_context()
4147 raw_spin_lock(&ctx->lock); in perf_adjust_freq_unthr_context()
4149 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { in perf_adjust_freq_unthr_context()
4150 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_adjust_freq_unthr_context()
4153 // XXX use visit thingy to avoid the -1,cpu match in perf_adjust_freq_unthr_context()
4157 perf_pmu_disable(event->pmu); in perf_adjust_freq_unthr_context()
4159 hwc = &event->hw; in perf_adjust_freq_unthr_context()
4161 if (hwc->interrupts == MAX_INTERRUPTS) { in perf_adjust_freq_unthr_context()
4162 hwc->interrupts = 0; in perf_adjust_freq_unthr_context()
4164 event->pmu->start(event, 0); in perf_adjust_freq_unthr_context()
4167 if (!event->attr.freq || !event->attr.sample_freq) in perf_adjust_freq_unthr_context()
4171 * stop the event and update event->count in perf_adjust_freq_unthr_context()
4173 event->pmu->stop(event, PERF_EF_UPDATE); in perf_adjust_freq_unthr_context()
4175 now = local64_read(&event->count); in perf_adjust_freq_unthr_context()
4176 delta = now - hwc->freq_count_stamp; in perf_adjust_freq_unthr_context()
4177 hwc->freq_count_stamp = now; in perf_adjust_freq_unthr_context()
4189 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); in perf_adjust_freq_unthr_context()
4191 perf_pmu_enable(event->pmu); in perf_adjust_freq_unthr_context()
4194 raw_spin_unlock(&ctx->lock); in perf_adjust_freq_unthr_context()
4203 * Rotate the first entry last of non-pinned groups. Rotation might be in rotate_ctx()
4206 if (ctx->rotate_disable) in rotate_ctx()
4209 perf_event_groups_delete(&ctx->flexible_groups, event); in rotate_ctx()
4210 perf_event_groups_insert(&ctx->flexible_groups, event); in rotate_ctx()
4221 .pmu = pmu_ctx->pmu, in ctx_event_to_rotate()
4225 event = list_first_entry_or_null(&pmu_ctx->flexible_active, in ctx_event_to_rotate()
4231 tree = &pmu_ctx->ctx->flexible_groups.tree; in ctx_event_to_rotate()
4233 if (!pmu_ctx->ctx->task) { in ctx_event_to_rotate()
4242 key.cpu = -1; in ctx_event_to_rotate()
4259 pmu_ctx->rotate_necessary = 0; in ctx_event_to_rotate()
4277 cpu_epc = &cpc->epc; in perf_rotate_context()
4278 pmu = cpu_epc->pmu; in perf_rotate_context()
4279 task_epc = cpc->task_epc; in perf_rotate_context()
4281 cpu_rotate = cpu_epc->rotate_necessary; in perf_rotate_context()
4282 task_rotate = task_epc ? task_epc->rotate_necessary : 0; in perf_rotate_context()
4287 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4300 update_context_time(task_epc->ctx); in perf_rotate_context()
4305 update_context_time(&cpuctx->ctx); in perf_rotate_context()
4307 rotate_ctx(&cpuctx->ctx, cpu_event); in perf_rotate_context()
4308 __pmu_ctx_sched_in(&cpuctx->ctx, pmu); in perf_rotate_context()
4312 rotate_ctx(task_epc->ctx, task_event); in perf_rotate_context()
4315 __pmu_ctx_sched_in(task_epc->ctx, pmu); in perf_rotate_context()
4318 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4335 perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled); in perf_event_task_tick()
4338 ctx = rcu_dereference(current->perf_event_ctxp); in perf_event_task_tick()
4347 if (!event->attr.enable_on_exec) in event_enable_on_exec()
4350 event->attr.enable_on_exec = 0; in event_enable_on_exec()
4351 if (event->state >= PERF_EVENT_STATE_INACTIVE) in event_enable_on_exec()
4360 * Enable all of a task's events that have been marked enable-on-exec.
4373 if (WARN_ON_ONCE(current->perf_event_ctxp != ctx)) in perf_event_enable_on_exec()
4376 if (!ctx->nr_events) in perf_event_enable_on_exec()
4383 list_for_each_entry(event, &ctx->event_list, event_entry) { in perf_event_enable_on_exec()
4412 * remove-on-exec, and feeds their values back to parent events.
4421 mutex_lock(&ctx->mutex); in perf_event_remove_on_exec()
4423 if (WARN_ON_ONCE(ctx->task != current)) in perf_event_remove_on_exec()
4426 list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { in perf_event_remove_on_exec()
4427 if (!event->attr.remove_on_exec) in perf_event_remove_on_exec()
4438 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_event_remove_on_exec()
4441 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_remove_on_exec()
4444 mutex_unlock(&ctx->mutex); in perf_event_remove_on_exec()
4463 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { in __perf_event_read_cpu()
4482 struct perf_event *sub, *event = data->event; in __perf_event_read()
4483 struct perf_event_context *ctx = event->ctx; in __perf_event_read()
4485 struct pmu *pmu = event->pmu; in __perf_event_read()
4491 * event->count would have been updated to a recent sample in __perf_event_read()
4494 if (ctx->task && cpuctx->task_ctx != ctx) in __perf_event_read()
4497 raw_spin_lock(&ctx->lock); in __perf_event_read()
4498 if (ctx->is_active & EVENT_TIME) { in __perf_event_read()
4504 if (data->group) in __perf_event_read()
4507 if (event->state != PERF_EVENT_STATE_ACTIVE) in __perf_event_read()
4510 if (!data->group) { in __perf_event_read()
4511 pmu->read(event); in __perf_event_read()
4512 data->ret = 0; in __perf_event_read()
4516 pmu->start_txn(pmu, PERF_PMU_TXN_READ); in __perf_event_read()
4518 pmu->read(event); in __perf_event_read()
4521 if (sub->state == PERF_EVENT_STATE_ACTIVE) { in __perf_event_read()
4526 sub->pmu->read(sub); in __perf_event_read()
4530 data->ret = pmu->commit_txn(pmu); in __perf_event_read()
4533 raw_spin_unlock(&ctx->lock); in __perf_event_read()
4538 return local64_read(&event->count) + atomic64_read(&event->child_count); in perf_event_count()
4554 * NMI-safe method to read a local event, that is an event that
4556 * - either for the current task, or for this CPU
4557 * - does not have inherit set, for inherited task events
4559 * - must not have a pmu::count method
4579 if (event->attr.inherit) { in perf_event_read_local()
4580 ret = -EOPNOTSUPP; in perf_event_read_local()
4584 /* If this is a per-task event, it must be for current */ in perf_event_read_local()
4585 if ((event->attach_state & PERF_ATTACH_TASK) && in perf_event_read_local()
4586 event->hw.target != current) { in perf_event_read_local()
4587 ret = -EINVAL; in perf_event_read_local()
4593 * a per-package event that can be read locally in perf_event_read_local()
4595 event_oncpu = __perf_event_read_cpu(event, event->oncpu); in perf_event_read_local()
4596 event_cpu = __perf_event_read_cpu(event, event->cpu); in perf_event_read_local()
4598 /* If this is a per-CPU event, it must be for this CPU */ in perf_event_read_local()
4599 if (!(event->attach_state & PERF_ATTACH_TASK) && in perf_event_read_local()
4601 ret = -EINVAL; in perf_event_read_local()
4606 if (event->attr.pinned && event_oncpu != smp_processor_id()) { in perf_event_read_local()
4607 ret = -EBUSY; in perf_event_read_local()
4612 * If the event is currently on this CPU, its either a per-task event, in perf_event_read_local()
4614 * oncpu == -1). in perf_event_read_local()
4617 event->pmu->read(event); in perf_event_read_local()
4619 *value = local64_read(&event->count); in perf_event_read_local()
4637 enum perf_event_state state = READ_ONCE(event->state); in perf_event_read()
4649 * Orders the ->state and ->oncpu loads such that if we see in perf_event_read()
4650 * ACTIVE we must also see the right ->oncpu. in perf_event_read()
4656 event_cpu = READ_ONCE(event->oncpu); in perf_event_read()
4676 * Therefore, either way, we'll have an up-to-date event count in perf_event_read()
4684 struct perf_event_context *ctx = event->ctx; in perf_event_read()
4687 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_event_read()
4688 state = event->state; in perf_event_read()
4690 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_read()
4698 if (ctx->is_active & EVENT_TIME) { in perf_event_read()
4706 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_read()
4717 raw_spin_lock_init(&ctx->lock); in __perf_event_init_context()
4718 mutex_init(&ctx->mutex); in __perf_event_init_context()
4719 INIT_LIST_HEAD(&ctx->pmu_ctx_list); in __perf_event_init_context()
4720 perf_event_groups_init(&ctx->pinned_groups); in __perf_event_init_context()
4721 perf_event_groups_init(&ctx->flexible_groups); in __perf_event_init_context()
4722 INIT_LIST_HEAD(&ctx->event_list); in __perf_event_init_context()
4723 refcount_set(&ctx->refcount, 1); in __perf_event_init_context()
4729 epc->pmu = pmu; in __perf_init_event_pmu_context()
4730 INIT_LIST_HEAD(&epc->pmu_ctx_entry); in __perf_init_event_pmu_context()
4731 INIT_LIST_HEAD(&epc->pinned_active); in __perf_init_event_pmu_context()
4732 INIT_LIST_HEAD(&epc->flexible_active); in __perf_init_event_pmu_context()
4733 atomic_set(&epc->refcount, 1); in __perf_init_event_pmu_context()
4747 ctx->task = get_task_struct(task); in alloc_perf_context()
4767 return ERR_PTR(-ESRCH); in find_lively_task_by_vpid()
4785 err = perf_allow_cpu(&event->attr); in find_get_context()
4789 cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in find_get_context()
4790 ctx = &cpuctx->ctx; in find_get_context()
4792 raw_spin_lock_irqsave(&ctx->lock, flags); in find_get_context()
4793 ++ctx->pin_count; in find_get_context()
4794 raw_spin_unlock_irqrestore(&ctx->lock, flags); in find_get_context()
4799 err = -EINVAL; in find_get_context()
4804 ++ctx->pin_count; in find_get_context()
4806 raw_spin_unlock_irqrestore(&ctx->lock, flags); in find_get_context()
4812 err = -ENOMEM; in find_get_context()
4817 mutex_lock(&task->perf_event_mutex); in find_get_context()
4822 if (task->flags & PF_EXITING) in find_get_context()
4823 err = -ESRCH; in find_get_context()
4824 else if (task->perf_event_ctxp) in find_get_context()
4825 err = -EAGAIN; in find_get_context()
4828 ++ctx->pin_count; in find_get_context()
4829 rcu_assign_pointer(task->perf_event_ctxp, ctx); in find_get_context()
4831 mutex_unlock(&task->perf_event_mutex); in find_get_context()
4836 if (err == -EAGAIN) in find_get_context()
4855 if (!ctx->task) { in find_get_pmu_context()
4863 cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu); in find_get_pmu_context()
4864 epc = &cpc->epc; in find_get_pmu_context()
4865 raw_spin_lock_irq(&ctx->lock); in find_get_pmu_context()
4866 if (!epc->ctx) { in find_get_pmu_context()
4867 atomic_set(&epc->refcount, 1); in find_get_pmu_context()
4868 epc->embedded = 1; in find_get_pmu_context()
4869 list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list); in find_get_pmu_context()
4870 epc->ctx = ctx; in find_get_pmu_context()
4872 WARN_ON_ONCE(epc->ctx != ctx); in find_get_pmu_context()
4873 atomic_inc(&epc->refcount); in find_get_pmu_context()
4875 raw_spin_unlock_irq(&ctx->lock); in find_get_pmu_context()
4881 return ERR_PTR(-ENOMEM); in find_get_pmu_context()
4883 if (event->attach_state & PERF_ATTACH_TASK_DATA) { in find_get_pmu_context()
4887 return ERR_PTR(-ENOMEM); in find_get_pmu_context()
4896 * lockdep_assert_held(&ctx->mutex); in find_get_pmu_context()
4899 * child_ctx->mutex. in find_get_pmu_context()
4902 raw_spin_lock_irq(&ctx->lock); in find_get_pmu_context()
4903 list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) { in find_get_pmu_context()
4904 if (epc->pmu == pmu) { in find_get_pmu_context()
4905 WARN_ON_ONCE(epc->ctx != ctx); in find_get_pmu_context()
4906 atomic_inc(&epc->refcount); in find_get_pmu_context()
4914 list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list); in find_get_pmu_context()
4915 epc->ctx = ctx; in find_get_pmu_context()
4918 if (task_ctx_data && !epc->task_ctx_data) { in find_get_pmu_context()
4919 epc->task_ctx_data = task_ctx_data; in find_get_pmu_context()
4921 ctx->nr_task_data++; in find_get_pmu_context()
4923 raw_spin_unlock_irq(&ctx->lock); in find_get_pmu_context()
4933 WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount)); in get_pmu_ctx()
4940 kfree(epc->task_ctx_data); in free_epc_rcu()
4946 struct perf_event_context *ctx = epc->ctx; in put_pmu_ctx()
4952 * lockdep_assert_held(&ctx->mutex); in put_pmu_ctx()
4954 * can't because of the call-site in _free_event()/put_event() in put_pmu_ctx()
4955 * which isn't always called under ctx->mutex. in put_pmu_ctx()
4957 if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags)) in put_pmu_ctx()
4960 WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry)); in put_pmu_ctx()
4962 list_del_init(&epc->pmu_ctx_entry); in put_pmu_ctx()
4963 epc->ctx = NULL; in put_pmu_ctx()
4965 WARN_ON_ONCE(!list_empty(&epc->pinned_active)); in put_pmu_ctx()
4966 WARN_ON_ONCE(!list_empty(&epc->flexible_active)); in put_pmu_ctx()
4968 raw_spin_unlock_irqrestore(&ctx->lock, flags); in put_pmu_ctx()
4970 if (epc->embedded) in put_pmu_ctx()
4973 call_rcu(&epc->rcu_head, free_epc_rcu); in put_pmu_ctx()
4982 if (event->ns) in free_event_rcu()
4983 put_pid_ns(event->ns); in free_event_rcu()
4993 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); in detach_sb_event()
4995 raw_spin_lock(&pel->lock); in detach_sb_event()
4996 list_del_rcu(&event->sb_list); in detach_sb_event()
4997 raw_spin_unlock(&pel->lock); in detach_sb_event()
5002 struct perf_event_attr *attr = &event->attr; in is_sb_event()
5004 if (event->parent) in is_sb_event()
5007 if (event->attach_state & PERF_ATTACH_TASK) in is_sb_event()
5010 if (attr->mmap || attr->mmap_data || attr->mmap2 || in is_sb_event()
5011 attr->comm || attr->comm_exec || in is_sb_event()
5012 attr->task || attr->ksymbol || in is_sb_event()
5013 attr->context_switch || attr->text_poke || in is_sb_event()
5014 attr->bpf_event) in is_sb_event()
5051 if (event->parent) in unaccount_event()
5054 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) in unaccount_event()
5056 if (event->attr.mmap || event->attr.mmap_data) in unaccount_event()
5058 if (event->attr.build_id) in unaccount_event()
5060 if (event->attr.comm) in unaccount_event()
5062 if (event->attr.namespaces) in unaccount_event()
5064 if (event->attr.cgroup) in unaccount_event()
5066 if (event->attr.task) in unaccount_event()
5068 if (event->attr.freq) in unaccount_event()
5070 if (event->attr.context_switch) { in unaccount_event()
5078 if (event->attr.ksymbol) in unaccount_event()
5080 if (event->attr.bpf_event) in unaccount_event()
5082 if (event->attr.text_poke) in unaccount_event()
5086 if (!atomic_add_unless(&perf_sched_count, -1, 1)) in unaccount_event()
5106 * 1) cpu-wide events in the presence of per-task events,
5107 * 2) per-task events in the presence of cpu-wide events,
5111 * _free_event()), the latter -- before the first perf_install_in_context().
5115 struct pmu *pmu = event->pmu; in exclusive_event_init()
5121 * Prevent co-existence of per-task and cpu-wide events on the in exclusive_event_init()
5124 * Negative pmu::exclusive_cnt means there are cpu-wide in exclusive_event_init()
5126 * per-task events. in exclusive_event_init()
5130 * to mean "per-task event", because unlike other attach states it in exclusive_event_init()
5133 if (event->attach_state & PERF_ATTACH_TASK) { in exclusive_event_init()
5134 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) in exclusive_event_init()
5135 return -EBUSY; in exclusive_event_init()
5137 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) in exclusive_event_init()
5138 return -EBUSY; in exclusive_event_init()
5146 struct pmu *pmu = event->pmu; in exclusive_event_destroy()
5152 if (event->attach_state & PERF_ATTACH_TASK) in exclusive_event_destroy()
5153 atomic_dec(&pmu->exclusive_cnt); in exclusive_event_destroy()
5155 atomic_inc(&pmu->exclusive_cnt); in exclusive_event_destroy()
5160 if ((e1->pmu == e2->pmu) && in exclusive_event_match()
5161 (e1->cpu == e2->cpu || in exclusive_event_match()
5162 e1->cpu == -1 || in exclusive_event_match()
5163 e2->cpu == -1)) in exclusive_event_match()
5172 struct pmu *pmu = event->pmu; in exclusive_event_installable()
5174 lockdep_assert_held(&ctx->mutex); in exclusive_event_installable()
5179 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { in exclusive_event_installable()
5192 irq_work_sync(&event->pending_irq); in _free_event()
5198 if (event->rb) { in _free_event()
5200 * Can happen when we close an event with re-directed output. in _free_event()
5205 mutex_lock(&event->mmap_mutex); in _free_event()
5207 mutex_unlock(&event->mmap_mutex); in _free_event()
5213 if (!event->parent) { in _free_event()
5214 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) in _free_event()
5220 kfree(event->addr_filter_ranges); in _free_event()
5222 if (event->destroy) in _free_event()
5223 event->destroy(event); in _free_event()
5226 * Must be after ->destroy(), due to uprobe_perf_close() using in _free_event()
5229 if (event->hw.target) in _free_event()
5230 put_task_struct(event->hw.target); in _free_event()
5232 if (event->pmu_ctx) in _free_event()
5233 put_pmu_ctx(event->pmu_ctx); in _free_event()
5239 if (event->ctx) in _free_event()
5240 put_ctx(event->ctx); in _free_event()
5243 module_put(event->pmu->module); in _free_event()
5245 call_rcu(&event->rcu_head, free_event_rcu); in _free_event()
5254 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, in free_event()
5256 atomic_long_read(&event->refcount), event)) { in free_event()
5257 /* leak to avoid use-after-free */ in free_event()
5276 * owner->perf_event_mutex. in perf_remove_from_owner()
5278 owner = READ_ONCE(event->owner); in perf_remove_from_owner()
5292 * holding ctx->mutex which would be an inversion wrt. the in perf_remove_from_owner()
5296 * ctx->mutex. in perf_remove_from_owner()
5298 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); in perf_remove_from_owner()
5301 * We have to re-check the event->owner field, if it is cleared in perf_remove_from_owner()
5306 if (event->owner) { in perf_remove_from_owner()
5307 list_del_init(&event->owner_entry); in perf_remove_from_owner()
5308 smp_store_release(&event->owner, NULL); in perf_remove_from_owner()
5310 mutex_unlock(&owner->perf_event_mutex); in perf_remove_from_owner()
5317 if (!atomic_long_dec_and_test(&event->refcount)) in put_event()
5330 struct perf_event_context *ctx = event->ctx; in perf_event_release_kernel()
5339 WARN_ON_ONCE(event->attach_state & in perf_event_release_kernel()
5348 WARN_ON_ONCE(ctx->parent_ctx); in perf_event_release_kernel()
5354 * Anybody acquiring event->child_mutex after the below loop _must_ in perf_event_release_kernel()
5366 mutex_lock(&event->child_mutex); in perf_event_release_kernel()
5367 list_for_each_entry(child, &event->child_list, child_list) { in perf_event_release_kernel()
5373 ctx = READ_ONCE(child->ctx); in perf_event_release_kernel()
5387 * can re-acquire child_mutex. in perf_event_release_kernel()
5389 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5390 mutex_lock(&ctx->mutex); in perf_event_release_kernel()
5391 mutex_lock(&event->child_mutex); in perf_event_release_kernel()
5398 tmp = list_first_entry_or_null(&event->child_list, in perf_event_release_kernel()
5402 list_move(&child->child_list, &free_list); in perf_event_release_kernel()
5410 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5411 mutex_unlock(&ctx->mutex); in perf_event_release_kernel()
5415 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5418 void *var = &child->ctx->refcount; in perf_event_release_kernel()
5420 list_del(&child->child_list); in perf_event_release_kernel()
5442 perf_event_release_kernel(file->private_data); in perf_release()
5454 mutex_lock(&event->child_mutex); in __perf_event_read_value()
5459 *enabled += event->total_time_enabled + in __perf_event_read_value()
5460 atomic64_read(&event->child_total_time_enabled); in __perf_event_read_value()
5461 *running += event->total_time_running + in __perf_event_read_value()
5462 atomic64_read(&event->child_total_time_running); in __perf_event_read_value()
5464 list_for_each_entry(child, &event->child_list, child_list) { in __perf_event_read_value()
5467 *enabled += child->total_time_enabled; in __perf_event_read_value()
5468 *running += child->total_time_running; in __perf_event_read_value()
5470 mutex_unlock(&event->child_mutex); in __perf_event_read_value()
5491 struct perf_event_context *ctx = leader->ctx; in __perf_read_group_add()
5501 raw_spin_lock_irqsave(&ctx->lock, flags); in __perf_read_group_add()
5507 * - leader->ctx->lock pins leader->sibling_list in __perf_read_group_add()
5508 * - parent->child_mutex pins parent->child_list in __perf_read_group_add()
5509 * - parent->ctx->mutex pins parent->sibling_list in __perf_read_group_add()
5511 * Because parent->ctx != leader->ctx (and child_list nests inside in __perf_read_group_add()
5512 * ctx->mutex), group destruction is not atomic between children, also in __perf_read_group_add()
5522 parent = leader->parent; in __perf_read_group_add()
5524 (parent->group_generation != leader->group_generation || in __perf_read_group_add()
5525 parent->nr_siblings != leader->nr_siblings)) { in __perf_read_group_add()
5526 ret = -ECHILD; in __perf_read_group_add()
5531 * Since we co-schedule groups, {enabled,running} times of siblings in __perf_read_group_add()
5536 values[n++] += leader->total_time_enabled + in __perf_read_group_add()
5537 atomic64_read(&leader->child_total_time_enabled); in __perf_read_group_add()
5541 values[n++] += leader->total_time_running + in __perf_read_group_add()
5542 atomic64_read(&leader->child_total_time_running); in __perf_read_group_add()
5552 values[n++] = atomic64_read(&leader->lost_samples); in __perf_read_group_add()
5559 values[n++] = atomic64_read(&sub->lost_samples); in __perf_read_group_add()
5563 raw_spin_unlock_irqrestore(&ctx->lock, flags); in __perf_read_group_add()
5570 struct perf_event *leader = event->group_leader, *child; in perf_read_group()
5571 struct perf_event_context *ctx = leader->ctx; in perf_read_group()
5575 lockdep_assert_held(&ctx->mutex); in perf_read_group()
5577 values = kzalloc(event->read_size, GFP_KERNEL); in perf_read_group()
5579 return -ENOMEM; in perf_read_group()
5581 values[0] = 1 + leader->nr_siblings; in perf_read_group()
5583 mutex_lock(&leader->child_mutex); in perf_read_group()
5589 list_for_each_entry(child, &leader->child_list, child_list) { in perf_read_group()
5595 mutex_unlock(&leader->child_mutex); in perf_read_group()
5597 ret = event->read_size; in perf_read_group()
5598 if (copy_to_user(buf, values, event->read_size)) in perf_read_group()
5599 ret = -EFAULT; in perf_read_group()
5603 mutex_unlock(&leader->child_mutex); in perf_read_group()
5624 values[n++] = atomic64_read(&event->lost_samples); in perf_read_one()
5627 return -EFAULT; in perf_read_one()
5636 if (event->state > PERF_EVENT_STATE_EXIT) in is_event_hup()
5639 mutex_lock(&event->child_mutex); in is_event_hup()
5640 no_children = list_empty(&event->child_list); in is_event_hup()
5641 mutex_unlock(&event->child_mutex); in is_event_hup()
5646 * Read the performance event - simple non blocking version for now
5651 u64 read_format = event->attr.read_format; in __perf_read()
5655 * Return end-of-file for a read on an event that is in in __perf_read()
5659 if (event->state == PERF_EVENT_STATE_ERROR) in __perf_read()
5662 if (count < event->read_size) in __perf_read()
5663 return -ENOSPC; in __perf_read()
5665 WARN_ON_ONCE(event->ctx->parent_ctx); in __perf_read()
5677 struct perf_event *event = file->private_data; in perf_read()
5694 struct perf_event *event = file->private_data; in perf_poll()
5698 poll_wait(file, &event->waitq, wait); in perf_poll()
5704 * Pin the event->rb by taking event->mmap_mutex; otherwise in perf_poll()
5707 mutex_lock(&event->mmap_mutex); in perf_poll()
5708 rb = event->rb; in perf_poll()
5710 events = atomic_xchg(&rb->poll, 0); in perf_poll()
5711 mutex_unlock(&event->mmap_mutex); in perf_poll()
5718 local64_set(&event->count, 0); in _perf_event_reset()
5729 WARN_ON_ONCE(event->attr.inherit); in perf_event_pause()
5731 count = local64_read(&event->count); in perf_event_pause()
5733 local64_set(&event->count, 0); in perf_event_pause()
5741 * Holding the top-level event's child_mutex means that any
5751 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_event_for_each_child()
5753 mutex_lock(&event->child_mutex); in perf_event_for_each_child()
5755 list_for_each_entry(child, &event->child_list, child_list) in perf_event_for_each_child()
5757 mutex_unlock(&event->child_mutex); in perf_event_for_each_child()
5763 struct perf_event_context *ctx = event->ctx; in perf_event_for_each()
5766 lockdep_assert_held(&ctx->mutex); in perf_event_for_each()
5768 event = event->group_leader; in perf_event_for_each()
5783 if (event->attr.freq) { in __perf_event_period()
5784 event->attr.sample_freq = value; in __perf_event_period()
5786 event->attr.sample_period = value; in __perf_event_period()
5787 event->hw.sample_period = value; in __perf_event_period()
5790 active = (event->state == PERF_EVENT_STATE_ACTIVE); in __perf_event_period()
5792 perf_pmu_disable(event->pmu); in __perf_event_period()
5795 * trying to unthrottle while we already re-started the event. in __perf_event_period()
5797 if (event->hw.interrupts == MAX_INTERRUPTS) { in __perf_event_period()
5798 event->hw.interrupts = 0; in __perf_event_period()
5801 event->pmu->stop(event, PERF_EF_UPDATE); in __perf_event_period()
5804 local64_set(&event->hw.period_left, 0); in __perf_event_period()
5807 event->pmu->start(event, PERF_EF_RELOAD); in __perf_event_period()
5808 perf_pmu_enable(event->pmu); in __perf_event_period()
5814 return event->pmu->check_period(event, value); in perf_event_check_period()
5820 return -EINVAL; in _perf_event_period()
5823 return -EINVAL; in _perf_event_period()
5825 if (event->attr.freq && value > sysctl_perf_event_sample_rate) in _perf_event_period()
5826 return -EINVAL; in _perf_event_period()
5829 return -EINVAL; in _perf_event_period()
5831 if (!event->attr.freq && (value & (1ULL << 63))) in _perf_event_period()
5832 return -EINVAL; in _perf_event_period()
5858 return -EBADF; in perf_fget_light()
5860 if (f.file->f_op != &perf_fops) { in perf_fget_light()
5862 return -EBADF; in perf_fget_light()
5898 return -EFAULT; in _perf_ioctl()
5907 return -EFAULT; in _perf_ioctl()
5914 if (arg != -1) { in _perf_ioctl()
5920 output_event = output.file->private_data; in _perf_ioctl()
5954 rb = rcu_dereference(event->rb); in _perf_ioctl()
5955 if (!rb || !rb->nr_pages) { in _perf_ioctl()
5957 return -EINVAL; in _perf_ioctl()
5978 return -ENOTTY; in _perf_ioctl()
5991 struct perf_event *event = file->private_data; in perf_ioctl()
6016 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ in perf_compat_ioctl()
6034 mutex_lock(¤t->perf_event_mutex); in perf_event_task_enable()
6035 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { in perf_event_task_enable()
6040 mutex_unlock(¤t->perf_event_mutex); in perf_event_task_enable()
6050 mutex_lock(¤t->perf_event_mutex); in perf_event_task_disable()
6051 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { in perf_event_task_disable()
6056 mutex_unlock(¤t->perf_event_mutex); in perf_event_task_disable()
6063 if (event->hw.state & PERF_HES_STOPPED) in perf_event_index()
6066 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_event_index()
6069 return event->pmu->event_idx(event); in perf_event_index()
6078 rb = rcu_dereference(event->rb); in perf_event_init_userpage()
6082 userpg = rb->user_page; in perf_event_init_userpage()
6085 userpg->cap_bit0_is_deprecated = 1; in perf_event_init_userpage()
6086 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); in perf_event_init_userpage()
6087 userpg->data_offset = PAGE_SIZE; in perf_event_init_userpage()
6088 userpg->data_size = perf_data_size(rb); in perf_event_init_userpage()
6111 rb = rcu_dereference(event->rb); in perf_event_update_userpage()
6126 userpg = rb->user_page; in perf_event_update_userpage()
6132 ++userpg->lock; in perf_event_update_userpage()
6134 userpg->index = perf_event_index(event); in perf_event_update_userpage()
6135 userpg->offset = perf_event_count(event); in perf_event_update_userpage()
6136 if (userpg->index) in perf_event_update_userpage()
6137 userpg->offset -= local64_read(&event->hw.prev_count); in perf_event_update_userpage()
6139 userpg->time_enabled = enabled + in perf_event_update_userpage()
6140 atomic64_read(&event->child_total_time_enabled); in perf_event_update_userpage()
6142 userpg->time_running = running + in perf_event_update_userpage()
6143 atomic64_read(&event->child_total_time_running); in perf_event_update_userpage()
6148 ++userpg->lock; in perf_event_update_userpage()
6157 struct perf_event *event = vmf->vma->vm_file->private_data; in perf_mmap_fault()
6161 if (vmf->flags & FAULT_FLAG_MKWRITE) { in perf_mmap_fault()
6162 if (vmf->pgoff == 0) in perf_mmap_fault()
6168 rb = rcu_dereference(event->rb); in perf_mmap_fault()
6172 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) in perf_mmap_fault()
6175 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); in perf_mmap_fault()
6176 if (!vmf->page) in perf_mmap_fault()
6179 get_page(vmf->page); in perf_mmap_fault()
6180 vmf->page->mapping = vmf->vma->vm_file->f_mapping; in perf_mmap_fault()
6181 vmf->page->index = vmf->pgoff; in perf_mmap_fault()
6196 WARN_ON_ONCE(event->parent); in ring_buffer_attach()
6198 if (event->rb) { in ring_buffer_attach()
6201 * event->rb_entry and wait/clear when adding event->rb_entry. in ring_buffer_attach()
6203 WARN_ON_ONCE(event->rcu_pending); in ring_buffer_attach()
6205 old_rb = event->rb; in ring_buffer_attach()
6206 spin_lock_irqsave(&old_rb->event_lock, flags); in ring_buffer_attach()
6207 list_del_rcu(&event->rb_entry); in ring_buffer_attach()
6208 spin_unlock_irqrestore(&old_rb->event_lock, flags); in ring_buffer_attach()
6210 event->rcu_batches = get_state_synchronize_rcu(); in ring_buffer_attach()
6211 event->rcu_pending = 1; in ring_buffer_attach()
6215 if (event->rcu_pending) { in ring_buffer_attach()
6216 cond_synchronize_rcu(event->rcu_batches); in ring_buffer_attach()
6217 event->rcu_pending = 0; in ring_buffer_attach()
6220 spin_lock_irqsave(&rb->event_lock, flags); in ring_buffer_attach()
6221 list_add_rcu(&event->rb_entry, &rb->event_list); in ring_buffer_attach()
6222 spin_unlock_irqrestore(&rb->event_lock, flags); in ring_buffer_attach()
6232 * mid-air, but then again, whoever does it like this is in ring_buffer_attach()
6238 rcu_assign_pointer(event->rb, rb); in ring_buffer_attach()
6247 wake_up_all(&event->waitq); in ring_buffer_attach()
6255 if (event->parent) in ring_buffer_wakeup()
6256 event = event->parent; in ring_buffer_wakeup()
6259 rb = rcu_dereference(event->rb); in ring_buffer_wakeup()
6261 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) in ring_buffer_wakeup()
6262 wake_up_all(&event->waitq); in ring_buffer_wakeup()
6271 if (event->parent) in ring_buffer_get()
6272 event = event->parent; in ring_buffer_get()
6275 rb = rcu_dereference(event->rb); in ring_buffer_get()
6277 if (!refcount_inc_not_zero(&rb->refcount)) in ring_buffer_get()
6287 if (!refcount_dec_and_test(&rb->refcount)) in ring_buffer_put()
6290 WARN_ON_ONCE(!list_empty(&rb->event_list)); in ring_buffer_put()
6292 call_rcu(&rb->rcu_head, rb_free_rcu); in ring_buffer_put()
6297 struct perf_event *event = vma->vm_file->private_data; in perf_mmap_open()
6299 atomic_inc(&event->mmap_count); in perf_mmap_open()
6300 atomic_inc(&event->rb->mmap_count); in perf_mmap_open()
6302 if (vma->vm_pgoff) in perf_mmap_open()
6303 atomic_inc(&event->rb->aux_mmap_count); in perf_mmap_open()
6305 if (event->pmu->event_mapped) in perf_mmap_open()
6306 event->pmu->event_mapped(event, vma->vm_mm); in perf_mmap_open()
6321 struct perf_event *event = vma->vm_file->private_data; in perf_mmap_close()
6323 struct user_struct *mmap_user = rb->mmap_user; in perf_mmap_close()
6324 int mmap_locked = rb->mmap_locked; in perf_mmap_close()
6328 if (event->pmu->event_unmapped) in perf_mmap_close()
6329 event->pmu->event_unmapped(event, vma->vm_mm); in perf_mmap_close()
6332 * rb->aux_mmap_count will always drop before rb->mmap_count and in perf_mmap_close()
6333 * event->mmap_count, so it is ok to use event->mmap_mutex to in perf_mmap_close()
6336 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && in perf_mmap_close()
6337 atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) { in perf_mmap_close()
6347 atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm); in perf_mmap_close()
6348 atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm); in perf_mmap_close()
6352 WARN_ON_ONCE(refcount_read(&rb->aux_refcount)); in perf_mmap_close()
6354 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6357 if (atomic_dec_and_test(&rb->mmap_count)) in perf_mmap_close()
6360 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) in perf_mmap_close()
6364 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6377 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { in perf_mmap_close()
6378 if (!atomic_long_inc_not_zero(&event->refcount)) { in perf_mmap_close()
6380 * This event is en-route to free_event() which will in perf_mmap_close()
6387 mutex_lock(&event->mmap_mutex); in perf_mmap_close()
6398 if (event->rb == rb) in perf_mmap_close()
6401 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6413 * It could be there's still a few 0-ref events on the list; they'll in perf_mmap_close()
6414 * get cleaned up by free_event() -- they'll also still have their in perf_mmap_close()
6421 atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked, in perf_mmap_close()
6422 &mmap_user->locked_vm); in perf_mmap_close()
6423 atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm); in perf_mmap_close()
6439 struct perf_event *event = file->private_data; in perf_mmap()
6450 * Don't allow mmap() of inherited per-task counters. This would in perf_mmap()
6454 if (event->cpu == -1 && event->attr.inherit) in perf_mmap()
6455 return -EINVAL; in perf_mmap()
6457 if (!(vma->vm_flags & VM_SHARED)) in perf_mmap()
6458 return -EINVAL; in perf_mmap()
6464 vma_size = vma->vm_end - vma->vm_start; in perf_mmap()
6466 if (vma->vm_pgoff == 0) { in perf_mmap()
6467 nr_pages = (vma_size / PAGE_SIZE) - 1; in perf_mmap()
6470 * AUX area mapping: if rb->aux_nr_pages != 0, it's already in perf_mmap()
6476 if (!event->rb) in perf_mmap()
6477 return -EINVAL; in perf_mmap()
6481 mutex_lock(&event->mmap_mutex); in perf_mmap()
6482 ret = -EINVAL; in perf_mmap()
6484 rb = event->rb; in perf_mmap()
6488 aux_offset = READ_ONCE(rb->user_page->aux_offset); in perf_mmap()
6489 aux_size = READ_ONCE(rb->user_page->aux_size); in perf_mmap()
6494 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) in perf_mmap()
6498 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) in perf_mmap()
6505 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) in perf_mmap()
6511 if (!atomic_inc_not_zero(&rb->mmap_count)) in perf_mmap()
6515 atomic_inc(&rb->aux_mmap_count); in perf_mmap()
6520 atomic_set(&rb->aux_mmap_count, 1); in perf_mmap()
6527 * If we have rb pages ensure they're a power-of-two number, so we in perf_mmap()
6531 return -EINVAL; in perf_mmap()
6534 return -EINVAL; in perf_mmap()
6536 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_mmap()
6538 mutex_lock(&event->mmap_mutex); in perf_mmap()
6539 if (event->rb) { in perf_mmap()
6540 if (data_page_nr(event->rb) != nr_pages) { in perf_mmap()
6541 ret = -EINVAL; in perf_mmap()
6545 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { in perf_mmap()
6551 mutex_unlock(&event->mmap_mutex); in perf_mmap()
6561 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); in perf_mmap()
6568 user_locked = atomic_long_read(&user->locked_vm); in perf_mmap()
6572 * user->locked_vm > user_lock_limit in perf_mmap()
6583 extra = user_locked - user_lock_limit; in perf_mmap()
6584 user_extra -= extra; in perf_mmap()
6589 locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra; in perf_mmap()
6593 ret = -EPERM; in perf_mmap()
6597 WARN_ON(!rb && event->rb); in perf_mmap()
6599 if (vma->vm_flags & VM_WRITE) in perf_mmap()
6604 event->attr.watermark ? event->attr.wakeup_watermark : 0, in perf_mmap()
6605 event->cpu, flags); in perf_mmap()
6608 ret = -ENOMEM; in perf_mmap()
6612 atomic_set(&rb->mmap_count, 1); in perf_mmap()
6613 rb->mmap_user = get_current_user(); in perf_mmap()
6614 rb->mmap_locked = extra; in perf_mmap()
6622 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, in perf_mmap()
6623 event->attr.aux_watermark, flags); in perf_mmap()
6625 rb->aux_mmap_locked = extra; in perf_mmap()
6630 atomic_long_add(user_extra, &user->locked_vm); in perf_mmap()
6631 atomic64_add(extra, &vma->vm_mm->pinned_vm); in perf_mmap()
6633 atomic_inc(&event->mmap_count); in perf_mmap()
6635 atomic_dec(&rb->mmap_count); in perf_mmap()
6638 mutex_unlock(&event->mmap_mutex); in perf_mmap()
6645 vma->vm_ops = &perf_mmap_vmops; in perf_mmap()
6647 if (event->pmu->event_mapped) in perf_mmap()
6648 event->pmu->event_mapped(event, vma->vm_mm); in perf_mmap()
6656 struct perf_event *event = filp->private_data; in perf_fasync()
6660 retval = fasync_helper(fd, filp, on, &event->fasync); in perf_fasync()
6684 * to user-space before waking everybody up.
6690 if (event->parent) in perf_event_fasync()
6691 event = event->parent; in perf_event_fasync()
6692 return &event->fasync; in perf_event_fasync()
6699 if (event->pending_kill) { in perf_event_wakeup()
6700 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); in perf_event_wakeup()
6701 event->pending_kill = 0; in perf_event_wakeup()
6709 * ctx->task or current has changed in the meantime. This can be the in perf_sigtrap()
6712 if (WARN_ON_ONCE(event->ctx->task != current)) in perf_sigtrap()
6719 if (current->flags & PF_EXITING) in perf_sigtrap()
6722 send_sig_perf((void __user *)event->pending_addr, in perf_sigtrap()
6723 event->orig_type, event->attr.sig_data); in perf_sigtrap()
6727 * Deliver the pending work in-event-context or follow the context.
6731 int cpu = READ_ONCE(event->oncpu); in __perf_pending_irq()
6744 if (event->pending_sigtrap) { in __perf_pending_irq()
6745 event->pending_sigtrap = 0; in __perf_pending_irq()
6747 local_dec(&event->ctx->nr_pending); in __perf_pending_irq()
6749 if (event->pending_disable) { in __perf_pending_irq()
6750 event->pending_disable = 0; in __perf_pending_irq()
6757 * CPU-A CPU-B in __perf_pending_irq()
6760 * @pending_disable = CPU-A; in __perf_pending_irq()
6763 * sched-out in __perf_pending_irq()
6764 * @pending_disable = -1; in __perf_pending_irq()
6766 * sched-in in __perf_pending_irq()
6768 * @pending_disable = CPU-B; in __perf_pending_irq()
6774 * But the event runs on CPU-B and wants disabling there. in __perf_pending_irq()
6776 irq_work_queue_on(&event->pending_irq, cpu); in __perf_pending_irq()
6791 * The wakeup isn't bound to the context of the event -- it can happen in perf_pending_irq()
6794 if (event->pending_wakeup) { in perf_pending_irq()
6795 event->pending_wakeup = 0; in perf_pending_irq()
6817 if (event->pending_work) { in perf_pending_task()
6818 event->pending_work = 0; in perf_pending_task()
6820 local_dec(&event->ctx->nr_pending); in perf_pending_task()
6833 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6834 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6835 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
6843 static_call_update(__perf_guest_state, cbs->state); in perf_register_guest_info_callbacks()
6844 static_call_update(__perf_guest_get_ip, cbs->get_ip); in perf_register_guest_info_callbacks()
6846 /* Implementing ->handle_intel_pt_intr is optional. */ in perf_register_guest_info_callbacks()
6847 if (cbs->handle_intel_pt_intr) in perf_register_guest_info_callbacks()
6849 cbs->handle_intel_pt_intr); in perf_register_guest_info_callbacks()
6888 regs_user->abi = perf_reg_abi(current); in perf_sample_regs_user()
6889 regs_user->regs = regs; in perf_sample_regs_user()
6890 } else if (!(current->flags & PF_KTHREAD)) { in perf_sample_regs_user()
6893 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; in perf_sample_regs_user()
6894 regs_user->regs = NULL; in perf_sample_regs_user()
6901 regs_intr->regs = regs; in perf_sample_regs_intr()
6902 regs_intr->abi = perf_reg_abi(current); in perf_sample_regs_intr()
6920 return TASK_SIZE - addr; in perf_ustack_task_size()
6935 * - TASK_SIZE in perf_sample_ustack_size()
6938 * - remaining sample size in perf_sample_ustack_size()
6955 stack_size = USHRT_MAX - header_size - sizeof(u64); in perf_sample_ustack_size()
6978 * - the size requested by user or the best one we can fit in perf_output_sample_ustack()
6981 * - user stack dump data in perf_output_sample_ustack()
6983 * - the actual dumped size in perf_output_sample_ustack()
6992 dyn_size = dump_size - rem; in perf_output_sample_ustack()
7005 struct perf_event *sampler = event->aux_event; in perf_prepare_sample_aux()
7008 data->aux_size = 0; in perf_prepare_sample_aux()
7013 if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE)) in perf_prepare_sample_aux()
7016 if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id())) in perf_prepare_sample_aux()
7027 if (READ_ONCE(rb->aux_in_sampling)) { in perf_prepare_sample_aux()
7028 data->aux_size = 0; in perf_prepare_sample_aux()
7031 data->aux_size = ALIGN(size, sizeof(u64)); in perf_prepare_sample_aux()
7036 return data->aux_size; in perf_prepare_sample_aux()
7048 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler in perf_pmu_snapshot_aux()
7050 * the IRQ ones, that is, for example, re-starting an event that's just in perf_pmu_snapshot_aux()
7061 WRITE_ONCE(rb->aux_in_sampling, 1); in perf_pmu_snapshot_aux()
7064 ret = event->pmu->snapshot_aux(event, handle, size); in perf_pmu_snapshot_aux()
7067 WRITE_ONCE(rb->aux_in_sampling, 0); in perf_pmu_snapshot_aux()
7077 struct perf_event *sampler = event->aux_event; in perf_aux_sample_output()
7082 if (WARN_ON_ONCE(!sampler || !data->aux_size)) in perf_aux_sample_output()
7089 size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size); in perf_aux_sample_output()
7093 * non-zero surplus that it didn't copy), which in its current in perf_aux_sample_output()
7101 * The pad comes from ALIGN()ing data->aux_size up to u64 in in perf_aux_sample_output()
7104 pad = data->aux_size - size; in perf_aux_sample_output()
7118 * A set of common sample data types saved even for non-sample records
7119 * when event->attr.sample_id_all is set.
7129 data->type = event->attr.sample_type; in __perf_event_header__init_id()
7130 data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL; in __perf_event_header__init_id()
7134 data->tid_entry.pid = perf_event_pid(event, current); in __perf_event_header__init_id()
7135 data->tid_entry.tid = perf_event_tid(event, current); in __perf_event_header__init_id()
7139 data->time = perf_event_clock(event); in __perf_event_header__init_id()
7142 data->id = primary_event_id(event); in __perf_event_header__init_id()
7145 data->stream_id = event->id; in __perf_event_header__init_id()
7148 data->cpu_entry.cpu = raw_smp_processor_id(); in __perf_event_header__init_id()
7149 data->cpu_entry.reserved = 0; in __perf_event_header__init_id()
7157 if (event->attr.sample_id_all) { in perf_event_header__init_id()
7158 header->size += event->id_header_size; in perf_event_header__init_id()
7159 __perf_event_header__init_id(data, event, event->attr.sample_type); in perf_event_header__init_id()
7166 u64 sample_type = data->type; in __perf_event__output_id_sample()
7169 perf_output_put(handle, data->tid_entry); in __perf_event__output_id_sample()
7172 perf_output_put(handle, data->time); in __perf_event__output_id_sample()
7175 perf_output_put(handle, data->id); in __perf_event__output_id_sample()
7178 perf_output_put(handle, data->stream_id); in __perf_event__output_id_sample()
7181 perf_output_put(handle, data->cpu_entry); in __perf_event__output_id_sample()
7184 perf_output_put(handle, data->id); in __perf_event__output_id_sample()
7191 if (event->attr.sample_id_all) in perf_event__output_id_sample()
7199 u64 read_format = event->attr.read_format; in perf_output_read_one()
7206 atomic64_read(&event->child_total_time_enabled); in perf_output_read_one()
7210 atomic64_read(&event->child_total_time_running); in perf_output_read_one()
7215 values[n++] = atomic64_read(&event->lost_samples); in perf_output_read_one()
7224 struct perf_event *leader = event->group_leader, *sub; in perf_output_read_group()
7225 u64 read_format = event->attr.read_format; in perf_output_read_group()
7236 values[n++] = 1 + leader->nr_siblings; in perf_output_read_group()
7245 (leader->state == PERF_EVENT_STATE_ACTIVE)) in perf_output_read_group()
7246 leader->pmu->read(leader); in perf_output_read_group()
7252 values[n++] = atomic64_read(&leader->lost_samples); in perf_output_read_group()
7260 (sub->state == PERF_EVENT_STATE_ACTIVE)) in perf_output_read_group()
7261 sub->pmu->read(sub); in perf_output_read_group()
7267 values[n++] = atomic64_read(&sub->lost_samples); in perf_output_read_group()
7282 * child list, not to mention that its impossible to IPI the children running
7289 u64 read_format = event->attr.read_format; in perf_output_read()
7303 if (event->attr.read_format & PERF_FORMAT_GROUP) in perf_output_read()
7314 u64 sample_type = data->type; in perf_output_sample()
7319 perf_output_put(handle, data->id); in perf_output_sample()
7322 perf_output_put(handle, data->ip); in perf_output_sample()
7325 perf_output_put(handle, data->tid_entry); in perf_output_sample()
7328 perf_output_put(handle, data->time); in perf_output_sample()
7331 perf_output_put(handle, data->addr); in perf_output_sample()
7334 perf_output_put(handle, data->id); in perf_output_sample()
7337 perf_output_put(handle, data->stream_id); in perf_output_sample()
7340 perf_output_put(handle, data->cpu_entry); in perf_output_sample()
7343 perf_output_put(handle, data->period); in perf_output_sample()
7351 size += data->callchain->nr; in perf_output_sample()
7353 __output_copy(handle, data->callchain, size); in perf_output_sample()
7357 struct perf_raw_record *raw = data->raw; in perf_output_sample()
7360 struct perf_raw_frag *frag = &raw->frag; in perf_output_sample()
7362 perf_output_put(handle, raw->size); in perf_output_sample()
7364 if (frag->copy) { in perf_output_sample()
7365 __output_custom(handle, frag->copy, in perf_output_sample()
7366 frag->data, frag->size); in perf_output_sample()
7368 __output_copy(handle, frag->data, in perf_output_sample()
7369 frag->size); in perf_output_sample()
7373 frag = frag->next; in perf_output_sample()
7375 if (frag->pad) in perf_output_sample()
7376 __output_skip(handle, NULL, frag->pad); in perf_output_sample()
7390 if (data->br_stack) { in perf_output_sample()
7393 size = data->br_stack->nr in perf_output_sample()
7396 perf_output_put(handle, data->br_stack->nr); in perf_output_sample()
7398 perf_output_put(handle, data->br_stack->hw_idx); in perf_output_sample()
7399 perf_output_copy(handle, data->br_stack->entries, size); in perf_output_sample()
7404 if (data->br_stack_cntr) { in perf_output_sample()
7405 size = data->br_stack->nr * sizeof(u64); in perf_output_sample()
7406 perf_output_copy(handle, data->br_stack_cntr, size); in perf_output_sample()
7418 u64 abi = data->regs_user.abi; in perf_output_sample()
7427 u64 mask = event->attr.sample_regs_user; in perf_output_sample()
7429 data->regs_user.regs, in perf_output_sample()
7436 data->stack_user_size, in perf_output_sample()
7437 data->regs_user.regs); in perf_output_sample()
7441 perf_output_put(handle, data->weight.full); in perf_output_sample()
7444 perf_output_put(handle, data->data_src.val); in perf_output_sample()
7447 perf_output_put(handle, data->txn); in perf_output_sample()
7450 u64 abi = data->regs_intr.abi; in perf_output_sample()
7458 u64 mask = event->attr.sample_regs_intr; in perf_output_sample()
7461 data->regs_intr.regs, in perf_output_sample()
7467 perf_output_put(handle, data->phys_addr); in perf_output_sample()
7470 perf_output_put(handle, data->cgroup); in perf_output_sample()
7473 perf_output_put(handle, data->data_page_size); in perf_output_sample()
7476 perf_output_put(handle, data->code_page_size); in perf_output_sample()
7479 perf_output_put(handle, data->aux_size); in perf_output_sample()
7481 if (data->aux_size) in perf_output_sample()
7485 if (!event->attr.watermark) { in perf_output_sample()
7486 int wakeup_events = event->attr.wakeup_events; in perf_output_sample()
7489 struct perf_buffer *rb = handle->rb; in perf_output_sample()
7490 int events = local_inc_return(&rb->events); in perf_output_sample()
7493 local_sub(wakeup_events, &rb->events); in perf_output_sample()
7494 local_inc(&rb->wakeup); in perf_output_sample()
7517 * Try IRQ-safe get_user_page_fast_only first. in perf_virt_to_phys()
7520 if (current->mm != NULL) { in perf_virt_to_phys()
7605 * Software page-table walkers must disable IRQs, in perf_get_page_size()
7610 mm = current->mm; in perf_get_page_size()
7631 bool kernel = !event->attr.exclude_callchain_kernel; in perf_callchain()
7632 bool user = !event->attr.exclude_callchain_user; in perf_callchain()
7633 /* Disallow cross-task user callchains. */ in perf_callchain()
7634 bool crosstask = event->ctx->task && event->ctx->task != current; in perf_callchain()
7635 const u32 max_stack = event->attr.sample_max_stack; in perf_callchain()
7655 u64 sample_type = event->attr.sample_type; in perf_prepare_sample()
7669 filtered_sample_type &= ~data->sample_flags; in perf_prepare_sample()
7672 /* Make sure it has the correct data->type for output */ in perf_prepare_sample()
7673 data->type = event->attr.sample_type; in perf_prepare_sample()
7680 data->ip = perf_instruction_pointer(regs); in perf_prepare_sample()
7681 data->sample_flags |= PERF_SAMPLE_IP; in perf_prepare_sample()
7688 data->raw = NULL; in perf_prepare_sample()
7689 data->dyn_size += sizeof(u64); in perf_prepare_sample()
7690 data->sample_flags |= PERF_SAMPLE_RAW; in perf_prepare_sample()
7694 data->br_stack = NULL; in perf_prepare_sample()
7695 data->dyn_size += sizeof(u64); in perf_prepare_sample()
7696 data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; in perf_prepare_sample()
7700 perf_sample_regs_user(&data->regs_user, regs); in perf_prepare_sample()
7707 if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) { in perf_prepare_sample()
7711 if (data->regs_user.regs) { in perf_prepare_sample()
7712 u64 mask = event->attr.sample_regs_user; in perf_prepare_sample()
7716 data->dyn_size += size; in perf_prepare_sample()
7717 data->sample_flags |= PERF_SAMPLE_REGS_USER; in perf_prepare_sample()
7727 u16 stack_size = event->attr.sample_stack_user; in perf_prepare_sample()
7732 data->regs_user.regs); in perf_prepare_sample()
7742 data->stack_user_size = stack_size; in perf_prepare_sample()
7743 data->dyn_size += size; in perf_prepare_sample()
7744 data->sample_flags |= PERF_SAMPLE_STACK_USER; in perf_prepare_sample()
7748 data->weight.full = 0; in perf_prepare_sample()
7749 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; in perf_prepare_sample()
7753 data->data_src.val = PERF_MEM_NA; in perf_prepare_sample()
7754 data->sample_flags |= PERF_SAMPLE_DATA_SRC; in perf_prepare_sample()
7758 data->txn = 0; in perf_prepare_sample()
7759 data->sample_flags |= PERF_SAMPLE_TRANSACTION; in perf_prepare_sample()
7763 data->addr = 0; in perf_prepare_sample()
7764 data->sample_flags |= PERF_SAMPLE_ADDR; in perf_prepare_sample()
7771 perf_sample_regs_intr(&data->regs_intr, regs); in perf_prepare_sample()
7773 if (data->regs_intr.regs) { in perf_prepare_sample()
7774 u64 mask = event->attr.sample_regs_intr; in perf_prepare_sample()
7779 data->dyn_size += size; in perf_prepare_sample()
7780 data->sample_flags |= PERF_SAMPLE_REGS_INTR; in perf_prepare_sample()
7784 data->phys_addr = perf_virt_to_phys(data->addr); in perf_prepare_sample()
7785 data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; in perf_prepare_sample()
7793 cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup; in perf_prepare_sample()
7794 data->cgroup = cgroup_id(cgrp); in perf_prepare_sample()
7795 data->sample_flags |= PERF_SAMPLE_CGROUP; in perf_prepare_sample()
7801 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, in perf_prepare_sample()
7805 data->data_page_size = perf_get_page_size(data->addr); in perf_prepare_sample()
7806 data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE; in perf_prepare_sample()
7810 data->code_page_size = perf_get_page_size(data->ip); in perf_prepare_sample()
7811 data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE; in perf_prepare_sample()
7826 size = min_t(size_t, U16_MAX - header_size, in perf_prepare_sample()
7827 event->attr.aux_sample_size); in perf_prepare_sample()
7832 data->dyn_size += size + sizeof(u64); /* size above */ in perf_prepare_sample()
7833 data->sample_flags |= PERF_SAMPLE_AUX; in perf_prepare_sample()
7842 header->type = PERF_RECORD_SAMPLE; in perf_prepare_header()
7843 header->size = perf_sample_data_size(data, event); in perf_prepare_header()
7844 header->misc = perf_misc_flags(regs); in perf_prepare_header()
7854 WARN_ON_ONCE(header->size & 7); in perf_prepare_header()
7934 .size = sizeof(read_event) + event->read_size, in perf_event_read_event()
7962 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { in perf_iterate_ctx()
7964 if (event->state < PERF_EVENT_STATE_INACTIVE) in perf_iterate_ctx()
7979 list_for_each_entry_rcu(event, &pel->list, sb_list) { in perf_iterate_sb_cpu()
7982 * if we observe event->ctx, both event and ctx will be in perf_iterate_sb_cpu()
7985 if (!smp_load_acquire(&event->ctx)) in perf_iterate_sb_cpu()
7988 if (event->state < PERF_EVENT_STATE_INACTIVE) in perf_iterate_sb_cpu()
7997 * Iterate all events that need to receive side-band events.
8023 ctx = rcu_dereference(current->perf_event_ctxp); in perf_iterate_sb()
8032 * Clear all file-based filters at exec, they'll have to be
8033 * re-instated when/if these objects are mmapped again.
8045 raw_spin_lock_irqsave(&ifh->lock, flags); in perf_event_addr_filters_exec()
8046 list_for_each_entry(filter, &ifh->list, entry) { in perf_event_addr_filters_exec()
8047 if (filter->path.dentry) { in perf_event_addr_filters_exec()
8048 event->addr_filter_ranges[count].start = 0; in perf_event_addr_filters_exec()
8049 event->addr_filter_ranges[count].size = 0; in perf_event_addr_filters_exec()
8057 event->addr_filters_gen++; in perf_event_addr_filters_exec()
8058 raw_spin_unlock_irqrestore(&ifh->lock, flags); in perf_event_addr_filters_exec()
8087 struct perf_event *parent = event->parent; in __perf_event_output_stop()
8089 struct perf_buffer *rb = ro->rb; in __perf_event_output_stop()
8102 * ring-buffer, but it will be the child that's actually using it. in __perf_event_output_stop()
8107 * So ring_buffer_attach() has to stop an aux event before re-assigning in __perf_event_output_stop()
8110 if (rcu_dereference(parent->rb) == rb) in __perf_event_output_stop()
8111 ro->err = __perf_event_stop(&sd); in __perf_event_output_stop()
8119 .rb = event->rb, in __perf_pmu_output_stop()
8123 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); in __perf_pmu_output_stop()
8124 if (cpuctx->task_ctx) in __perf_pmu_output_stop()
8125 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, in __perf_pmu_output_stop()
8139 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { in perf_pmu_output_stop()
8141 * For per-CPU events, we need to make sure that neither they in perf_pmu_output_stop()
8142 * nor their children are running; for cpu==-1 events it's in perf_pmu_output_stop()
8146 cpu = iter->cpu; in perf_pmu_output_stop()
8147 if (cpu == -1) in perf_pmu_output_stop()
8148 cpu = READ_ONCE(iter->oncpu); in perf_pmu_output_stop()
8150 if (cpu == -1) in perf_pmu_output_stop()
8154 if (err == -EAGAIN) { in perf_pmu_output_stop()
8163 * task tracking -- fork/exit
8185 return event->attr.comm || event->attr.mmap || in perf_event_task_match()
8186 event->attr.mmap2 || event->attr.mmap_data || in perf_event_task_match()
8187 event->attr.task; in perf_event_task_match()
8196 struct task_struct *task = task_event->task; in perf_event_task_output()
8197 int ret, size = task_event->event_id.header.size; in perf_event_task_output()
8202 perf_event_header__init_id(&task_event->event_id.header, &sample, event); in perf_event_task_output()
8205 task_event->event_id.header.size); in perf_event_task_output()
8209 task_event->event_id.pid = perf_event_pid(event, task); in perf_event_task_output()
8210 task_event->event_id.tid = perf_event_tid(event, task); in perf_event_task_output()
8212 if (task_event->event_id.header.type == PERF_RECORD_EXIT) { in perf_event_task_output()
8213 task_event->event_id.ppid = perf_event_pid(event, in perf_event_task_output()
8214 task->real_parent); in perf_event_task_output()
8215 task_event->event_id.ptid = perf_event_pid(event, in perf_event_task_output()
8216 task->real_parent); in perf_event_task_output()
8218 task_event->event_id.ppid = perf_event_pid(event, current); in perf_event_task_output()
8219 task_event->event_id.ptid = perf_event_tid(event, current); in perf_event_task_output()
8222 task_event->event_id.time = perf_event_clock(event); in perf_event_task_output()
8224 perf_output_put(&handle, task_event->event_id); in perf_event_task_output()
8230 task_event->event_id.header.size = size; in perf_event_task_output()
8291 return event->attr.comm; in perf_event_comm_match()
8300 int size = comm_event->event_id.header.size; in perf_event_comm_output()
8306 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); in perf_event_comm_output()
8308 comm_event->event_id.header.size); in perf_event_comm_output()
8313 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); in perf_event_comm_output()
8314 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); in perf_event_comm_output()
8316 perf_output_put(&handle, comm_event->event_id); in perf_event_comm_output()
8317 __output_copy(&handle, comm_event->comm, in perf_event_comm_output()
8318 comm_event->comm_size); in perf_event_comm_output()
8324 comm_event->event_id.header.size = size; in perf_event_comm_output()
8333 strscpy(comm, comm_event->task->comm, sizeof(comm)); in perf_event_comm_event()
8336 comm_event->comm = comm; in perf_event_comm_event()
8337 comm_event->comm_size = size; in perf_event_comm_event()
8339 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; in perf_event_comm_event()
8390 return event->attr.namespaces; in perf_event_namespaces_match()
8399 u16 header_size = namespaces_event->event_id.header.size; in perf_event_namespaces_output()
8405 perf_event_header__init_id(&namespaces_event->event_id.header, in perf_event_namespaces_output()
8408 namespaces_event->event_id.header.size); in perf_event_namespaces_output()
8412 namespaces_event->event_id.pid = perf_event_pid(event, in perf_event_namespaces_output()
8413 namespaces_event->task); in perf_event_namespaces_output()
8414 namespaces_event->event_id.tid = perf_event_tid(event, in perf_event_namespaces_output()
8415 namespaces_event->task); in perf_event_namespaces_output()
8417 perf_output_put(&handle, namespaces_event->event_id); in perf_event_namespaces_output()
8423 namespaces_event->event_id.header.size = header_size; in perf_event_namespaces_output()
8436 ns_inode = ns_path.dentry->d_inode; in perf_fill_ns_link_info()
8437 ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev); in perf_fill_ns_link_info()
8438 ns_link_info->ino = ns_inode->i_ino; in perf_fill_ns_link_info()
8518 return event->attr.cgroup; in perf_event_cgroup_match()
8526 u16 header_size = cgroup_event->event_id.header.size; in perf_event_cgroup_output()
8532 perf_event_header__init_id(&cgroup_event->event_id.header, in perf_event_cgroup_output()
8535 cgroup_event->event_id.header.size); in perf_event_cgroup_output()
8539 perf_output_put(&handle, cgroup_event->event_id); in perf_event_cgroup_output()
8540 __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); in perf_event_cgroup_output()
8546 cgroup_event->event_id.header.size = header_size; in perf_event_cgroup_output()
8575 cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64)); in perf_event_cgroup()
8631 struct vm_area_struct *vma = mmap_event->vma; in perf_event_mmap_match()
8632 int executable = vma->vm_flags & VM_EXEC; in perf_event_mmap_match()
8634 return (!executable && event->attr.mmap_data) || in perf_event_mmap_match()
8635 (executable && (event->attr.mmap || event->attr.mmap2)); in perf_event_mmap_match()
8644 int size = mmap_event->event_id.header.size; in perf_event_mmap_output()
8645 u32 type = mmap_event->event_id.header.type; in perf_event_mmap_output()
8652 if (event->attr.mmap2) { in perf_event_mmap_output()
8653 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; in perf_event_mmap_output()
8654 mmap_event->event_id.header.size += sizeof(mmap_event->maj); in perf_event_mmap_output()
8655 mmap_event->event_id.header.size += sizeof(mmap_event->min); in perf_event_mmap_output()
8656 mmap_event->event_id.header.size += sizeof(mmap_event->ino); in perf_event_mmap_output()
8657 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); in perf_event_mmap_output()
8658 mmap_event->event_id.header.size += sizeof(mmap_event->prot); in perf_event_mmap_output()
8659 mmap_event->event_id.header.size += sizeof(mmap_event->flags); in perf_event_mmap_output()
8662 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); in perf_event_mmap_output()
8664 mmap_event->event_id.header.size); in perf_event_mmap_output()
8668 mmap_event->event_id.pid = perf_event_pid(event, current); in perf_event_mmap_output()
8669 mmap_event->event_id.tid = perf_event_tid(event, current); in perf_event_mmap_output()
8671 use_build_id = event->attr.build_id && mmap_event->build_id_size; in perf_event_mmap_output()
8673 if (event->attr.mmap2 && use_build_id) in perf_event_mmap_output()
8674 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; in perf_event_mmap_output()
8676 perf_output_put(&handle, mmap_event->event_id); in perf_event_mmap_output()
8678 if (event->attr.mmap2) { in perf_event_mmap_output()
8680 u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 }; in perf_event_mmap_output()
8683 __output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX); in perf_event_mmap_output()
8685 perf_output_put(&handle, mmap_event->maj); in perf_event_mmap_output()
8686 perf_output_put(&handle, mmap_event->min); in perf_event_mmap_output()
8687 perf_output_put(&handle, mmap_event->ino); in perf_event_mmap_output()
8688 perf_output_put(&handle, mmap_event->ino_generation); in perf_event_mmap_output()
8690 perf_output_put(&handle, mmap_event->prot); in perf_event_mmap_output()
8691 perf_output_put(&handle, mmap_event->flags); in perf_event_mmap_output()
8694 __output_copy(&handle, mmap_event->file_name, in perf_event_mmap_output()
8695 mmap_event->file_size); in perf_event_mmap_output()
8701 mmap_event->event_id.header.size = size; in perf_event_mmap_output()
8702 mmap_event->event_id.header.type = type; in perf_event_mmap_output()
8707 struct vm_area_struct *vma = mmap_event->vma; in perf_event_mmap_event()
8708 struct file *file = vma->vm_file; in perf_event_mmap_event()
8717 if (vma->vm_flags & VM_READ) in perf_event_mmap_event()
8719 if (vma->vm_flags & VM_WRITE) in perf_event_mmap_event()
8721 if (vma->vm_flags & VM_EXEC) in perf_event_mmap_event()
8724 if (vma->vm_flags & VM_MAYSHARE) in perf_event_mmap_event()
8729 if (vma->vm_flags & VM_LOCKED) in perf_event_mmap_event()
8748 name = file_path(file, buf, PATH_MAX - sizeof(u64)); in perf_event_mmap_event()
8753 inode = file_inode(vma->vm_file); in perf_event_mmap_event()
8754 dev = inode->i_sb->s_dev; in perf_event_mmap_event()
8755 ino = inode->i_ino; in perf_event_mmap_event()
8756 gen = inode->i_generation; in perf_event_mmap_event()
8762 if (vma->vm_ops && vma->vm_ops->name) in perf_event_mmap_event()
8763 name = (char *) vma->vm_ops->name(vma); in perf_event_mmap_event()
8789 mmap_event->file_name = name; in perf_event_mmap_event()
8790 mmap_event->file_size = size; in perf_event_mmap_event()
8791 mmap_event->maj = maj; in perf_event_mmap_event()
8792 mmap_event->min = min; in perf_event_mmap_event()
8793 mmap_event->ino = ino; in perf_event_mmap_event()
8794 mmap_event->ino_generation = gen; in perf_event_mmap_event()
8795 mmap_event->prot = prot; in perf_event_mmap_event()
8796 mmap_event->flags = flags; in perf_event_mmap_event()
8798 if (!(vma->vm_flags & VM_EXEC)) in perf_event_mmap_event()
8799 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; in perf_event_mmap_event()
8801 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; in perf_event_mmap_event()
8804 build_id_parse(vma, mmap_event->build_id, &mmap_event->build_id_size); in perf_event_mmap_event()
8820 /* d_inode(NULL) won't be equal to any mapped user-space file */ in perf_addr_filter_match()
8821 if (!filter->path.dentry) in perf_addr_filter_match()
8824 if (d_inode(filter->path.dentry) != file_inode(file)) in perf_addr_filter_match()
8827 if (filter->offset > offset + size) in perf_addr_filter_match()
8830 if (filter->offset + filter->size < offset) in perf_addr_filter_match()
8840 unsigned long vma_size = vma->vm_end - vma->vm_start; in perf_addr_filter_vma_adjust()
8841 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; in perf_addr_filter_vma_adjust()
8842 struct file *file = vma->vm_file; in perf_addr_filter_vma_adjust()
8847 if (filter->offset < off) { in perf_addr_filter_vma_adjust()
8848 fr->start = vma->vm_start; in perf_addr_filter_vma_adjust()
8849 fr->size = min(vma_size, filter->size - (off - filter->offset)); in perf_addr_filter_vma_adjust()
8851 fr->start = vma->vm_start + filter->offset - off; in perf_addr_filter_vma_adjust()
8852 fr->size = min(vma->vm_end - fr->start, filter->size); in perf_addr_filter_vma_adjust()
8869 if (!vma->vm_file) in __perf_addr_filters_adjust()
8872 raw_spin_lock_irqsave(&ifh->lock, flags); in __perf_addr_filters_adjust()
8873 list_for_each_entry(filter, &ifh->list, entry) { in __perf_addr_filters_adjust()
8875 &event->addr_filter_ranges[count])) in __perf_addr_filters_adjust()
8882 event->addr_filters_gen++; in __perf_addr_filters_adjust()
8883 raw_spin_unlock_irqrestore(&ifh->lock, flags); in __perf_addr_filters_adjust()
8900 if (!(vma->vm_flags & VM_EXEC)) in perf_addr_filters_adjust()
8904 ctx = rcu_dereference(current->perf_event_ctxp); in perf_addr_filters_adjust()
8929 .start = vma->vm_start, in perf_event_mmap()
8930 .len = vma->vm_end - vma->vm_start, in perf_event_mmap()
8931 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, in perf_event_mmap()
9029 return event->attr.context_switch; in perf_event_switch_match()
9042 /* Only CPU-wide events are allowed to see next/prev pid/tid */ in perf_event_switch_output()
9043 if (event->ctx->task) { in perf_event_switch_output()
9044 se->event_id.header.type = PERF_RECORD_SWITCH; in perf_event_switch_output()
9045 se->event_id.header.size = sizeof(se->event_id.header); in perf_event_switch_output()
9047 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; in perf_event_switch_output()
9048 se->event_id.header.size = sizeof(se->event_id); in perf_event_switch_output()
9049 se->event_id.next_prev_pid = in perf_event_switch_output()
9050 perf_event_pid(event, se->next_prev); in perf_event_switch_output()
9051 se->event_id.next_prev_tid = in perf_event_switch_output()
9052 perf_event_tid(event, se->next_prev); in perf_event_switch_output()
9055 perf_event_header__init_id(&se->event_id.header, &sample, event); in perf_event_switch_output()
9057 ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); in perf_event_switch_output()
9061 if (event->ctx->task) in perf_event_switch_output()
9062 perf_output_put(&handle, se->event_id.header); in perf_event_switch_output()
9064 perf_output_put(&handle, se->event_id); in perf_event_switch_output()
9092 if (!sched_in && task->on_rq) { in perf_event_switch()
9123 .stream_id = event->id, in perf_log_throttle()
9159 return event->attr.ksymbol; in perf_event_ksymbol_match()
9172 perf_event_header__init_id(&ksymbol_event->event_id.header, in perf_event_ksymbol_output()
9175 ksymbol_event->event_id.header.size); in perf_event_ksymbol_output()
9179 perf_output_put(&handle, ksymbol_event->event_id); in perf_event_ksymbol_output()
9180 __output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len); in perf_event_ksymbol_output()
9249 return event->attr.bpf_event; in perf_event_bpf_match()
9262 perf_event_header__init_id(&bpf_event->event_id.header, in perf_event_bpf_output()
9265 bpf_event->event_id.header.size); in perf_event_bpf_output()
9269 perf_output_put(&handle, bpf_event->event_id); in perf_event_bpf_output()
9281 if (prog->aux->func_cnt == 0) { in perf_event_bpf_emit_ksymbols()
9283 (u64)(unsigned long)prog->bpf_func, in perf_event_bpf_emit_ksymbols()
9284 prog->jited_len, unregister, in perf_event_bpf_emit_ksymbols()
9285 prog->aux->ksym.name); in perf_event_bpf_emit_ksymbols()
9287 for (i = 0; i < prog->aux->func_cnt; i++) { in perf_event_bpf_emit_ksymbols()
9288 struct bpf_prog *subprog = prog->aux->func[i]; in perf_event_bpf_emit_ksymbols()
9292 (u64)(unsigned long)subprog->bpf_func, in perf_event_bpf_emit_ksymbols()
9293 subprog->jited_len, unregister, in perf_event_bpf_emit_ksymbols()
9294 subprog->aux->ksym.name); in perf_event_bpf_emit_ksymbols()
9331 .id = prog->aux->id, in perf_event_bpf_event()
9337 memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE); in perf_event_bpf_event()
9357 return event->attr.text_poke; in perf_event_text_poke_match()
9371 perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); in perf_event_text_poke_output()
9374 text_poke_event->event_id.header.size); in perf_event_text_poke_output()
9378 perf_output_put(&handle, text_poke_event->event_id); in perf_event_text_poke_output()
9379 perf_output_put(&handle, text_poke_event->old_len); in perf_event_text_poke_output()
9380 perf_output_put(&handle, text_poke_event->new_len); in perf_event_text_poke_output()
9382 __output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len); in perf_event_text_poke_output()
9383 __output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len); in perf_event_text_poke_output()
9385 if (text_poke_event->pad) in perf_event_text_poke_output()
9386 __output_copy(&handle, &padding, text_poke_event->pad); in perf_event_text_poke_output()
9404 pad = ALIGN(tot, sizeof(u64)) - tot; in perf_event_text_poke()
9427 event->attach_state |= PERF_ATTACH_ITRACE; in perf_event_itrace_started()
9441 if (event->parent) in perf_log_itrace_start()
9442 event = event->parent; in perf_log_itrace_start()
9444 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || in perf_log_itrace_start()
9445 event->attach_state & PERF_ATTACH_ITRACE) in perf_log_itrace_start()
9476 if (event->parent) in perf_report_aux_output_id()
9477 event = event->parent; in perf_report_aux_output_id()
9500 struct hw_perf_event *hwc = &event->hw; in __perf_event_account_interrupt()
9505 if (seq != hwc->interrupts_seq) { in __perf_event_account_interrupt()
9506 hwc->interrupts_seq = seq; in __perf_event_account_interrupt()
9507 hwc->interrupts = 1; in __perf_event_account_interrupt()
9509 hwc->interrupts++; in __perf_event_account_interrupt()
9511 hwc->interrupts > max_samples_per_tick)) { in __perf_event_account_interrupt()
9514 hwc->interrupts = MAX_INTERRUPTS; in __perf_event_account_interrupt()
9520 if (event->attr.freq) { in __perf_event_account_interrupt()
9522 s64 delta = now - hwc->freq_time_stamp; in __perf_event_account_interrupt()
9524 hwc->freq_time_stamp = now; in __perf_event_account_interrupt()
9527 perf_adjust_period(event, delta, hwc->last_period, true); in __perf_event_account_interrupt()
9545 if (event->attr.exclude_kernel && !user_mode(regs)) in sample_is_allowed()
9559 int events = atomic_read(&event->event_limit); in __perf_event_overflow()
9563 * Non-sampling counters might still use the PMI to fold short in __perf_event_overflow()
9576 event->pending_kill = POLL_IN; in __perf_event_overflow()
9577 if (events && atomic_dec_and_test(&event->event_limit)) { in __perf_event_overflow()
9579 event->pending_kill = POLL_HUP; in __perf_event_overflow()
9583 if (event->attr.sigtrap) { in __perf_event_overflow()
9595 if (!event->pending_sigtrap) { in __perf_event_overflow()
9596 event->pending_sigtrap = pending_id; in __perf_event_overflow()
9597 local_inc(&event->ctx->nr_pending); in __perf_event_overflow()
9598 } else if (event->attr.exclude_kernel && valid_sample) { in __perf_event_overflow()
9606 * 2. Events that can overflow again before the IRQ- in __perf_event_overflow()
9609 * check 32-bit hash of the current IP. in __perf_event_overflow()
9611 WARN_ON_ONCE(event->pending_sigtrap != pending_id); in __perf_event_overflow()
9614 event->pending_addr = 0; in __perf_event_overflow()
9615 if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR)) in __perf_event_overflow()
9616 event->pending_addr = data->addr; in __perf_event_overflow()
9617 irq_work_queue(&event->pending_irq); in __perf_event_overflow()
9620 READ_ONCE(event->overflow_handler)(event, data, regs); in __perf_event_overflow()
9622 if (*perf_event_fasync(event) && event->pending_kill) { in __perf_event_overflow()
9623 event->pending_wakeup = 1; in __perf_event_overflow()
9624 irq_work_queue(&event->pending_irq); in __perf_event_overflow()
9653 * We directly increment event->count and keep a second value in
9654 * event->hw.period_left to count intervals. This period event
9655 * is kept in the range [-sample_period, 0] so that we can use the
9661 struct hw_perf_event *hwc = &event->hw; in perf_swevent_set_period()
9662 u64 period = hwc->last_period; in perf_swevent_set_period()
9666 hwc->last_period = hwc->sample_period; in perf_swevent_set_period()
9668 old = local64_read(&hwc->period_left); in perf_swevent_set_period()
9676 val -= offset; in perf_swevent_set_period()
9677 } while (!local64_try_cmpxchg(&hwc->period_left, &old, val)); in perf_swevent_set_period()
9686 struct hw_perf_event *hwc = &event->hw; in perf_swevent_overflow()
9692 if (hwc->interrupts == MAX_INTERRUPTS) in perf_swevent_overflow()
9695 for (; overflow; overflow--) { in perf_swevent_overflow()
9700 * hwc->interrupts == MAX_INTERRUPTS. in perf_swevent_overflow()
9712 struct hw_perf_event *hwc = &event->hw; in perf_swevent_event()
9714 local64_add(nr, &event->count); in perf_swevent_event()
9722 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { in perf_swevent_event()
9723 data->period = nr; in perf_swevent_event()
9726 data->period = event->hw.last_period; in perf_swevent_event()
9728 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) in perf_swevent_event()
9731 if (local64_add_negative(nr, &hwc->period_left)) in perf_swevent_event()
9740 if (event->hw.state & PERF_HES_STOPPED) in perf_exclude_event()
9744 if (event->attr.exclude_user && user_mode(regs)) in perf_exclude_event()
9747 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_exclude_event()
9760 if (event->attr.type != type) in perf_swevent_match()
9763 if (event->attr.config != event_id) in perf_swevent_match()
9784 return &hlist->heads[hash]; in __find_swevent_head()
9793 hlist = rcu_dereference(swhash->swevent_hlist); in find_swevent_head_rcu()
9805 u32 event_id = event->attr.config; in find_swevent_head()
9806 u64 type = event->attr.type; in find_swevent_head()
9813 hlist = rcu_dereference_protected(swhash->swevent_hlist, in find_swevent_head()
9814 lockdep_is_held(&event->ctx->lock)); in find_swevent_head()
9849 return get_recursion_context(swhash->recursion); in perf_swevent_get_recursion_context()
9857 put_recursion_context(swhash->recursion, rctx); in perf_swevent_put_recursion_context()
9894 struct hw_perf_event *hwc = &event->hw; in perf_swevent_add()
9898 hwc->last_period = hwc->sample_period; in perf_swevent_add()
9902 hwc->state = !(flags & PERF_EF_START); in perf_swevent_add()
9906 return -EINVAL; in perf_swevent_add()
9908 hlist_add_head_rcu(&event->hlist_entry, head); in perf_swevent_add()
9916 hlist_del_rcu(&event->hlist_entry); in perf_swevent_del()
9921 event->hw.state = 0; in perf_swevent_start()
9926 event->hw.state = PERF_HES_STOPPED; in perf_swevent_stop()
9933 return rcu_dereference_protected(swhash->swevent_hlist, in swevent_hlist_deref()
9934 lockdep_is_held(&swhash->hlist_mutex)); in swevent_hlist_deref()
9944 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); in swevent_hlist_release()
9952 mutex_lock(&swhash->hlist_mutex); in swevent_hlist_put_cpu()
9954 if (!--swhash->hlist_refcount) in swevent_hlist_put_cpu()
9957 mutex_unlock(&swhash->hlist_mutex); in swevent_hlist_put_cpu()
9973 mutex_lock(&swhash->hlist_mutex); in swevent_hlist_get_cpu()
9980 err = -ENOMEM; in swevent_hlist_get_cpu()
9983 rcu_assign_pointer(swhash->swevent_hlist, hlist); in swevent_hlist_get_cpu()
9985 swhash->hlist_refcount++; in swevent_hlist_get_cpu()
9987 mutex_unlock(&swhash->hlist_mutex); in swevent_hlist_get_cpu()
10020 u64 event_id = event->attr.config; in sw_perf_event_destroy()
10022 WARN_ON(event->parent); in sw_perf_event_destroy()
10033 u64 event_id = event->attr.config; in perf_swevent_init()
10035 if (event->attr.type != PERF_TYPE_SOFTWARE) in perf_swevent_init()
10036 return -ENOENT; in perf_swevent_init()
10042 return -EOPNOTSUPP; in perf_swevent_init()
10046 event->attr.type = perf_cpu_clock.type; in perf_swevent_init()
10047 return -ENOENT; in perf_swevent_init()
10049 event->attr.type = perf_task_clock.type; in perf_swevent_init()
10050 return -ENOENT; in perf_swevent_init()
10057 return -ENOENT; in perf_swevent_init()
10059 if (!event->parent) { in perf_swevent_init()
10067 event->destroy = sw_perf_event_destroy; in perf_swevent_init()
10097 if (event->attr.type != PERF_TYPE_TRACEPOINT) in perf_tp_event_init()
10098 return -ENOENT; in perf_tp_event_init()
10104 return -EOPNOTSUPP; in perf_tp_event_init()
10110 event->destroy = tp_perf_event_destroy; in perf_tp_event_init()
10129 void *record = data->raw->frag.data; in perf_tp_filter_match()
10132 if (event->parent) in perf_tp_filter_match()
10133 event = event->parent; in perf_tp_filter_match()
10135 if (likely(!event->filter) || filter_match_preds(event->filter, record)) in perf_tp_filter_match()
10144 if (event->hw.state & PERF_HES_STOPPED) in perf_tp_event_match()
10147 * If exclude_kernel, only trace user-space tracepoints (uprobes) in perf_tp_event_match()
10149 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_tp_event_match()
10170 perf_tp_event(call->event.type, count, raw_data, size, regs, head, in perf_trace_run_bpf_submit()
10182 if (event->attr.config != entry->type) in __perf_tp_event_target_task()
10185 if (event->attr.sigtrap) in __perf_tp_event_target_task()
10200 perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) { in perf_tp_event_target_task()
10206 perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) { in perf_tp_event_target_task()
10237 * Here use the same on-stack perf_sample_data, in perf_tp_event()
10238 * some members in data are event-specific and in perf_tp_event()
10239 * need to be re-computed for different sweveents. in perf_tp_event()
10240 * Re-initialize data->sample_flags safely to avoid in perf_tp_event()
10242 * because data->sample_flags is set. in perf_tp_event()
10257 ctx = rcu_dereference(task->perf_event_ctxp); in perf_tp_event()
10261 raw_spin_lock(&ctx->lock); in perf_tp_event()
10263 raw_spin_unlock(&ctx->lock); in perf_tp_event()
10290 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10329 if (event->attr.type != perf_kprobe.type) in perf_kprobe_event_init()
10330 return -ENOENT; in perf_kprobe_event_init()
10333 return -EACCES; in perf_kprobe_event_init()
10339 return -EOPNOTSUPP; in perf_kprobe_event_init()
10341 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; in perf_kprobe_event_init()
10346 event->destroy = perf_kprobe_destroy; in perf_kprobe_event_init()
10353 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10389 if (event->attr.type != perf_uprobe.type) in perf_uprobe_event_init()
10390 return -ENOENT; in perf_uprobe_event_init()
10393 return -EACCES; in perf_uprobe_event_init()
10399 return -EOPNOTSUPP; in perf_uprobe_event_init()
10401 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; in perf_uprobe_event_init()
10402 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT; in perf_uprobe_event_init()
10407 event->destroy = perf_uprobe_destroy; in perf_uprobe_event_init()
10417 perf_pmu_register(&perf_kprobe, "kprobe", -1); in perf_tp_register()
10420 perf_pmu_register(&perf_uprobe, "uprobe", -1); in perf_tp_register()
10445 prog = READ_ONCE(event->prog); in bpf_overflow_handler()
10456 event->orig_overflow_handler(event, data, regs); in bpf_overflow_handler()
10463 if (event->overflow_handler_context) in perf_event_set_bpf_handler()
10465 return -EINVAL; in perf_event_set_bpf_handler()
10467 if (event->prog) in perf_event_set_bpf_handler()
10468 return -EEXIST; in perf_event_set_bpf_handler()
10470 if (prog->type != BPF_PROG_TYPE_PERF_EVENT) in perf_event_set_bpf_handler()
10471 return -EINVAL; in perf_event_set_bpf_handler()
10473 if (event->attr.precise_ip && in perf_event_set_bpf_handler()
10474 prog->call_get_stack && in perf_event_set_bpf_handler()
10475 (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) || in perf_event_set_bpf_handler()
10476 event->attr.exclude_callchain_kernel || in perf_event_set_bpf_handler()
10477 event->attr.exclude_callchain_user)) { in perf_event_set_bpf_handler()
10487 return -EPROTO; in perf_event_set_bpf_handler()
10490 event->prog = prog; in perf_event_set_bpf_handler()
10491 event->bpf_cookie = bpf_cookie; in perf_event_set_bpf_handler()
10492 event->orig_overflow_handler = READ_ONCE(event->overflow_handler); in perf_event_set_bpf_handler()
10493 WRITE_ONCE(event->overflow_handler, bpf_overflow_handler); in perf_event_set_bpf_handler()
10499 struct bpf_prog *prog = event->prog; in perf_event_free_bpf_handler()
10504 WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler); in perf_event_free_bpf_handler()
10505 event->prog = NULL; in perf_event_free_bpf_handler()
10513 return -EOPNOTSUPP; in perf_event_set_bpf_handler()
10526 if (event->pmu == &perf_tracepoint) in perf_event_is_tracing()
10529 if (event->pmu == &perf_kprobe) in perf_event_is_tracing()
10533 if (event->pmu == &perf_uprobe) in perf_event_is_tracing()
10547 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE; in perf_event_set_bpf_prog()
10548 is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE; in perf_event_set_bpf_prog()
10549 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; in perf_event_set_bpf_prog()
10550 is_syscall_tp = is_syscall_trace_event(event->tp_event); in perf_event_set_bpf_prog()
10553 return -EINVAL; in perf_event_set_bpf_prog()
10555 if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) || in perf_event_set_bpf_prog()
10556 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) || in perf_event_set_bpf_prog()
10557 (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) in perf_event_set_bpf_prog()
10558 return -EINVAL; in perf_event_set_bpf_prog()
10560 if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe) in perf_event_set_bpf_prog()
10562 return -EINVAL; in perf_event_set_bpf_prog()
10565 if (prog->kprobe_override && !is_kprobe) in perf_event_set_bpf_prog()
10566 return -EINVAL; in perf_event_set_bpf_prog()
10569 int off = trace_event_get_offsets(event->tp_event); in perf_event_set_bpf_prog()
10571 if (prog->aux->max_ctx_offset > off) in perf_event_set_bpf_prog()
10572 return -EACCES; in perf_event_set_bpf_prog()
10600 return -ENOENT; in perf_event_set_bpf_prog()
10614 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); in perf_bp_event()
10616 if (!bp->hw.state && !perf_exclude_event(bp, regs)) in perf_bp_event()
10627 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); in perf_addr_filter_new()
10634 INIT_LIST_HEAD(&filter->entry); in perf_addr_filter_new()
10635 list_add_tail(&filter->entry, filters); in perf_addr_filter_new()
10645 path_put(&filter->path); in free_filters_list()
10646 list_del(&filter->entry); in free_filters_list()
10664 if (event->parent) in perf_addr_filters_splice()
10667 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); in perf_addr_filters_splice()
10669 list_splice_init(&event->addr_filters.list, &list); in perf_addr_filters_splice()
10671 list_splice(head, &event->addr_filters.list); in perf_addr_filters_splice()
10673 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); in perf_addr_filters_splice()
10691 if (!vma->vm_file) in perf_addr_filter_apply()
10706 struct task_struct *task = READ_ONCE(event->ctx->task); in perf_event_addr_filters_apply()
10713 * We may observe TASK_TOMBSTONE, which means that the event tear-down in perf_event_addr_filters_apply()
10719 if (ifh->nr_file_filters) { in perf_event_addr_filters_apply()
10727 raw_spin_lock_irqsave(&ifh->lock, flags); in perf_event_addr_filters_apply()
10728 list_for_each_entry(filter, &ifh->list, entry) { in perf_event_addr_filters_apply()
10729 if (filter->path.dentry) { in perf_event_addr_filters_apply()
10734 event->addr_filter_ranges[count].start = 0; in perf_event_addr_filters_apply()
10735 event->addr_filter_ranges[count].size = 0; in perf_event_addr_filters_apply()
10737 perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]); in perf_event_addr_filters_apply()
10739 event->addr_filter_ranges[count].start = filter->offset; in perf_event_addr_filters_apply()
10740 event->addr_filter_ranges[count].size = filter->size; in perf_event_addr_filters_apply()
10746 event->addr_filters_gen++; in perf_event_addr_filters_apply()
10747 raw_spin_unlock_irqrestore(&ifh->lock, flags); in perf_event_addr_filters_apply()
10749 if (ifh->nr_file_filters) { in perf_event_addr_filters_apply()
10779 IF_ACT_NONE = -1,
10818 int ret = -EINVAL; in perf_event_parse_addr_filter()
10822 return -ENOMEM; in perf_event_parse_addr_filter()
10830 ret = -EINVAL; in perf_event_parse_addr_filter()
10850 filter->action = actions[token]; in perf_event_parse_addr_filter()
10865 ret = kstrtoul(args[0].from, 0, &filter->offset); in perf_event_parse_addr_filter()
10871 ret = kstrtoul(args[1].from, 0, &filter->size); in perf_event_parse_addr_filter()
10882 ret = -ENOMEM; in perf_event_parse_addr_filter()
10900 ret = -EINVAL; in perf_event_parse_addr_filter()
10903 * ACTION "filter" must have a non-zero length region in perf_event_parse_addr_filter()
10906 if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER && in perf_event_parse_addr_filter()
10907 !filter->size) in perf_event_parse_addr_filter()
10915 * For now, we only support file-based filters in perf_event_parse_addr_filter()
10916 * in per-task events; doing so for CPU-wide in perf_event_parse_addr_filter()
10922 ret = -EOPNOTSUPP; in perf_event_parse_addr_filter()
10923 if (!event->ctx->task) in perf_event_parse_addr_filter()
10928 &filter->path); in perf_event_parse_addr_filter()
10932 ret = -EINVAL; in perf_event_parse_addr_filter()
10933 if (!filter->path.dentry || in perf_event_parse_addr_filter()
10934 !S_ISREG(d_inode(filter->path.dentry) in perf_event_parse_addr_filter()
10935 ->i_mode)) in perf_event_parse_addr_filter()
10938 event->addr_filters.nr_file_filters++; in perf_event_parse_addr_filter()
10976 lockdep_assert_held(&event->ctx->mutex); in perf_event_set_addr_filter()
10978 if (WARN_ON_ONCE(event->parent)) in perf_event_set_addr_filter()
10979 return -EINVAL; in perf_event_set_addr_filter()
10985 ret = event->pmu->addr_filters_validate(&filters); in perf_event_set_addr_filter()
11001 event->addr_filters.nr_file_filters = 0; in perf_event_set_addr_filter()
11008 int ret = -EINVAL; in perf_event_set_filter()
11017 struct perf_event_context *ctx = event->ctx; in perf_event_set_filter()
11022 * the tracepoint muck will deadlock against ctx->mutex, but in perf_event_set_filter()
11024 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we in perf_event_set_filter()
11030 mutex_unlock(&ctx->mutex); in perf_event_set_filter()
11031 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); in perf_event_set_filter()
11032 mutex_lock(&ctx->mutex); in perf_event_set_filter()
11056 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_swevent_hrtimer()
11059 event->pmu->read(event); in perf_swevent_hrtimer()
11061 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_swevent_hrtimer()
11065 if (!(event->attr.exclude_idle && is_idle_task(current))) in perf_swevent_hrtimer()
11070 period = max_t(u64, 10000, event->hw.sample_period); in perf_swevent_hrtimer()
11078 struct hw_perf_event *hwc = &event->hw; in perf_swevent_start_hrtimer()
11084 period = local64_read(&hwc->period_left); in perf_swevent_start_hrtimer()
11089 local64_set(&hwc->period_left, 0); in perf_swevent_start_hrtimer()
11091 period = max_t(u64, 10000, hwc->sample_period); in perf_swevent_start_hrtimer()
11093 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), in perf_swevent_start_hrtimer()
11099 struct hw_perf_event *hwc = &event->hw; in perf_swevent_cancel_hrtimer()
11102 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); in perf_swevent_cancel_hrtimer()
11103 local64_set(&hwc->period_left, ktime_to_ns(remaining)); in perf_swevent_cancel_hrtimer()
11105 hrtimer_cancel(&hwc->hrtimer); in perf_swevent_cancel_hrtimer()
11111 struct hw_perf_event *hwc = &event->hw; in perf_swevent_init_hrtimer()
11116 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in perf_swevent_init_hrtimer()
11117 hwc->hrtimer.function = perf_swevent_hrtimer; in perf_swevent_init_hrtimer()
11120 * Since hrtimers have a fixed rate, we can do a static freq->period in perf_swevent_init_hrtimer()
11123 if (event->attr.freq) { in perf_swevent_init_hrtimer()
11124 long freq = event->attr.sample_freq; in perf_swevent_init_hrtimer()
11126 event->attr.sample_period = NSEC_PER_SEC / freq; in perf_swevent_init_hrtimer()
11127 hwc->sample_period = event->attr.sample_period; in perf_swevent_init_hrtimer()
11128 local64_set(&hwc->period_left, hwc->sample_period); in perf_swevent_init_hrtimer()
11129 hwc->last_period = hwc->sample_period; in perf_swevent_init_hrtimer()
11130 event->attr.freq = 0; in perf_swevent_init_hrtimer()
11144 prev = local64_xchg(&event->hw.prev_count, now); in cpu_clock_event_update()
11145 local64_add(now - prev, &event->count); in cpu_clock_event_update()
11150 local64_set(&event->hw.prev_count, local_clock()); in cpu_clock_event_start()
11181 if (event->attr.type != perf_cpu_clock.type) in cpu_clock_event_init()
11182 return -ENOENT; in cpu_clock_event_init()
11184 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) in cpu_clock_event_init()
11185 return -ENOENT; in cpu_clock_event_init()
11191 return -EOPNOTSUPP; in cpu_clock_event_init()
11221 prev = local64_xchg(&event->hw.prev_count, now); in task_clock_event_update()
11222 delta = now - prev; in task_clock_event_update()
11223 local64_add(delta, &event->count); in task_clock_event_update()
11228 local64_set(&event->hw.prev_count, event->ctx->time); in task_clock_event_start()
11235 task_clock_event_update(event, event->ctx->time); in task_clock_event_stop()
11255 u64 delta = now - event->ctx->timestamp; in task_clock_event_read()
11256 u64 time = event->ctx->time + delta; in task_clock_event_read()
11263 if (event->attr.type != perf_task_clock.type) in task_clock_event_init()
11264 return -ENOENT; in task_clock_event_init()
11266 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) in task_clock_event_init()
11267 return -ENOENT; in task_clock_event_init()
11273 return -EOPNOTSUPP; in task_clock_event_init()
11356 free_percpu(pmu->cpu_pmu_context); in free_pmu_context()
11368 return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); in nr_addr_filters_show()
11379 return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type); in type_show()
11390 return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms); in perf_event_mux_interval_ms_show()
11408 return -EINVAL; in perf_event_mux_interval_ms_store()
11411 if (timer == pmu->hrtimer_interval_ms) in perf_event_mux_interval_ms_store()
11415 pmu->hrtimer_interval_ms = timer; in perf_event_mux_interval_ms_store()
11421 cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu); in perf_event_mux_interval_ms_store()
11422 cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); in perf_event_mux_interval_ms_store()
11445 if (n == 2 && !pmu->nr_addr_filters) in pmu_dev_is_visible()
11448 return a->mode; in pmu_dev_is_visible()
11474 int ret = -ENOMEM; in pmu_dev_alloc()
11476 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); in pmu_dev_alloc()
11477 if (!pmu->dev) in pmu_dev_alloc()
11480 pmu->dev->groups = pmu->attr_groups; in pmu_dev_alloc()
11481 device_initialize(pmu->dev); in pmu_dev_alloc()
11483 dev_set_drvdata(pmu->dev, pmu); in pmu_dev_alloc()
11484 pmu->dev->bus = &pmu_bus; in pmu_dev_alloc()
11485 pmu->dev->parent = pmu->parent; in pmu_dev_alloc()
11486 pmu->dev->release = pmu_dev_release; in pmu_dev_alloc()
11488 ret = dev_set_name(pmu->dev, "%s", pmu->name); in pmu_dev_alloc()
11492 ret = device_add(pmu->dev); in pmu_dev_alloc()
11496 if (pmu->attr_update) { in pmu_dev_alloc()
11497 ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update); in pmu_dev_alloc()
11506 device_del(pmu->dev); in pmu_dev_alloc()
11509 put_device(pmu->dev); in pmu_dev_alloc()
11521 ret = -ENOMEM; in perf_pmu_register()
11522 pmu->pmu_disable_count = alloc_percpu(int); in perf_pmu_register()
11523 if (!pmu->pmu_disable_count) in perf_pmu_register()
11526 pmu->type = -1; in perf_pmu_register()
11528 ret = -EINVAL; in perf_pmu_register()
11532 pmu->name = name; in perf_pmu_register()
11544 pmu->type = type; in perf_pmu_register()
11546 if (pmu_bus_running && !pmu->dev) { in perf_pmu_register()
11552 ret = -ENOMEM; in perf_pmu_register()
11553 pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context); in perf_pmu_register()
11554 if (!pmu->cpu_pmu_context) in perf_pmu_register()
11560 cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu); in perf_pmu_register()
11561 __perf_init_event_pmu_context(&cpc->epc, pmu); in perf_pmu_register()
11565 if (!pmu->start_txn) { in perf_pmu_register()
11566 if (pmu->pmu_enable) { in perf_pmu_register()
11572 pmu->start_txn = perf_pmu_start_txn; in perf_pmu_register()
11573 pmu->commit_txn = perf_pmu_commit_txn; in perf_pmu_register()
11574 pmu->cancel_txn = perf_pmu_cancel_txn; in perf_pmu_register()
11576 pmu->start_txn = perf_pmu_nop_txn; in perf_pmu_register()
11577 pmu->commit_txn = perf_pmu_nop_int; in perf_pmu_register()
11578 pmu->cancel_txn = perf_pmu_nop_void; in perf_pmu_register()
11582 if (!pmu->pmu_enable) { in perf_pmu_register()
11583 pmu->pmu_enable = perf_pmu_nop_void; in perf_pmu_register()
11584 pmu->pmu_disable = perf_pmu_nop_void; in perf_pmu_register()
11587 if (!pmu->check_period) in perf_pmu_register()
11588 pmu->check_period = perf_event_nop_int; in perf_pmu_register()
11590 if (!pmu->event_idx) in perf_pmu_register()
11591 pmu->event_idx = perf_event_idx_default; in perf_pmu_register()
11593 list_add_rcu(&pmu->entry, &pmus); in perf_pmu_register()
11594 atomic_set(&pmu->exclusive_cnt, 0); in perf_pmu_register()
11602 if (pmu->dev && pmu->dev != PMU_NULL_DEV) { in perf_pmu_register()
11603 device_del(pmu->dev); in perf_pmu_register()
11604 put_device(pmu->dev); in perf_pmu_register()
11608 idr_remove(&pmu_idr, pmu->type); in perf_pmu_register()
11611 free_percpu(pmu->pmu_disable_count); in perf_pmu_register()
11619 list_del_rcu(&pmu->entry); in perf_pmu_unregister()
11628 free_percpu(pmu->pmu_disable_count); in perf_pmu_unregister()
11629 idr_remove(&pmu_idr, pmu->type); in perf_pmu_unregister()
11630 if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) { in perf_pmu_unregister()
11631 if (pmu->nr_addr_filters) in perf_pmu_unregister()
11632 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); in perf_pmu_unregister()
11633 device_del(pmu->dev); in perf_pmu_unregister()
11634 put_device(pmu->dev); in perf_pmu_unregister()
11643 return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) || in has_extended_regs()
11644 (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK); in has_extended_regs()
11652 if (!try_module_get(pmu->module)) in perf_try_init_event()
11653 return -ENODEV; in perf_try_init_event()
11656 * A number of pmu->event_init() methods iterate the sibling_list to, in perf_try_init_event()
11658 * if this is a sibling event, acquire the ctx->mutex to protect in perf_try_init_event()
11661 if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) { in perf_try_init_event()
11663 * This ctx->mutex can nest when we're called through in perf_try_init_event()
11666 ctx = perf_event_ctx_lock_nested(event->group_leader, in perf_try_init_event()
11671 event->pmu = pmu; in perf_try_init_event()
11672 ret = pmu->event_init(event); in perf_try_init_event()
11675 perf_event_ctx_unlock(event->group_leader, ctx); in perf_try_init_event()
11678 if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && in perf_try_init_event()
11680 ret = -EOPNOTSUPP; in perf_try_init_event()
11682 if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE && in perf_try_init_event()
11684 ret = -EINVAL; in perf_try_init_event()
11686 if (ret && event->destroy) in perf_try_init_event()
11687 event->destroy(event); in perf_try_init_event()
11691 module_put(pmu->module); in perf_try_init_event()
11705 * Save original type before calling pmu->event_init() since certain in perf_init_event()
11706 * pmus overwrites event->attr.type to forward event to another pmu. in perf_init_event()
11708 event->orig_type = event->attr.type; in perf_init_event()
11711 if (event->parent && event->parent->pmu) { in perf_init_event()
11712 pmu = event->parent->pmu; in perf_init_event()
11722 type = event->attr.type; in perf_init_event()
11724 type = event->attr.config >> PERF_PMU_TYPE_SHIFT; in perf_init_event()
11729 event->attr.config &= PERF_HW_EVENT_MASK; in perf_init_event()
11738 if (event->attr.type != type && type != PERF_TYPE_RAW && in perf_init_event()
11739 !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)) in perf_init_event()
11743 if (ret == -ENOENT && event->attr.type != type && !extended_type) { in perf_init_event()
11744 type = event->attr.type; in perf_init_event()
11759 if (ret != -ENOENT) { in perf_init_event()
11765 pmu = ERR_PTR(-ENOENT); in perf_init_event()
11774 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); in attach_sb_event()
11776 raw_spin_lock(&pel->lock); in attach_sb_event()
11777 list_add_rcu(&event->sb_list, &pel->list); in attach_sb_event()
11778 raw_spin_unlock(&pel->lock); in attach_sb_event()
11782 * We keep a list of all !task (and therefore per-cpu) events
11783 * that need to receive side-band records.
11785 * This avoids having to scan all the various PMU per-cpu contexts
11819 if (event->parent) in account_event()
11822 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) in account_event()
11824 if (event->attr.mmap || event->attr.mmap_data) in account_event()
11826 if (event->attr.build_id) in account_event()
11828 if (event->attr.comm) in account_event()
11830 if (event->attr.namespaces) in account_event()
11832 if (event->attr.cgroup) in account_event()
11834 if (event->attr.task) in account_event()
11836 if (event->attr.freq) in account_event()
11838 if (event->attr.context_switch) { in account_event()
11846 if (event->attr.ksymbol) in account_event()
11848 if (event->attr.bpf_event) in account_event()
11850 if (event->attr.text_poke) in account_event()
11874 * increments to by-pass the mutex. in account_event()
11898 long err = -EINVAL; in perf_event_alloc()
11902 if (!task || cpu != -1) in perf_event_alloc()
11903 return ERR_PTR(-EINVAL); in perf_event_alloc()
11905 if (attr->sigtrap && !task) { in perf_event_alloc()
11907 return ERR_PTR(-EINVAL); in perf_event_alloc()
11910 node = (cpu >= 0) ? cpu_to_node(cpu) : -1; in perf_event_alloc()
11914 return ERR_PTR(-ENOMEM); in perf_event_alloc()
11923 mutex_init(&event->child_mutex); in perf_event_alloc()
11924 INIT_LIST_HEAD(&event->child_list); in perf_event_alloc()
11926 INIT_LIST_HEAD(&event->event_entry); in perf_event_alloc()
11927 INIT_LIST_HEAD(&event->sibling_list); in perf_event_alloc()
11928 INIT_LIST_HEAD(&event->active_list); in perf_event_alloc()
11930 INIT_LIST_HEAD(&event->rb_entry); in perf_event_alloc()
11931 INIT_LIST_HEAD(&event->active_entry); in perf_event_alloc()
11932 INIT_LIST_HEAD(&event->addr_filters.list); in perf_event_alloc()
11933 INIT_HLIST_NODE(&event->hlist_entry); in perf_event_alloc()
11936 init_waitqueue_head(&event->waitq); in perf_event_alloc()
11937 init_irq_work(&event->pending_irq, perf_pending_irq); in perf_event_alloc()
11938 init_task_work(&event->pending_task, perf_pending_task); in perf_event_alloc()
11940 mutex_init(&event->mmap_mutex); in perf_event_alloc()
11941 raw_spin_lock_init(&event->addr_filters.lock); in perf_event_alloc()
11943 atomic_long_set(&event->refcount, 1); in perf_event_alloc()
11944 event->cpu = cpu; in perf_event_alloc()
11945 event->attr = *attr; in perf_event_alloc()
11946 event->group_leader = group_leader; in perf_event_alloc()
11947 event->pmu = NULL; in perf_event_alloc()
11948 event->oncpu = -1; in perf_event_alloc()
11950 event->parent = parent_event; in perf_event_alloc()
11952 event->ns = get_pid_ns(task_active_pid_ns(current)); in perf_event_alloc()
11953 event->id = atomic64_inc_return(&perf_event_id); in perf_event_alloc()
11955 event->state = PERF_EVENT_STATE_INACTIVE; in perf_event_alloc()
11958 event->event_caps = parent_event->event_caps; in perf_event_alloc()
11961 event->attach_state = PERF_ATTACH_TASK; in perf_event_alloc()
11967 event->hw.target = get_task_struct(task); in perf_event_alloc()
11970 event->clock = &local_clock; in perf_event_alloc()
11972 event->clock = parent_event->clock; in perf_event_alloc()
11975 overflow_handler = parent_event->overflow_handler; in perf_event_alloc()
11976 context = parent_event->overflow_handler_context; in perf_event_alloc()
11979 struct bpf_prog *prog = parent_event->prog; in perf_event_alloc()
11982 event->prog = prog; in perf_event_alloc()
11983 event->orig_overflow_handler = in perf_event_alloc()
11984 parent_event->orig_overflow_handler; in perf_event_alloc()
11990 event->overflow_handler = overflow_handler; in perf_event_alloc()
11991 event->overflow_handler_context = context; in perf_event_alloc()
11993 event->overflow_handler = perf_event_output_backward; in perf_event_alloc()
11994 event->overflow_handler_context = NULL; in perf_event_alloc()
11996 event->overflow_handler = perf_event_output_forward; in perf_event_alloc()
11997 event->overflow_handler_context = NULL; in perf_event_alloc()
12004 hwc = &event->hw; in perf_event_alloc()
12005 hwc->sample_period = attr->sample_period; in perf_event_alloc()
12006 if (attr->freq && attr->sample_freq) in perf_event_alloc()
12007 hwc->sample_period = 1; in perf_event_alloc()
12008 hwc->last_period = hwc->sample_period; in perf_event_alloc()
12010 local64_set(&hwc->period_left, hwc->sample_period); in perf_event_alloc()
12016 if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ)) in perf_event_alloc()
12020 event->attr.branch_sample_type = 0; in perf_event_alloc()
12029 * Disallow uncore-task events. Similarly, disallow uncore-cgroup in perf_event_alloc()
12033 if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) { in perf_event_alloc()
12034 err = -EINVAL; in perf_event_alloc()
12038 if (event->attr.aux_output && in perf_event_alloc()
12039 !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) { in perf_event_alloc()
12040 err = -EOPNOTSUPP; in perf_event_alloc()
12044 if (cgroup_fd != -1) { in perf_event_alloc()
12055 event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters, in perf_event_alloc()
12058 if (!event->addr_filter_ranges) { in perf_event_alloc()
12059 err = -ENOMEM; in perf_event_alloc()
12067 if (event->parent) { in perf_event_alloc()
12070 raw_spin_lock_irq(&ifh->lock); in perf_event_alloc()
12071 memcpy(event->addr_filter_ranges, in perf_event_alloc()
12072 event->parent->addr_filter_ranges, in perf_event_alloc()
12073 pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range)); in perf_event_alloc()
12074 raw_spin_unlock_irq(&ifh->lock); in perf_event_alloc()
12078 event->addr_filters_gen = 1; in perf_event_alloc()
12081 if (!event->parent) { in perf_event_alloc()
12082 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { in perf_event_alloc()
12083 err = get_callchain_buffers(attr->sample_max_stack); in perf_event_alloc()
12099 if (!event->parent) { in perf_event_alloc()
12100 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) in perf_event_alloc()
12104 kfree(event->addr_filter_ranges); in perf_event_alloc()
12112 if (event->destroy) in perf_event_alloc()
12113 event->destroy(event); in perf_event_alloc()
12114 module_put(pmu->module); in perf_event_alloc()
12116 if (event->hw.target) in perf_event_alloc()
12117 put_task_struct(event->hw.target); in perf_event_alloc()
12118 call_rcu(&event->rcu_head, free_event_rcu); in perf_event_alloc()
12132 ret = get_user(size, &uattr->size); in perf_copy_attr()
12144 if (ret == -E2BIG) in perf_copy_attr()
12149 attr->size = size; in perf_copy_attr()
12151 if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) in perf_copy_attr()
12152 return -EINVAL; in perf_copy_attr()
12154 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) in perf_copy_attr()
12155 return -EINVAL; in perf_copy_attr()
12157 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) in perf_copy_attr()
12158 return -EINVAL; in perf_copy_attr()
12160 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { in perf_copy_attr()
12161 u64 mask = attr->branch_sample_type; in perf_copy_attr()
12164 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) in perf_copy_attr()
12165 return -EINVAL; in perf_copy_attr()
12169 return -EINVAL; in perf_copy_attr()
12175 if (!attr->exclude_kernel) in perf_copy_attr()
12178 if (!attr->exclude_user) in perf_copy_attr()
12181 if (!attr->exclude_hv) in perf_copy_attr()
12186 attr->branch_sample_type = mask; in perf_copy_attr()
12196 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { in perf_copy_attr()
12197 ret = perf_reg_validate(attr->sample_regs_user); in perf_copy_attr()
12202 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { in perf_copy_attr()
12204 return -ENOSYS; in perf_copy_attr()
12211 if (attr->sample_stack_user >= USHRT_MAX) in perf_copy_attr()
12212 return -EINVAL; in perf_copy_attr()
12213 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) in perf_copy_attr()
12214 return -EINVAL; in perf_copy_attr()
12217 if (!attr->sample_max_stack) in perf_copy_attr()
12218 attr->sample_max_stack = sysctl_perf_event_max_stack; in perf_copy_attr()
12220 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) in perf_copy_attr()
12221 ret = perf_reg_validate(attr->sample_regs_intr); in perf_copy_attr()
12224 if (attr->sample_type & PERF_SAMPLE_CGROUP) in perf_copy_attr()
12225 return -EINVAL; in perf_copy_attr()
12227 if ((attr->sample_type & PERF_SAMPLE_WEIGHT) && in perf_copy_attr()
12228 (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) in perf_copy_attr()
12229 return -EINVAL; in perf_copy_attr()
12231 if (!attr->inherit && attr->inherit_thread) in perf_copy_attr()
12232 return -EINVAL; in perf_copy_attr()
12234 if (attr->remove_on_exec && attr->enable_on_exec) in perf_copy_attr()
12235 return -EINVAL; in perf_copy_attr()
12237 if (attr->sigtrap && !attr->remove_on_exec) in perf_copy_attr()
12238 return -EINVAL; in perf_copy_attr()
12244 put_user(sizeof(*attr), &uattr->size); in perf_copy_attr()
12245 ret = -E2BIG; in perf_copy_attr()
12262 int ret = -EINVAL; in perf_event_set_output()
12265 mutex_lock(&event->mmap_mutex); in perf_event_set_output()
12274 * Don't allow cross-cpu buffers in perf_event_set_output()
12276 if (output_event->cpu != event->cpu) in perf_event_set_output()
12280 * If its not a per-cpu rb, it must be the same task. in perf_event_set_output()
12282 if (output_event->cpu == -1 && output_event->hw.target != event->hw.target) in perf_event_set_output()
12288 if (output_event->clock != event->clock) in perf_event_set_output()
12302 event->pmu != output_event->pmu) in perf_event_set_output()
12307 * output_event is already on rb->event_list, and the list iteration in perf_event_set_output()
12310 * observe !rb->mmap_count. in perf_event_set_output()
12312 mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex); in perf_event_set_output()
12315 if (atomic_read(&event->mmap_count)) in perf_event_set_output()
12325 if (!atomic_read(&rb->mmap_count)) { in perf_event_set_output()
12335 mutex_unlock(&event->mmap_mutex); in perf_event_set_output()
12337 mutex_unlock(&output_event->mmap_mutex); in perf_event_set_output()
12349 event->clock = &ktime_get_mono_fast_ns; in perf_event_set_clock()
12354 event->clock = &ktime_get_raw_fast_ns; in perf_event_set_clock()
12359 event->clock = &ktime_get_real_ns; in perf_event_set_clock()
12363 event->clock = &ktime_get_boottime_ns; in perf_event_set_clock()
12367 event->clock = &ktime_get_clocktai_ns; in perf_event_set_clock()
12371 return -EINVAL; in perf_event_set_clock()
12374 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) in perf_event_set_clock()
12375 return -EINVAL; in perf_event_set_clock()
12386 if (attr->sigtrap) { in perf_check_permission()
12392 is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL); in perf_check_permission()
12412 * sys_perf_event_open - open a performance event, associate it to a task/cpu
12437 int cgroup_fd = -1; in SYSCALL_DEFINE5()
12441 return -EINVAL; in SYSCALL_DEFINE5()
12460 return -EACCES; in SYSCALL_DEFINE5()
12465 return -EINVAL; in SYSCALL_DEFINE5()
12468 return -EINVAL; in SYSCALL_DEFINE5()
12491 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) in SYSCALL_DEFINE5()
12492 return -EINVAL; in SYSCALL_DEFINE5()
12501 if (group_fd != -1) { in SYSCALL_DEFINE5()
12505 group_leader = group.file->private_data; in SYSCALL_DEFINE5()
12512 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { in SYSCALL_DEFINE5()
12521 group_leader->attr.inherit != attr.inherit) { in SYSCALL_DEFINE5()
12522 err = -EINVAL; in SYSCALL_DEFINE5()
12537 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { in SYSCALL_DEFINE5()
12538 err = -EOPNOTSUPP; in SYSCALL_DEFINE5()
12547 pmu = event->pmu; in SYSCALL_DEFINE5()
12555 if (pmu->task_ctx_nr == perf_sw_context) in SYSCALL_DEFINE5()
12556 event->event_caps |= PERF_EV_CAP_SOFTWARE; in SYSCALL_DEFINE5()
12559 err = down_read_interruptible(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
12569 err = -EACCES; in SYSCALL_DEFINE5()
12583 mutex_lock(&ctx->mutex); in SYSCALL_DEFINE5()
12585 if (ctx->task == TASK_TOMBSTONE) { in SYSCALL_DEFINE5()
12586 err = -ESRCH; in SYSCALL_DEFINE5()
12597 struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in SYSCALL_DEFINE5()
12599 if (!cpuctx->online) { in SYSCALL_DEFINE5()
12600 err = -ENODEV; in SYSCALL_DEFINE5()
12606 err = -EINVAL; in SYSCALL_DEFINE5()
12610 * becoming part of another group-sibling): in SYSCALL_DEFINE5()
12612 if (group_leader->group_leader != group_leader) in SYSCALL_DEFINE5()
12616 if (group_leader->clock != event->clock) in SYSCALL_DEFINE5()
12624 if (group_leader->cpu != event->cpu) in SYSCALL_DEFINE5()
12630 if (group_leader->ctx != ctx) in SYSCALL_DEFINE5()
12652 pmu = group_leader->pmu_ctx->pmu; in SYSCALL_DEFINE5()
12655 (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { in SYSCALL_DEFINE5()
12666 group_leader->pmu_ctx->pmu != pmu) in SYSCALL_DEFINE5()
12679 event->pmu_ctx = pmu_ctx; in SYSCALL_DEFINE5()
12688 err = -E2BIG; in SYSCALL_DEFINE5()
12693 err = -EINVAL; in SYSCALL_DEFINE5()
12702 err = -EBUSY; in SYSCALL_DEFINE5()
12706 WARN_ON_ONCE(ctx->parent_ctx); in SYSCALL_DEFINE5()
12722 put_pmu_ctx(group_leader->pmu_ctx); in SYSCALL_DEFINE5()
12726 put_pmu_ctx(sibling->pmu_ctx); in SYSCALL_DEFINE5()
12733 * (through the sibling list, which is still in-tact), we can in SYSCALL_DEFINE5()
12736 * By installing siblings first we NO-OP because they're not in SYSCALL_DEFINE5()
12740 sibling->pmu_ctx = pmu_ctx; in SYSCALL_DEFINE5()
12743 perf_install_in_context(ctx, sibling, sibling->cpu); in SYSCALL_DEFINE5()
12751 group_leader->pmu_ctx = pmu_ctx; in SYSCALL_DEFINE5()
12754 perf_install_in_context(ctx, group_leader, group_leader->cpu); in SYSCALL_DEFINE5()
12766 event->owner = current; in SYSCALL_DEFINE5()
12768 perf_install_in_context(ctx, event, event->cpu); in SYSCALL_DEFINE5()
12771 mutex_unlock(&ctx->mutex); in SYSCALL_DEFINE5()
12774 up_read(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
12778 mutex_lock(¤t->perf_event_mutex); in SYSCALL_DEFINE5()
12779 list_add_tail(&event->owner_entry, ¤t->perf_event_list); in SYSCALL_DEFINE5()
12780 mutex_unlock(¤t->perf_event_mutex); in SYSCALL_DEFINE5()
12793 put_pmu_ctx(event->pmu_ctx); in SYSCALL_DEFINE5()
12794 event->pmu_ctx = NULL; /* _free_event() */ in SYSCALL_DEFINE5()
12796 mutex_unlock(&ctx->mutex); in SYSCALL_DEFINE5()
12801 up_read(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
12839 if (attr->aux_output) in perf_event_create_kernel_counter()
12840 return ERR_PTR(-EINVAL); in perf_event_create_kernel_counter()
12843 overflow_handler, context, -1); in perf_event_create_kernel_counter()
12850 event->owner = TASK_TOMBSTONE; in perf_event_create_kernel_counter()
12851 pmu = event->pmu; in perf_event_create_kernel_counter()
12853 if (pmu->task_ctx_nr == perf_sw_context) in perf_event_create_kernel_counter()
12854 event->event_caps |= PERF_EV_CAP_SOFTWARE; in perf_event_create_kernel_counter()
12865 WARN_ON_ONCE(ctx->parent_ctx); in perf_event_create_kernel_counter()
12866 mutex_lock(&ctx->mutex); in perf_event_create_kernel_counter()
12867 if (ctx->task == TASK_TOMBSTONE) { in perf_event_create_kernel_counter()
12868 err = -ESRCH; in perf_event_create_kernel_counter()
12877 event->pmu_ctx = pmu_ctx; in perf_event_create_kernel_counter()
12888 if (!cpuctx->online) { in perf_event_create_kernel_counter()
12889 err = -ENODEV; in perf_event_create_kernel_counter()
12895 err = -EBUSY; in perf_event_create_kernel_counter()
12899 perf_install_in_context(ctx, event, event->cpu); in perf_event_create_kernel_counter()
12901 mutex_unlock(&ctx->mutex); in perf_event_create_kernel_counter()
12907 event->pmu_ctx = NULL; /* _free_event() */ in perf_event_create_kernel_counter()
12909 mutex_unlock(&ctx->mutex); in perf_event_create_kernel_counter()
12928 put_pmu_ctx(event->pmu_ctx); in __perf_pmu_remove()
12929 list_add(&event->migrate_entry, events); in __perf_pmu_remove()
12933 put_pmu_ctx(sibling->pmu_ctx); in __perf_pmu_remove()
12934 list_add(&sibling->migrate_entry, events); in __perf_pmu_remove()
12944 struct perf_event_context *old_ctx = event->ctx; in __perf_pmu_install_event()
12948 event->cpu = cpu; in __perf_pmu_install_event()
12950 event->pmu_ctx = epc; in __perf_pmu_install_event()
12952 if (event->state >= PERF_EVENT_STATE_OFF) in __perf_pmu_install_event()
12953 event->state = PERF_EVENT_STATE_INACTIVE; in __perf_pmu_install_event()
12957 * Now that event->ctx is updated and visible, put the old ctx. in __perf_pmu_install_event()
12968 * Re-instate events in 2 passes. in __perf_pmu_install()
12976 if (event->group_leader == event) in __perf_pmu_install()
12979 list_del(&event->migrate_entry); in __perf_pmu_install()
12988 list_del(&event->migrate_entry); in __perf_pmu_install()
12999 * Since per-cpu context is persistent, no need to grab an extra in perf_pmu_migrate_context()
13002 src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx; in perf_pmu_migrate_context()
13003 dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx; in perf_pmu_migrate_context()
13009 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); in perf_pmu_migrate_context()
13011 __perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events); in perf_pmu_migrate_context()
13012 __perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events); in perf_pmu_migrate_context()
13016 * Wait for the events to quiesce before re-instating them. in perf_pmu_migrate_context()
13023 mutex_unlock(&dst_ctx->mutex); in perf_pmu_migrate_context()
13024 mutex_unlock(&src_ctx->mutex); in perf_pmu_migrate_context()
13030 struct perf_event *parent_event = child_event->parent; in sync_child_event()
13033 if (child_event->attr.inherit_stat) { in sync_child_event()
13034 struct task_struct *task = child_event->ctx->task; in sync_child_event()
13045 atomic64_add(child_val, &parent_event->child_count); in sync_child_event()
13046 atomic64_add(child_event->total_time_enabled, in sync_child_event()
13047 &parent_event->child_total_time_enabled); in sync_child_event()
13048 atomic64_add(child_event->total_time_running, in sync_child_event()
13049 &parent_event->child_total_time_running); in sync_child_event()
13055 struct perf_event *parent_event = event->parent; in perf_event_exit_event()
13072 mutex_lock(&parent_event->child_mutex); in perf_event_exit_event()
13077 raw_spin_lock_irq(&ctx->lock); in perf_event_exit_event()
13078 if (event->state > PERF_EVENT_STATE_EXIT) in perf_event_exit_event()
13080 raw_spin_unlock_irq(&ctx->lock); in perf_event_exit_event()
13086 mutex_unlock(&parent_event->child_mutex); in perf_event_exit_event()
13114 * In order to reduce the amount of tricky in ctx tear-down, we hold in perf_event_exit_task_context()
13123 mutex_lock(&child_ctx->mutex); in perf_event_exit_task_context()
13126 * In a single ctx::lock section, de-schedule the events and detach the in perf_event_exit_task_context()
13130 raw_spin_lock_irq(&child_ctx->lock); in perf_event_exit_task_context()
13134 * Now that the context is inactive, destroy the task <-> ctx relation in perf_event_exit_task_context()
13137 RCU_INIT_POINTER(child->perf_event_ctxp, NULL); in perf_event_exit_task_context()
13139 WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE); in perf_event_exit_task_context()
13143 raw_spin_unlock_irq(&child_ctx->lock); in perf_event_exit_task_context()
13155 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) in perf_event_exit_task_context()
13158 mutex_unlock(&child_ctx->mutex); in perf_event_exit_task_context()
13173 mutex_lock(&child->perf_event_mutex); in perf_event_exit_task()
13174 list_for_each_entry_safe(event, tmp, &child->perf_event_list, in perf_event_exit_task()
13176 list_del_init(&event->owner_entry); in perf_event_exit_task()
13181 * we need to serialize on the owner->perf_event_mutex. in perf_event_exit_task()
13183 smp_store_release(&event->owner, NULL); in perf_event_exit_task()
13185 mutex_unlock(&child->perf_event_mutex); in perf_event_exit_task()
13192 * child contexts and sets child->perf_event_ctxp[] to NULL. in perf_event_exit_task()
13201 struct perf_event *parent = event->parent; in perf_free_event()
13206 mutex_lock(&parent->child_mutex); in perf_free_event()
13207 list_del_init(&event->child_list); in perf_free_event()
13208 mutex_unlock(&parent->child_mutex); in perf_free_event()
13212 raw_spin_lock_irq(&ctx->lock); in perf_free_event()
13215 raw_spin_unlock_irq(&ctx->lock); in perf_free_event()
13231 ctx = rcu_access_pointer(task->perf_event_ctxp); in perf_event_free_task()
13235 mutex_lock(&ctx->mutex); in perf_event_free_task()
13236 raw_spin_lock_irq(&ctx->lock); in perf_event_free_task()
13238 * Destroy the task <-> ctx relation and mark the context dead. in perf_event_free_task()
13243 RCU_INIT_POINTER(task->perf_event_ctxp, NULL); in perf_event_free_task()
13244 WRITE_ONCE(ctx->task, TASK_TOMBSTONE); in perf_event_free_task()
13246 raw_spin_unlock_irq(&ctx->lock); in perf_event_free_task()
13249 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) in perf_event_free_task()
13252 mutex_unlock(&ctx->mutex); in perf_event_free_task()
13263 * _free_event()'s put_task_struct(event->hw.target) will be a in perf_event_free_task()
13264 * use-after-free. in perf_event_free_task()
13268 wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1); in perf_event_free_task()
13274 WARN_ON_ONCE(task->perf_event_ctxp); in perf_event_delayed_put()
13281 return ERR_PTR(-EBADF); in perf_event_get()
13283 if (file->f_op != &perf_fops) { in perf_event_get()
13285 return ERR_PTR(-EBADF); in perf_event_get()
13293 if (file->f_op != &perf_fops) in perf_get_event()
13294 return ERR_PTR(-EINVAL); in perf_get_event()
13296 return file->private_data; in perf_get_event()
13302 return ERR_PTR(-EINVAL); in perf_event_attrs()
13304 return &event->attr; in perf_event_attrs()
13311 * - valid pointer on success
13312 * - NULL for orphaned events
13313 * - IS_ERR() on error
13323 enum perf_event_state parent_state = parent_event->state; in inherit_event()
13334 if (parent_event->parent) in inherit_event()
13335 parent_event = parent_event->parent; in inherit_event()
13337 child_event = perf_event_alloc(&parent_event->attr, in inherit_event()
13338 parent_event->cpu, in inherit_event()
13341 NULL, NULL, -1); in inherit_event()
13345 pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event); in inherit_event()
13350 child_event->pmu_ctx = pmu_ctx; in inherit_event()
13353 * is_orphaned_event() and list_add_tail(&parent_event->child_list) in inherit_event()
13358 mutex_lock(&parent_event->child_mutex); in inherit_event()
13360 !atomic_long_inc_not_zero(&parent_event->refcount)) { in inherit_event()
13361 mutex_unlock(&parent_event->child_mutex); in inherit_event()
13375 child_event->state = PERF_EVENT_STATE_INACTIVE; in inherit_event()
13377 child_event->state = PERF_EVENT_STATE_OFF; in inherit_event()
13379 if (parent_event->attr.freq) { in inherit_event()
13380 u64 sample_period = parent_event->hw.sample_period; in inherit_event()
13381 struct hw_perf_event *hwc = &child_event->hw; in inherit_event()
13383 hwc->sample_period = sample_period; in inherit_event()
13384 hwc->last_period = sample_period; in inherit_event()
13386 local64_set(&hwc->period_left, sample_period); in inherit_event()
13389 child_event->ctx = child_ctx; in inherit_event()
13390 child_event->overflow_handler = parent_event->overflow_handler; in inherit_event()
13391 child_event->overflow_handler_context in inherit_event()
13392 = parent_event->overflow_handler_context; in inherit_event()
13403 raw_spin_lock_irqsave(&child_ctx->lock, flags); in inherit_event()
13405 child_event->attach_state |= PERF_ATTACH_CHILD; in inherit_event()
13406 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); in inherit_event()
13411 list_add_tail(&child_event->child_list, &parent_event->child_list); in inherit_event()
13412 mutex_unlock(&parent_event->child_mutex); in inherit_event()
13424 * - 0 on success
13425 * - <0 on error
13452 if (sub->aux_event == parent_event && child_ctr && in inherit_group()
13454 return -EINVAL; in inherit_group()
13457 leader->group_generation = parent_event->group_generation; in inherit_group()
13462 * Creates the child task context and tries to inherit the event-group.
13469 * - 0 on success
13470 * - <0 on error
13481 if (!event->attr.inherit || in inherit_task_group()
13482 (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) || in inherit_task_group()
13484 (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) { in inherit_task_group()
13489 child_ctx = child->perf_event_ctxp; in inherit_task_group()
13499 return -ENOMEM; in inherit_task_group()
13501 child->perf_event_ctxp = child_ctx; in inherit_task_group()
13524 if (likely(!parent->perf_event_ctxp)) in perf_event_init_context()
13537 * it non-NULL earlier, the only reason for it to become NULL in perf_event_init_context()
13543 * Lock the parent list. No need to lock the child - not PID in perf_event_init_context()
13546 mutex_lock(&parent_ctx->mutex); in perf_event_init_context()
13549 * We dont have to disable NMIs - we are only looking at in perf_event_init_context()
13552 perf_event_groups_for_each(event, &parent_ctx->pinned_groups) { in perf_event_init_context()
13560 * We can't hold ctx->lock when iterating the ->flexible_group list due in perf_event_init_context()
13564 raw_spin_lock_irqsave(&parent_ctx->lock, flags); in perf_event_init_context()
13565 parent_ctx->rotate_disable = 1; in perf_event_init_context()
13566 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); in perf_event_init_context()
13568 perf_event_groups_for_each(event, &parent_ctx->flexible_groups) { in perf_event_init_context()
13575 raw_spin_lock_irqsave(&parent_ctx->lock, flags); in perf_event_init_context()
13576 parent_ctx->rotate_disable = 0; in perf_event_init_context()
13578 child_ctx = child->perf_event_ctxp; in perf_event_init_context()
13586 * parent_ctx->lock avoids it from being uncloned. in perf_event_init_context()
13588 cloned_ctx = parent_ctx->parent_ctx; in perf_event_init_context()
13590 child_ctx->parent_ctx = cloned_ctx; in perf_event_init_context()
13591 child_ctx->parent_gen = parent_ctx->parent_gen; in perf_event_init_context()
13593 child_ctx->parent_ctx = parent_ctx; in perf_event_init_context()
13594 child_ctx->parent_gen = parent_ctx->generation; in perf_event_init_context()
13596 get_ctx(child_ctx->parent_ctx); in perf_event_init_context()
13599 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); in perf_event_init_context()
13601 mutex_unlock(&parent_ctx->mutex); in perf_event_init_context()
13616 child->perf_event_ctxp = NULL; in perf_event_init_task()
13617 mutex_init(&child->perf_event_mutex); in perf_event_init_task()
13618 INIT_LIST_HEAD(&child->perf_event_list); in perf_event_init_task()
13639 mutex_init(&swhash->hlist_mutex); in perf_event_init_all_cpus()
13647 __perf_event_init_context(&cpuctx->ctx); in perf_event_init_all_cpus()
13648 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); in perf_event_init_all_cpus()
13649 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); in perf_event_init_all_cpus()
13650 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); in perf_event_init_all_cpus()
13651 cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default); in perf_event_init_all_cpus()
13652 cpuctx->heap = cpuctx->heap_default; in perf_event_init_all_cpus()
13660 mutex_lock(&swhash->hlist_mutex); in perf_swevent_init_cpu()
13661 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { in perf_swevent_init_cpu()
13666 rcu_assign_pointer(swhash->swevent_hlist, hlist); in perf_swevent_init_cpu()
13668 mutex_unlock(&swhash->hlist_mutex); in perf_swevent_init_cpu()
13678 raw_spin_lock(&ctx->lock); in __perf_event_exit_context()
13680 list_for_each_entry(event, &ctx->event_list, event_entry) in __perf_event_exit_context()
13682 raw_spin_unlock(&ctx->lock); in __perf_event_exit_context()
13690 // XXX simplify cpuctx->online in perf_event_exit_cpu_context()
13693 ctx = &cpuctx->ctx; in perf_event_exit_cpu_context()
13695 mutex_lock(&ctx->mutex); in perf_event_exit_cpu_context()
13697 cpuctx->online = 0; in perf_event_exit_cpu_context()
13698 mutex_unlock(&ctx->mutex); in perf_event_exit_cpu_context()
13718 ctx = &cpuctx->ctx; in perf_event_init_cpu()
13720 mutex_lock(&ctx->mutex); in perf_event_init_cpu()
13721 cpuctx->online = 1; in perf_event_init_cpu()
13722 mutex_unlock(&ctx->mutex); in perf_event_init_cpu()
13763 perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1); in perf_event_init()
13764 perf_pmu_register(&perf_task_clock, "task_clock", -1); in perf_event_init()
13788 if (pmu_attr->event_str) in perf_event_sysfs_show()
13789 return sprintf(page, "%s\n", pmu_attr->event_str); in perf_event_sysfs_show()
13807 if (pmu->dev) in perf_event_sysfs_init()
13811 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); in perf_event_sysfs_init()
13831 return ERR_PTR(-ENOMEM); in perf_cgroup_css_alloc()
13833 jc->info = alloc_percpu(struct perf_cgroup_info); in perf_cgroup_css_alloc()
13834 if (!jc->info) { in perf_cgroup_css_alloc()
13836 return ERR_PTR(-ENOMEM); in perf_cgroup_css_alloc()
13839 return &jc->css; in perf_cgroup_css_alloc()
13846 free_percpu(jc->info); in perf_cgroup_css_free()
13852 perf_event_cgroup(css->cgroup); in perf_cgroup_css_online()