39 	struct perf_event *event[MAX_HWEVENTS];  member
109 static inline void perf_get_data_addr(struct perf_event *event, struct pt_regs *regs, u64 *addrp) {…  in perf_get_data_addr()  argument
124 static bool is_ebb_event(struct perf_event *event) { return false; }  in is_ebb_event()  argument
125 static int ebb_event_check(struct perf_event *event) { return 0; }  in ebb_event_check()  argument
126 static void ebb_event_add(struct perf_event *event) { }  in ebb_event_add()  argument
133 static inline void power_pmu_bhrb_enable(struct perf_event *event) {}  in power_pmu_bhrb_enable()  argument
134 static inline void power_pmu_bhrb_disable(struct perf_event *event) {}  in power_pmu_bhrb_disable()  argument
139 static inline void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw) {}  in power_pmu_bhrb_read()  argument
205 static inline void perf_get_data_addr(struct perf_event *event, struct pt_regs *regs, u64 *addrp)  in perf_get_data_addr()  argument
230 	if (is_kernel_addr(mfspr(SPRN_SDAR)) && event->attr.exclude_kernel)  in perf_get_data_addr()
410 static void power_pmu_bhrb_enable(struct perf_event *event)  in power_pmu_bhrb_enable()  argument
418 	if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {  in power_pmu_bhrb_enable()
420 		cpuhw->bhrb_context = event->ctx;  in power_pmu_bhrb_enable()
423 	perf_sched_cb_inc(event->pmu);  in power_pmu_bhrb_enable()
426 static void power_pmu_bhrb_disable(struct perf_event *event)  in power_pmu_bhrb_disable()  argument
435 	perf_sched_cb_dec(event->pmu);  in power_pmu_bhrb_disable()
487 static void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw)  in power_pmu_bhrb_read()  argument
518 			    is_kernel_addr(addr) && event->attr.exclude_kernel)  in power_pmu_bhrb_read()
575 static bool is_ebb_event(struct perf_event *event)  in is_ebb_event()  argument
583 	       ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1);  in is_ebb_event()
586 static int ebb_event_check(struct perf_event *event)  in ebb_event_check()  argument
588 	struct perf_event *leader = event->group_leader;  in ebb_event_check()
591 	if (is_ebb_event(leader) != is_ebb_event(event))  in ebb_event_check()
594 	if (is_ebb_event(event)) {  in ebb_event_check()
595 		if (!(event->attach_state & PERF_ATTACH_TASK))  in ebb_event_check()
601 		if (event->attr.freq ||  in ebb_event_check()
602 		    event->attr.inherit ||  in ebb_event_check()
603 		    event->attr.sample_type ||  in ebb_event_check()
604 		    event->attr.sample_period ||  in ebb_event_check()
605 		    event->attr.enable_on_exec)  in ebb_event_check()
612 static void ebb_event_add(struct perf_event *event)  in ebb_event_add()  argument
614 	if (!is_ebb_event(event) || current->thread.used_ebb)  in ebb_event_add()
891 		idx = cpuhw->event[i]->hw.idx;  in any_pmc_overflown()
968 				   int n_ev, struct perf_event **event)  in power_check_constraints()  argument
991 					 &cpuhw->avalues[i][0], event[i]->attr.config1))  in power_check_constraints()
1027 					     event[i]->attr.config1);  in power_check_constraints()
1094 	struct perf_event *event;  in check_excludes()  local
1114 		event = ctrs[i];  in check_excludes()
1116 			eu = event->attr.exclude_user;  in check_excludes()
1117 			ek = event->attr.exclude_kernel;  in check_excludes()
1118 			eh = event->attr.exclude_hv;  in check_excludes()
1120 		} else if (event->attr.exclude_user != eu ||  in check_excludes()
1121 			   event->attr.exclude_kernel != ek ||  in check_excludes()
1122 			   event->attr.exclude_hv != eh) {  in check_excludes()
1154 static void power_pmu_read(struct perf_event *event)  in power_pmu_read()  argument
1158 	if (event->hw.state & PERF_HES_STOPPED)  in power_pmu_read()
1161 	if (!event->hw.idx)  in power_pmu_read()
1164 	if (is_ebb_event(event)) {  in power_pmu_read()
1165 		val = read_pmc(event->hw.idx);  in power_pmu_read()
1166 		local64_set(&event->hw.prev_count, val);  in power_pmu_read()
1176 		prev = local64_read(&event->hw.prev_count);  in power_pmu_read()
1178 		val = read_pmc(event->hw.idx);  in power_pmu_read()
1182 	} while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);  in power_pmu_read()
1184 	local64_add(delta, &event->count);  in power_pmu_read()
1196 		prev = local64_read(&event->hw.period_left);  in power_pmu_read()
1200 	} while (local64_cmpxchg(&event->hw.period_left, prev, val) != prev);  in power_pmu_read()
1217 	struct perf_event *event;  in freeze_limited_counters()  local
1222 		event = cpuhw->limited_counter[i];  in freeze_limited_counters()
1223 		if (!event->hw.idx)  in freeze_limited_counters()
1225 		val = (event->hw.idx == 5) ? pmc5 : pmc6;  in freeze_limited_counters()
1226 		prev = local64_read(&event->hw.prev_count);  in freeze_limited_counters()
1227 		event->hw.idx = 0;  in freeze_limited_counters()
1230 			local64_add(delta, &event->count);  in freeze_limited_counters()
1237 	struct perf_event *event;  in thaw_limited_counters()  local
1242 		event = cpuhw->limited_counter[i];  in thaw_limited_counters()
1243 		event->hw.idx = cpuhw->limited_hwidx[i];  in thaw_limited_counters()
1244 		val = (event->hw.idx == 5) ? pmc5 : pmc6;  in thaw_limited_counters()
1245 		prev = local64_read(&event->hw.prev_count);  in thaw_limited_counters()
1247 			local64_set(&event->hw.prev_count, val);  in thaw_limited_counters()
1248 		perf_event_update_userpage(event);  in thaw_limited_counters()
1416 	struct perf_event *event;  in power_pmu_enable()  local
1447 	ebb = is_ebb_event(cpuhw->event[0]);  in power_pmu_enable()
1478 			       &cpuhw->mmcr, cpuhw->event, ppmu->flags)) {  in power_pmu_enable()
1490 		event = cpuhw->event[0];  in power_pmu_enable()
1491 		if (event->attr.exclude_user)  in power_pmu_enable()
1493 		if (event->attr.exclude_kernel)  in power_pmu_enable()
1495 		if (event->attr.exclude_hv)  in power_pmu_enable()
1520 		event = cpuhw->event[i];  in power_pmu_enable()
1521 		if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) {  in power_pmu_enable()
1522 			power_pmu_read(event);  in power_pmu_enable()
1523 			write_pmc(event->hw.idx, 0);  in power_pmu_enable()
1524 			event->hw.idx = 0;  in power_pmu_enable()
1533 		event = cpuhw->event[i];  in power_pmu_enable()
1534 		if (event->hw.idx)  in power_pmu_enable()
1538 			cpuhw->limited_counter[n_lim] = event;  in power_pmu_enable()
1545 			val = local64_read(&event->hw.prev_count);  in power_pmu_enable()
1548 			if (event->hw.sample_period) {  in power_pmu_enable()
1549 				left = local64_read(&event->hw.period_left);  in power_pmu_enable()
1553 			local64_set(&event->hw.prev_count, val);  in power_pmu_enable()
1556 		event->hw.idx = idx;  in power_pmu_enable()
1557 		if (event->hw.state & PERF_HES_STOPPED)  in power_pmu_enable()
1561 		perf_event_update_userpage(event);  in power_pmu_enable()
1595 	struct perf_event *event;  in collect_events()  local
1604 	for_each_sibling_event(event, group) {  in collect_events()
1605 		if (event->pmu->task_ctx_nr == perf_hw_context &&  in collect_events()
1606 		    event->state != PERF_EVENT_STATE_OFF) {  in collect_events()
1609 			ctrs[n] = event;  in collect_events()
1610 			flags[n] = event->hw.event_base;  in collect_events()
1611 			events[n++] = event->hw.config;  in collect_events()
1623 static int power_pmu_add(struct perf_event *event, int ef_flags)  in power_pmu_add()  argument
1631 	perf_pmu_disable(event->pmu);  in power_pmu_add()
1641 	cpuhw->event[n0] = event;  in power_pmu_add()
1642 	cpuhw->events[n0] = event->hw.config;  in power_pmu_add()
1643 	cpuhw->flags[n0] = event->hw.event_base;  in power_pmu_add()
1652 		event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;  in power_pmu_add()
1654 		event->hw.state = 0;  in power_pmu_add()
1664 	if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1))  in power_pmu_add()
1666 	if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1, cpuhw->event))  in power_pmu_add()
1668 	event->hw.config = cpuhw->events[n0];  in power_pmu_add()
1671 	ebb_event_add(event);  in power_pmu_add()
1678 	if (has_branch_stack(event)) {  in power_pmu_add()
1683 				event->attr.branch_sample_type);  in power_pmu_add()
1687 			power_pmu_bhrb_enable(event);  in power_pmu_add()
1691 	perf_pmu_enable(event->pmu);  in power_pmu_add()
1699 static void power_pmu_del(struct perf_event *event, int ef_flags)  in power_pmu_del()  argument
1706 	perf_pmu_disable(event->pmu);  in power_pmu_del()
1708 	power_pmu_read(event);  in power_pmu_del()
1712 		if (event == cpuhw->event[i]) {  in power_pmu_del()
1714 				cpuhw->event[i-1] = cpuhw->event[i];  in power_pmu_del()
1719 			ppmu->disable_pmc(event->hw.idx - 1, &cpuhw->mmcr);  in power_pmu_del()
1720 			if (event->hw.idx) {  in power_pmu_del()
1721 				write_pmc(event->hw.idx, 0);  in power_pmu_del()
1722 				event->hw.idx = 0;  in power_pmu_del()
1724 			perf_event_update_userpage(event);  in power_pmu_del()
1729 		if (event == cpuhw->limited_counter[i])  in power_pmu_del()
1743 	if (has_branch_stack(event))  in power_pmu_del()
1744 		power_pmu_bhrb_disable(event);  in power_pmu_del()
1746 	perf_pmu_enable(event->pmu);  in power_pmu_del()
1755 static void power_pmu_start(struct perf_event *event, int ef_flags)  in power_pmu_start()  argument
1761 	if (!event->hw.idx || !event->hw.sample_period)  in power_pmu_start()
1764 	if (!(event->hw.state & PERF_HES_STOPPED))  in power_pmu_start()
1768 		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));  in power_pmu_start()
1771 	perf_pmu_disable(event->pmu);  in power_pmu_start()
1773 	event->hw.state = 0;  in power_pmu_start()
1774 	left = local64_read(&event->hw.period_left);  in power_pmu_start()
1780 	write_pmc(event->hw.idx, val);  in power_pmu_start()
1782 	perf_event_update_userpage(event);  in power_pmu_start()
1783 	perf_pmu_enable(event->pmu);  in power_pmu_start()
1787 static void power_pmu_stop(struct perf_event *event, int ef_flags)  in power_pmu_stop()  argument
1791 	if (!event->hw.idx || !event->hw.sample_period)  in power_pmu_stop()
1794 	if (event->hw.state & PERF_HES_STOPPED)  in power_pmu_stop()
1798 	perf_pmu_disable(event->pmu);  in power_pmu_stop()
1800 	power_pmu_read(event);  in power_pmu_stop()
1801 	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;  in power_pmu_stop()
1802 	write_pmc(event->hw.idx, 0);  in power_pmu_stop()
1804 	perf_event_update_userpage(event);  in power_pmu_stop()
1805 	perf_pmu_enable(event->pmu);  in power_pmu_stop()
1874 	if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))  in power_pmu_commit_txn()
1876 	i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n, cpuhw->event);  in power_pmu_commit_txn()
1881 		cpuhw->event[i]->hw.config = cpuhw->events[i];  in power_pmu_commit_txn()
1895 static int can_go_on_limited_pmc(struct perf_event *event, u64 ev,  in can_go_on_limited_pmc()  argument
1901 	if (event->attr.exclude_user  in can_go_on_limited_pmc()
1902 	    || event->attr.exclude_kernel  in can_go_on_limited_pmc()
1903 	    || event->attr.exclude_hv  in can_go_on_limited_pmc()
1904 	    || event->attr.sample_period)  in can_go_on_limited_pmc()
1948 static void hw_perf_event_destroy(struct perf_event *event)  in hw_perf_event_destroy()  argument
2000 static int power_pmu_event_init(struct perf_event *event)  in power_pmu_event_init()  argument
2014 	if (has_branch_stack(event)) {  in power_pmu_event_init()
2020 	switch (event->attr.type) {  in power_pmu_event_init()
2022 		ev = event->attr.config;  in power_pmu_event_init()
2031 		err = hw_perf_cache_event(event->attr.config, &ev);  in power_pmu_event_init()
2039 		ev = event->attr.config;  in power_pmu_event_init()
2056 	    ppmu->check_attr_config(event))  in power_pmu_event_init()
2059 	event->hw.config_base = ev;  in power_pmu_event_init()
2060 	event->hw.idx = 0;  in power_pmu_event_init()
2068 		event->attr.exclude_hv = 0;  in power_pmu_event_init()
2077 	if (event->attach_state & PERF_ATTACH_TASK)  in power_pmu_event_init()
2085 		if (can_go_on_limited_pmc(event, ev, flags)) {  in power_pmu_event_init()
2100 	err = ebb_event_check(event);  in power_pmu_event_init()
2110 	if (event->group_leader != event) {  in power_pmu_event_init()
2111 		n = collect_events(event->group_leader, ppmu->n_counter - 1,  in power_pmu_event_init()
2117 	ctrs[n] = event;  in power_pmu_event_init()
2127 	if (has_branch_stack(event)) {  in power_pmu_event_init()
2142 		if (hweight64(event->attr.branch_sample_type & ~PERF_SAMPLE_BRANCH_PLM_ALL) > 1) {  in power_pmu_event_init()
2149 					event->attr.branch_sample_type);  in power_pmu_event_init()
2162 	event->hw.config = events[n];  in power_pmu_event_init()
2163 	event->hw.event_base = cflags[n];  in power_pmu_event_init()
2164 	event->hw.last_period = event->hw.sample_period;  in power_pmu_event_init()
2165 	local64_set(&event->hw.period_left, event->hw.last_period);  in power_pmu_event_init()
2171 	if (is_ebb_event(event))  in power_pmu_event_init()
2172 		local64_set(&event->hw.prev_count, 0);  in power_pmu_event_init()
2190 	event->destroy = hw_perf_event_destroy;  in power_pmu_event_init()
2195 static int power_pmu_event_idx(struct perf_event *event)  in power_pmu_event_idx()  argument
2197 	return event->hw.idx;  in power_pmu_event_idx()
2238 static void record_and_restart(struct perf_event *event, unsigned long val,  in record_and_restart()  argument
2241 	u64 period = event->hw.sample_period;  in record_and_restart()
2242 	const u64 last_period = event->hw.last_period;  in record_and_restart()
2246 	if (event->hw.state & PERF_HES_STOPPED) {  in record_and_restart()
2247 		write_pmc(event->hw.idx, 0);  in record_and_restart()
2252 	prev = local64_read(&event->hw.prev_count);  in record_and_restart()
2254 	local64_add(delta, &event->count);  in record_and_restart()
2261 	left = local64_read(&event->hw.period_left) - delta;  in record_and_restart()
2275 			if (event->attr.sample_type & PERF_SAMPLE_IP)  in record_and_restart()
2280 			event->hw.last_period = event->hw.sample_period;  in record_and_restart()
2286 	write_pmc(event->hw.idx, val);  in record_and_restart()
2287 	local64_set(&event->hw.prev_count, val);  in record_and_restart()
2288 	local64_set(&event->hw.period_left, left);  in record_and_restart()
2289 	perf_event_update_userpage(event);  in record_and_restart()
2297 	if (event->attr.exclude_kernel &&  in record_and_restart()
2298 	    (event->attr.sample_type & PERF_SAMPLE_IP) &&  in record_and_restart()
2309 	if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&  in record_and_restart()
2314 			atomic64_inc(&event->lost_samples);  in record_and_restart()
2326 		if (event->attr.sample_type & PERF_SAMPLE_ADDR_TYPE)  in record_and_restart()
2327 			perf_get_data_addr(event, regs, &data.addr);  in record_and_restart()
2329 		if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {  in record_and_restart()
2332 			power_pmu_bhrb_read(event, cpuhw);  in record_and_restart()
2333 			perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack, NULL);  in record_and_restart()
2336 		if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&  in record_and_restart()
2342 		if (event->attr.sample_type & PERF_SAMPLE_WEIGHT_TYPE &&  in record_and_restart()
2344 			ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type);  in record_and_restart()
2347 		perf_event_overflow(event, &data, regs);  in record_and_restart()
2350 		perf_event_account_interrupt(event);  in record_and_restart()
2416 	struct perf_event *event;  in __perf_event_interrupt()  local
2444 			event = cpuhw->event[j];  in __perf_event_interrupt()
2445 			if (event->hw.idx == (i + 1)) {  in __perf_event_interrupt()
2447 				record_and_restart(event, cpuhw->pmcs[i], regs);  in __perf_event_interrupt()
2466 			event = cpuhw->event[i];  in __perf_event_interrupt()
2467 			if (!event->hw.idx || is_limited_pmc(event->hw.idx))  in __perf_event_interrupt()
2469 			if (pmc_overflow_power7(cpuhw->pmcs[event->hw.idx - 1])) {  in __perf_event_interrupt()
2472 				record_and_restart(event,  in __perf_event_interrupt()
2473 						   cpuhw->pmcs[event->hw.idx - 1],  in __perf_event_interrupt()