1 /*
2 * Performance events x86 architecture header
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15 #include <linux/perf_event.h>
16
17 #include <asm/fpu/xstate.h>
18 #include <asm/intel_ds.h>
19 #include <asm/cpu.h>
20
21 /* To enable MSR tracing please use the generic trace points. */
22
23 /*
24 * | NHM/WSM | SNB |
25 * register -------------------------------
26 * | HT | no HT | HT | no HT |
27 *-----------------------------------------
28 * offcore | core | core | cpu | core |
29 * lbr_sel | core | core | cpu | core |
30 * ld_lat | cpu | core | cpu | core |
31 *-----------------------------------------
32 *
33 * Given that there is a small number of shared regs,
34 * we can pre-allocate their slot in the per-cpu
35 * per-core reg tables.
36 */
37 enum extra_reg_type {
38 EXTRA_REG_NONE = -1, /* not used */
39
40 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
41 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
42 EXTRA_REG_LBR = 2, /* lbr_select */
43 EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */
44 EXTRA_REG_FE = 4, /* fe_* */
45 EXTRA_REG_SNOOP_0 = 5, /* snoop response 0 */
46 EXTRA_REG_SNOOP_1 = 6, /* snoop response 1 */
47
48 EXTRA_REG_MAX /* number of entries needed */
49 };
50
51 struct event_constraint {
52 union {
53 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
54 u64 idxmsk64;
55 };
56 u64 code;
57 u64 cmask;
58 int weight;
59 int overlap;
60 int flags;
61 unsigned int size;
62 };
63
constraint_match(struct event_constraint * c,u64 ecode)64 static inline bool constraint_match(struct event_constraint *c, u64 ecode)
65 {
66 return ((ecode & c->cmask) - c->code) <= (u64)c->size;
67 }
68
69 #define PERF_ARCH(name, val) \
70 PERF_X86_EVENT_##name = val,
71
72 /*
73 * struct hw_perf_event.flags flags
74 */
75 enum {
76 #include "perf_event_flags.h"
77 };
78
79 #undef PERF_ARCH
80
81 #define PERF_ARCH(name, val) \
82 static_assert((PERF_X86_EVENT_##name & PERF_EVENT_FLAG_ARCH) == \
83 PERF_X86_EVENT_##name);
84
85 #include "perf_event_flags.h"
86
87 #undef PERF_ARCH
88
is_topdown_count(struct perf_event * event)89 static inline bool is_topdown_count(struct perf_event *event)
90 {
91 return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
92 }
93
is_metric_event(struct perf_event * event)94 static inline bool is_metric_event(struct perf_event *event)
95 {
96 u64 config = event->attr.config;
97
98 return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
99 ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING) &&
100 ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
101 }
102
is_slots_event(struct perf_event * event)103 static inline bool is_slots_event(struct perf_event *event)
104 {
105 return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
106 }
107
is_topdown_event(struct perf_event * event)108 static inline bool is_topdown_event(struct perf_event *event)
109 {
110 return is_metric_event(event) || is_slots_event(event);
111 }
112
113 int is_x86_event(struct perf_event *event);
114
check_leader_group(struct perf_event * leader,int flags)115 static inline bool check_leader_group(struct perf_event *leader, int flags)
116 {
117 return is_x86_event(leader) ? !!(leader->hw.flags & flags) : false;
118 }
119
is_branch_counters_group(struct perf_event * event)120 static inline bool is_branch_counters_group(struct perf_event *event)
121 {
122 return check_leader_group(event->group_leader, PERF_X86_EVENT_BRANCH_COUNTERS);
123 }
124
is_pebs_counter_event_group(struct perf_event * event)125 static inline bool is_pebs_counter_event_group(struct perf_event *event)
126 {
127 return check_leader_group(event->group_leader, PERF_X86_EVENT_PEBS_CNTR);
128 }
129
130 struct amd_nb {
131 int nb_id; /* NorthBridge id */
132 int refcnt; /* reference count */
133 struct perf_event *owners[X86_PMC_IDX_MAX];
134 struct event_constraint event_constraints[X86_PMC_IDX_MAX];
135 };
136
137 #define PEBS_COUNTER_MASK ((1ULL << MAX_PEBS_EVENTS) - 1)
138 #define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60)
139 #define PEBS_OUTPUT_OFFSET 61
140 #define PEBS_OUTPUT_MASK (3ull << PEBS_OUTPUT_OFFSET)
141 #define PEBS_OUTPUT_PT (1ull << PEBS_OUTPUT_OFFSET)
142 #define PEBS_VIA_PT_MASK (PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD)
143
144 /*
145 * Flags PEBS can handle without an PMI.
146 *
147 * TID can only be handled by flushing at context switch.
148 * REGS_USER can be handled for events limited to ring 3.
149 *
150 */
151 #define LARGE_PEBS_FLAGS \
152 (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
153 PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
154 PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
155 PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
156 PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
157 PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE | \
158 PERF_SAMPLE_WEIGHT_TYPE)
159
160 #define PEBS_GP_REGS \
161 ((1ULL << PERF_REG_X86_AX) | \
162 (1ULL << PERF_REG_X86_BX) | \
163 (1ULL << PERF_REG_X86_CX) | \
164 (1ULL << PERF_REG_X86_DX) | \
165 (1ULL << PERF_REG_X86_DI) | \
166 (1ULL << PERF_REG_X86_SI) | \
167 (1ULL << PERF_REG_X86_SP) | \
168 (1ULL << PERF_REG_X86_BP) | \
169 (1ULL << PERF_REG_X86_IP) | \
170 (1ULL << PERF_REG_X86_FLAGS) | \
171 (1ULL << PERF_REG_X86_R8) | \
172 (1ULL << PERF_REG_X86_R9) | \
173 (1ULL << PERF_REG_X86_R10) | \
174 (1ULL << PERF_REG_X86_R11) | \
175 (1ULL << PERF_REG_X86_R12) | \
176 (1ULL << PERF_REG_X86_R13) | \
177 (1ULL << PERF_REG_X86_R14) | \
178 (1ULL << PERF_REG_X86_R15))
179
180 /*
181 * Per register state.
182 */
183 struct er_account {
184 raw_spinlock_t lock; /* per-core: protect structure */
185 u64 config; /* extra MSR config */
186 u64 reg; /* extra MSR number */
187 atomic_t ref; /* reference count */
188 };
189
190 /*
191 * Per core/cpu state
192 *
193 * Used to coordinate shared registers between HT threads or
194 * among events on a single PMU.
195 */
196 struct intel_shared_regs {
197 struct er_account regs[EXTRA_REG_MAX];
198 int refcnt; /* per-core: #HT threads */
199 unsigned core_id; /* per-core: core id */
200 };
201
202 enum intel_excl_state_type {
203 INTEL_EXCL_UNUSED = 0, /* counter is unused */
204 INTEL_EXCL_SHARED = 1, /* counter can be used by both threads */
205 INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
206 };
207
208 struct intel_excl_states {
209 enum intel_excl_state_type state[X86_PMC_IDX_MAX];
210 bool sched_started; /* true if scheduling has started */
211 };
212
213 struct intel_excl_cntrs {
214 raw_spinlock_t lock;
215
216 struct intel_excl_states states[2];
217
218 union {
219 u16 has_exclusive[2];
220 u32 exclusive_present;
221 };
222
223 int refcnt; /* per-core: #HT threads */
224 unsigned core_id; /* per-core: core id */
225 };
226
227 struct x86_perf_task_context;
228 #define MAX_LBR_ENTRIES 32
229
230 enum {
231 LBR_FORMAT_32 = 0x00,
232 LBR_FORMAT_LIP = 0x01,
233 LBR_FORMAT_EIP = 0x02,
234 LBR_FORMAT_EIP_FLAGS = 0x03,
235 LBR_FORMAT_EIP_FLAGS2 = 0x04,
236 LBR_FORMAT_INFO = 0x05,
237 LBR_FORMAT_TIME = 0x06,
238 LBR_FORMAT_INFO2 = 0x07,
239 LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_INFO2,
240 };
241
242 enum {
243 X86_PERF_KFREE_SHARED = 0,
244 X86_PERF_KFREE_EXCL = 1,
245 X86_PERF_KFREE_MAX
246 };
247
248 struct cpu_hw_events {
249 /*
250 * Generic x86 PMC bits
251 */
252 struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
253 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
254 unsigned long dirty[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
255 int enabled;
256
257 int n_events; /* the # of events in the below arrays */
258 int n_added; /* the # last events in the below arrays;
259 they've never been enabled yet */
260 int n_txn; /* the # last events in the below arrays;
261 added in the current transaction */
262 int n_txn_pair;
263 int n_txn_metric;
264 int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
265 u64 tags[X86_PMC_IDX_MAX];
266
267 struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
268 struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
269
270 int n_excl; /* the number of exclusive events */
271
272 unsigned int txn_flags;
273 int is_fake;
274
275 /*
276 * Intel DebugStore bits
277 */
278 struct debug_store *ds;
279 void *ds_pebs_vaddr;
280 void *ds_bts_vaddr;
281 u64 pebs_enabled;
282 int n_pebs;
283 int n_large_pebs;
284 int n_pebs_via_pt;
285 int pebs_output;
286
287 /* Current super set of events hardware configuration */
288 u64 pebs_data_cfg;
289 u64 active_pebs_data_cfg;
290 int pebs_record_size;
291
292 /* Intel Fixed counter configuration */
293 u64 fixed_ctrl_val;
294 u64 active_fixed_ctrl_val;
295
296 /*
297 * Intel LBR bits
298 */
299 int lbr_users;
300 int lbr_pebs_users;
301 struct perf_branch_stack lbr_stack;
302 struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
303 u64 lbr_counters[MAX_LBR_ENTRIES]; /* branch stack extra */
304 union {
305 struct er_account *lbr_sel;
306 struct er_account *lbr_ctl;
307 };
308 u64 br_sel;
309 void *last_task_ctx;
310 int last_log_id;
311 int lbr_select;
312 void *lbr_xsave;
313
314 /*
315 * Intel host/guest exclude bits
316 */
317 u64 intel_ctrl_guest_mask;
318 u64 intel_ctrl_host_mask;
319 struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
320
321 /*
322 * Intel checkpoint mask
323 */
324 u64 intel_cp_status;
325
326 /*
327 * manage shared (per-core, per-cpu) registers
328 * used on Intel NHM/WSM/SNB
329 */
330 struct intel_shared_regs *shared_regs;
331 /*
332 * manage exclusive counter access between hyperthread
333 */
334 struct event_constraint *constraint_list; /* in enable order */
335 struct intel_excl_cntrs *excl_cntrs;
336 int excl_thread_id; /* 0 or 1 */
337
338 /*
339 * SKL TSX_FORCE_ABORT shadow
340 */
341 u64 tfa_shadow;
342
343 /*
344 * Perf Metrics
345 */
346 /* number of accepted metrics events */
347 int n_metric;
348
349 /*
350 * AMD specific bits
351 */
352 struct amd_nb *amd_nb;
353 int brs_active; /* BRS is enabled */
354
355 /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
356 u64 perf_ctr_virt_mask;
357 int n_pair; /* Large increment events */
358
359 void *kfree_on_online[X86_PERF_KFREE_MAX];
360
361 struct pmu *pmu;
362 };
363
364 #define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
365 { .idxmsk64 = (n) }, \
366 .code = (c), \
367 .size = (e) - (c), \
368 .cmask = (m), \
369 .weight = (w), \
370 .overlap = (o), \
371 .flags = f, \
372 }
373
374 #define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
375 __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
376
377 #define EVENT_CONSTRAINT(c, n, m) \
378 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
379
380 /*
381 * The constraint_match() function only works for 'simple' event codes
382 * and not for extended (AMD64_EVENTSEL_EVENT) events codes.
383 */
384 #define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
385 __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
386
387 #define INTEL_EXCLEVT_CONSTRAINT(c, n) \
388 __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
389 0, PERF_X86_EVENT_EXCL)
390
391 /*
392 * The overlap flag marks event constraints with overlapping counter
393 * masks. This is the case if the counter mask of such an event is not
394 * a subset of any other counter mask of a constraint with an equal or
395 * higher weight, e.g.:
396 *
397 * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
398 * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
399 * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
400 *
401 * The event scheduler may not select the correct counter in the first
402 * cycle because it needs to know which subsequent events will be
403 * scheduled. It may fail to schedule the events then. So we set the
404 * overlap flag for such constraints to give the scheduler a hint which
405 * events to select for counter rescheduling.
406 *
407 * Care must be taken as the rescheduling algorithm is O(n!) which
408 * will increase scheduling cycles for an over-committed system
409 * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros
410 * and its counter masks must be kept at a minimum.
411 */
412 #define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
413 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
414
415 /*
416 * Constraint on the Event code.
417 */
418 #define INTEL_EVENT_CONSTRAINT(c, n) \
419 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
420
421 /*
422 * Constraint on a range of Event codes
423 */
424 #define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n) \
425 EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
426
427 /*
428 * Constraint on the Event code + UMask + fixed-mask
429 *
430 * filter mask to validate fixed counter events.
431 * the following filters disqualify for fixed counters:
432 * - inv
433 * - edge
434 * - cnt-mask
435 * - in_tx
436 * - in_tx_checkpointed
437 * The other filters are supported by fixed counters.
438 * The any-thread option is supported starting with v3.
439 */
440 #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
441 #define FIXED_EVENT_CONSTRAINT(c, n) \
442 EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
443
444 /*
445 * The special metric counters do not actually exist. They are calculated from
446 * the combination of the FxCtr3 + MSR_PERF_METRICS.
447 *
448 * The special metric counters are mapped to a dummy offset for the scheduler.
449 * The sharing between multiple users of the same metric without multiplexing
450 * is not allowed, even though the hardware supports that in principle.
451 */
452
453 #define METRIC_EVENT_CONSTRAINT(c, n) \
454 EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)), \
455 INTEL_ARCH_EVENT_MASK)
456
457 /*
458 * Constraint on the Event code + UMask
459 */
460 #define INTEL_UEVENT_CONSTRAINT(c, n) \
461 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
462
463 /* Constraint on specific umask bit only + event */
464 #define INTEL_UBIT_EVENT_CONSTRAINT(c, n) \
465 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
466
467 /* Like UEVENT_CONSTRAINT, but match flags too */
468 #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n) \
469 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
470
471 #define INTEL_EXCLUEVT_CONSTRAINT(c, n) \
472 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
473 HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
474
475 #define INTEL_PLD_CONSTRAINT(c, n) \
476 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
477 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
478
479 #define INTEL_PSD_CONSTRAINT(c, n) \
480 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
481 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT)
482
483 #define INTEL_PST_CONSTRAINT(c, n) \
484 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
485 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
486
487 #define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \
488 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
489 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID)
490
491 #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n) \
492 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
493 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW)
494
495 #define INTEL_HYBRID_STLAT_CONSTRAINT(c, n) \
496 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
497 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_ST_HSW)
498
499 /* Event constraint, but match on all event flags too. */
500 #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
501 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
502
503 #define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n) \
504 EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
505
506 /* Check only flags, but allow all event/umask */
507 #define INTEL_ALL_EVENT_CONSTRAINT(code, n) \
508 EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
509
510 /* Check flags and event code, and set the HSW store flag */
511 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
512 __EVENT_CONSTRAINT(code, n, \
513 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
514 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
515
516 /* Check flags and event code, and set the HSW load flag */
517 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
518 __EVENT_CONSTRAINT(code, n, \
519 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
520 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
521
522 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
523 __EVENT_CONSTRAINT_RANGE(code, end, n, \
524 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
525 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
526
527 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
528 __EVENT_CONSTRAINT(code, n, \
529 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
530 HWEIGHT(n), 0, \
531 PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
532
533 /* Check flags and event code/umask, and set the HSW store flag */
534 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
535 __EVENT_CONSTRAINT(code, n, \
536 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
537 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
538
539 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
540 __EVENT_CONSTRAINT(code, n, \
541 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
542 HWEIGHT(n), 0, \
543 PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
544
545 /* Check flags and event code/umask, and set the HSW load flag */
546 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
547 __EVENT_CONSTRAINT(code, n, \
548 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
549 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
550
551 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
552 __EVENT_CONSTRAINT(code, n, \
553 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
554 HWEIGHT(n), 0, \
555 PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
556
557 /* Check flags and event code/umask, and set the HSW N/A flag */
558 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
559 __EVENT_CONSTRAINT(code, n, \
560 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
561 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
562
563
564 /*
565 * We define the end marker as having a weight of -1
566 * to enable blacklisting of events using a counter bitmask
567 * of zero and thus a weight of zero.
568 * The end marker has a weight that cannot possibly be
569 * obtained from counting the bits in the bitmask.
570 */
571 #define EVENT_CONSTRAINT_END { .weight = -1 }
572
573 /*
574 * Check for end marker with weight == -1
575 */
576 #define for_each_event_constraint(e, c) \
577 for ((e) = (c); (e)->weight != -1; (e)++)
578
579 /*
580 * Extra registers for specific events.
581 *
582 * Some events need large masks and require external MSRs.
583 * Those extra MSRs end up being shared for all events on
584 * a PMU and sometimes between PMU of sibling HT threads.
585 * In either case, the kernel needs to handle conflicting
586 * accesses to those extra, shared, regs. The data structure
587 * to manage those registers is stored in cpu_hw_event.
588 */
589 struct extra_reg {
590 unsigned int event;
591 unsigned int msr;
592 u64 config_mask;
593 u64 valid_mask;
594 int idx; /* per_xxx->regs[] reg index */
595 bool extra_msr_access;
596 };
597
598 #define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
599 .event = (e), \
600 .msr = (ms), \
601 .config_mask = (m), \
602 .valid_mask = (vm), \
603 .idx = EXTRA_REG_##i, \
604 .extra_msr_access = true, \
605 }
606
607 #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
608 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
609
610 #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
611 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
612 ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
613
614 #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
615 INTEL_UEVENT_EXTRA_REG(c, \
616 MSR_PEBS_LD_LAT_THRESHOLD, \
617 0xffff, \
618 LDLAT)
619
620 #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
621
622 union perf_capabilities {
623 struct {
624 u64 lbr_format:6;
625 u64 pebs_trap:1;
626 u64 pebs_arch_reg:1;
627 u64 pebs_format:4;
628 u64 smm_freeze:1;
629 /*
630 * PMU supports separate counter range for writing
631 * values > 32bit.
632 */
633 u64 full_width_write:1;
634 u64 pebs_baseline:1;
635 u64 perf_metrics:1;
636 u64 pebs_output_pt_available:1;
637 u64 pebs_timing_info:1;
638 u64 anythread_deprecated:1;
639 u64 rdpmc_metrics_clear:1;
640 };
641 u64 capabilities;
642 };
643
644 struct x86_pmu_quirk {
645 struct x86_pmu_quirk *next;
646 void (*func)(void);
647 };
648
649 union x86_pmu_config {
650 struct {
651 u64 event:8,
652 umask:8,
653 usr:1,
654 os:1,
655 edge:1,
656 pc:1,
657 interrupt:1,
658 __reserved1:1,
659 en:1,
660 inv:1,
661 cmask:8,
662 event2:4,
663 __reserved2:4,
664 go:1,
665 ho:1;
666 } bits;
667 u64 value;
668 };
669
670 #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
671
672 enum {
673 x86_lbr_exclusive_lbr,
674 x86_lbr_exclusive_bts,
675 x86_lbr_exclusive_pt,
676 x86_lbr_exclusive_max,
677 };
678
679 #define PERF_PEBS_DATA_SOURCE_MAX 0x100
680 #define PERF_PEBS_DATA_SOURCE_MASK (PERF_PEBS_DATA_SOURCE_MAX - 1)
681 #define PERF_PEBS_DATA_SOURCE_GRT_MAX 0x10
682 #define PERF_PEBS_DATA_SOURCE_GRT_MASK (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
683
684 #define X86_HYBRID_PMU_ATOM_IDX 0
685 #define X86_HYBRID_PMU_CORE_IDX 1
686 #define X86_HYBRID_PMU_TINY_IDX 2
687
688 enum hybrid_pmu_type {
689 not_hybrid,
690 hybrid_small = BIT(X86_HYBRID_PMU_ATOM_IDX),
691 hybrid_big = BIT(X86_HYBRID_PMU_CORE_IDX),
692 hybrid_tiny = BIT(X86_HYBRID_PMU_TINY_IDX),
693
694 /* The belows are only used for matching */
695 hybrid_big_small = hybrid_big | hybrid_small,
696 hybrid_small_tiny = hybrid_small | hybrid_tiny,
697 hybrid_big_small_tiny = hybrid_big | hybrid_small_tiny,
698 };
699
700 struct x86_hybrid_pmu {
701 struct pmu pmu;
702 const char *name;
703 enum hybrid_pmu_type pmu_type;
704 cpumask_t supported_cpus;
705 union perf_capabilities intel_cap;
706 u64 intel_ctrl;
707 u64 pebs_events_mask;
708 u64 config_mask;
709 union {
710 u64 cntr_mask64;
711 unsigned long cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
712 };
713 union {
714 u64 fixed_cntr_mask64;
715 unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
716 };
717 struct event_constraint unconstrained;
718
719 u64 hw_cache_event_ids
720 [PERF_COUNT_HW_CACHE_MAX]
721 [PERF_COUNT_HW_CACHE_OP_MAX]
722 [PERF_COUNT_HW_CACHE_RESULT_MAX];
723 u64 hw_cache_extra_regs
724 [PERF_COUNT_HW_CACHE_MAX]
725 [PERF_COUNT_HW_CACHE_OP_MAX]
726 [PERF_COUNT_HW_CACHE_RESULT_MAX];
727 struct event_constraint *event_constraints;
728 struct event_constraint *pebs_constraints;
729 struct extra_reg *extra_regs;
730
731 unsigned int late_ack :1,
732 mid_ack :1,
733 enabled_ack :1;
734
735 u64 pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
736 };
737
hybrid_pmu(struct pmu * pmu)738 static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu)
739 {
740 return container_of(pmu, struct x86_hybrid_pmu, pmu);
741 }
742
743 extern struct static_key_false perf_is_hybrid;
744 #define is_hybrid() static_branch_unlikely(&perf_is_hybrid)
745
746 #define hybrid(_pmu, _field) \
747 (*({ \
748 typeof(&x86_pmu._field) __Fp = &x86_pmu._field; \
749 \
750 if (is_hybrid() && (_pmu)) \
751 __Fp = &hybrid_pmu(_pmu)->_field; \
752 \
753 __Fp; \
754 }))
755
756 #define hybrid_var(_pmu, _var) \
757 (*({ \
758 typeof(&_var) __Fp = &_var; \
759 \
760 if (is_hybrid() && (_pmu)) \
761 __Fp = &hybrid_pmu(_pmu)->_var; \
762 \
763 __Fp; \
764 }))
765
766 #define hybrid_bit(_pmu, _field) \
767 ({ \
768 bool __Fp = x86_pmu._field; \
769 \
770 if (is_hybrid() && (_pmu)) \
771 __Fp = hybrid_pmu(_pmu)->_field; \
772 \
773 __Fp; \
774 })
775
776 /*
777 * struct x86_pmu - generic x86 pmu
778 */
779 struct x86_pmu {
780 /*
781 * Generic x86 PMC bits
782 */
783 const char *name;
784 int version;
785 int (*handle_irq)(struct pt_regs *);
786 void (*disable_all)(void);
787 void (*enable_all)(int added);
788 void (*enable)(struct perf_event *);
789 void (*disable)(struct perf_event *);
790 void (*assign)(struct perf_event *event, int idx);
791 void (*add)(struct perf_event *);
792 void (*del)(struct perf_event *);
793 void (*read)(struct perf_event *event);
794 int (*set_period)(struct perf_event *event);
795 u64 (*update)(struct perf_event *event);
796 int (*hw_config)(struct perf_event *event);
797 int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
798 void (*late_setup)(void);
799 unsigned eventsel;
800 unsigned perfctr;
801 unsigned fixedctr;
802 int (*addr_offset)(int index, bool eventsel);
803 int (*rdpmc_index)(int index);
804 u64 (*event_map)(int);
805 int max_events;
806 u64 config_mask;
807 union {
808 u64 cntr_mask64;
809 unsigned long cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
810 };
811 union {
812 u64 fixed_cntr_mask64;
813 unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
814 };
815 int cntval_bits;
816 u64 cntval_mask;
817 union {
818 unsigned long events_maskl;
819 unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
820 };
821 int events_mask_len;
822 int apic;
823 u64 max_period;
824 struct event_constraint *
825 (*get_event_constraints)(struct cpu_hw_events *cpuc,
826 int idx,
827 struct perf_event *event);
828
829 void (*put_event_constraints)(struct cpu_hw_events *cpuc,
830 struct perf_event *event);
831
832 void (*start_scheduling)(struct cpu_hw_events *cpuc);
833
834 void (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
835
836 void (*stop_scheduling)(struct cpu_hw_events *cpuc);
837
838 struct event_constraint *event_constraints;
839 struct x86_pmu_quirk *quirks;
840 void (*limit_period)(struct perf_event *event, s64 *l);
841
842 /* PMI handler bits */
843 unsigned int late_ack :1,
844 mid_ack :1,
845 enabled_ack :1;
846 /*
847 * sysfs attrs
848 */
849 int attr_rdpmc_broken;
850 int attr_rdpmc;
851 struct attribute **format_attrs;
852
853 ssize_t (*events_sysfs_show)(char *page, u64 config);
854 const struct attribute_group **attr_update;
855
856 unsigned long attr_freeze_on_smi;
857
858 /*
859 * CPU Hotplug hooks
860 */
861 int (*cpu_prepare)(int cpu);
862 void (*cpu_starting)(int cpu);
863 void (*cpu_dying)(int cpu);
864 void (*cpu_dead)(int cpu);
865
866 void (*check_microcode)(void);
867 void (*sched_task)(struct perf_event_pmu_context *pmu_ctx,
868 struct task_struct *task, bool sched_in);
869
870 /*
871 * Intel Arch Perfmon v2+
872 */
873 u64 intel_ctrl;
874 union perf_capabilities intel_cap;
875
876 /*
877 * Intel DebugStore bits
878 */
879 unsigned int bts :1,
880 bts_active :1,
881 pebs :1,
882 pebs_active :1,
883 pebs_broken :1,
884 pebs_prec_dist :1,
885 pebs_no_tlb :1,
886 pebs_no_isolation :1,
887 pebs_block :1,
888 pebs_ept :1;
889 int pebs_record_size;
890 int pebs_buffer_size;
891 u64 pebs_events_mask;
892 void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
893 struct event_constraint *pebs_constraints;
894 void (*pebs_aliases)(struct perf_event *event);
895 u64 (*pebs_latency_data)(struct perf_event *event, u64 status);
896 unsigned long large_pebs_flags;
897 u64 rtm_abort_event;
898 u64 pebs_capable;
899
900 /*
901 * Intel LBR
902 */
903 unsigned int lbr_tos, lbr_from, lbr_to,
904 lbr_info, lbr_nr; /* LBR base regs and size */
905 union {
906 u64 lbr_sel_mask; /* LBR_SELECT valid bits */
907 u64 lbr_ctl_mask; /* LBR_CTL valid bits */
908 };
909 union {
910 const int *lbr_sel_map; /* lbr_select mappings */
911 int *lbr_ctl_map; /* LBR_CTL mappings */
912 };
913 u64 lbr_callstack_users; /* lbr callstack system wide users */
914 bool lbr_double_abort; /* duplicated lbr aborts */
915 bool lbr_pt_coexist; /* (LBR|BTS) may coexist with PT */
916
917 unsigned int lbr_has_info:1;
918 unsigned int lbr_has_tsx:1;
919 unsigned int lbr_from_flags:1;
920 unsigned int lbr_to_cycles:1;
921
922 /*
923 * Intel Architectural LBR CPUID Enumeration
924 */
925 unsigned int lbr_depth_mask:8;
926 unsigned int lbr_deep_c_reset:1;
927 unsigned int lbr_lip:1;
928 unsigned int lbr_cpl:1;
929 unsigned int lbr_filter:1;
930 unsigned int lbr_call_stack:1;
931 unsigned int lbr_mispred:1;
932 unsigned int lbr_timed_lbr:1;
933 unsigned int lbr_br_type:1;
934 unsigned int lbr_counters:4;
935
936 void (*lbr_reset)(void);
937 void (*lbr_read)(struct cpu_hw_events *cpuc);
938 void (*lbr_save)(void *ctx);
939 void (*lbr_restore)(void *ctx);
940
941 /*
942 * Intel PT/LBR/BTS are exclusive
943 */
944 atomic_t lbr_exclusive[x86_lbr_exclusive_max];
945
946 /*
947 * Intel perf metrics
948 */
949 int num_topdown_events;
950
951 /*
952 * AMD bits
953 */
954 unsigned int amd_nb_constraints : 1;
955 u64 perf_ctr_pair_en;
956
957 /*
958 * Extra registers for events
959 */
960 struct extra_reg *extra_regs;
961 unsigned int flags;
962
963 /*
964 * Intel host/guest support (KVM)
965 */
966 struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data);
967
968 /*
969 * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
970 */
971 int (*check_period) (struct perf_event *event, u64 period);
972
973 int (*aux_output_match) (struct perf_event *event);
974
975 void (*filter)(struct pmu *pmu, int cpu, bool *ret);
976 /*
977 * Hybrid support
978 *
979 * Most PMU capabilities are the same among different hybrid PMUs.
980 * The global x86_pmu saves the architecture capabilities, which
981 * are available for all PMUs. The hybrid_pmu only includes the
982 * unique capabilities.
983 */
984 int num_hybrid_pmus;
985 struct x86_hybrid_pmu *hybrid_pmu;
986 enum intel_cpu_type (*get_hybrid_cpu_type) (void);
987 };
988
989 struct x86_perf_task_context_opt {
990 int lbr_callstack_users;
991 int lbr_stack_state;
992 int log_id;
993 };
994
995 struct x86_perf_task_context {
996 u64 lbr_sel;
997 int tos;
998 int valid_lbrs;
999 struct x86_perf_task_context_opt opt;
1000 struct lbr_entry lbr[MAX_LBR_ENTRIES];
1001 };
1002
1003 struct x86_perf_task_context_arch_lbr {
1004 struct x86_perf_task_context_opt opt;
1005 struct lbr_entry entries[];
1006 };
1007
1008 /*
1009 * Add padding to guarantee the 64-byte alignment of the state buffer.
1010 *
1011 * The structure is dynamically allocated. The size of the LBR state may vary
1012 * based on the number of LBR registers.
1013 *
1014 * Do not put anything after the LBR state.
1015 */
1016 struct x86_perf_task_context_arch_lbr_xsave {
1017 struct x86_perf_task_context_opt opt;
1018
1019 union {
1020 struct xregs_state xsave;
1021 struct {
1022 struct fxregs_state i387;
1023 struct xstate_header header;
1024 struct arch_lbr_state lbr;
1025 } __attribute__ ((packed, aligned (XSAVE_ALIGNMENT)));
1026 };
1027 };
1028
1029 #define x86_add_quirk(func_) \
1030 do { \
1031 static struct x86_pmu_quirk __quirk __initdata = { \
1032 .func = func_, \
1033 }; \
1034 __quirk.next = x86_pmu.quirks; \
1035 x86_pmu.quirks = &__quirk; \
1036 } while (0)
1037
1038 /*
1039 * x86_pmu flags
1040 */
1041 #define PMU_FL_NO_HT_SHARING 0x1 /* no hyper-threading resource sharing */
1042 #define PMU_FL_HAS_RSP_1 0x2 /* has 2 equivalent offcore_rsp regs */
1043 #define PMU_FL_EXCL_CNTRS 0x4 /* has exclusive counter requirements */
1044 #define PMU_FL_EXCL_ENABLED 0x8 /* exclusive counter active */
1045 #define PMU_FL_PEBS_ALL 0x10 /* all events are valid PEBS events */
1046 #define PMU_FL_TFA 0x20 /* deal with TSX force abort */
1047 #define PMU_FL_PAIR 0x40 /* merge counters for large incr. events */
1048 #define PMU_FL_INSTR_LATENCY 0x80 /* Support Instruction Latency in PEBS Memory Info Record */
1049 #define PMU_FL_MEM_LOADS_AUX 0x100 /* Require an auxiliary event for the complete memory info */
1050 #define PMU_FL_RETIRE_LATENCY 0x200 /* Support Retire Latency in PEBS */
1051 #define PMU_FL_BR_CNTR 0x400 /* Support branch counter logging */
1052
1053 #define EVENT_VAR(_id) event_attr_##_id
1054 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
1055
1056 #define EVENT_ATTR(_name, _id) \
1057 static struct perf_pmu_events_attr EVENT_VAR(_id) = { \
1058 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
1059 .id = PERF_COUNT_HW_##_id, \
1060 .event_str = NULL, \
1061 };
1062
1063 #define EVENT_ATTR_STR(_name, v, str) \
1064 static struct perf_pmu_events_attr event_attr_##v = { \
1065 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
1066 .id = 0, \
1067 .event_str = str, \
1068 };
1069
1070 #define EVENT_ATTR_STR_HT(_name, v, noht, ht) \
1071 static struct perf_pmu_events_ht_attr event_attr_##v = { \
1072 .attr = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
1073 .id = 0, \
1074 .event_str_noht = noht, \
1075 .event_str_ht = ht, \
1076 }
1077
1078 #define EVENT_ATTR_STR_HYBRID(_name, v, str, _pmu) \
1079 static struct perf_pmu_events_hybrid_attr event_attr_##v = { \
1080 .attr = __ATTR(_name, 0444, events_hybrid_sysfs_show, NULL),\
1081 .id = 0, \
1082 .event_str = str, \
1083 .pmu_type = _pmu, \
1084 }
1085
1086 #define FORMAT_HYBRID_PTR(_id) (&format_attr_hybrid_##_id.attr.attr)
1087
1088 #define FORMAT_ATTR_HYBRID(_name, _pmu) \
1089 static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\
1090 .attr = __ATTR_RO(_name), \
1091 .pmu_type = _pmu, \
1092 }
1093
1094 struct pmu *x86_get_pmu(unsigned int cpu);
1095 extern struct x86_pmu x86_pmu __read_mostly;
1096
1097 DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period);
1098 DECLARE_STATIC_CALL(x86_pmu_update, *x86_pmu.update);
1099 DECLARE_STATIC_CALL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs);
1100 DECLARE_STATIC_CALL(x86_pmu_late_setup, *x86_pmu.late_setup);
1101
task_context_opt(void * ctx)1102 static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
1103 {
1104 if (static_cpu_has(X86_FEATURE_ARCH_LBR))
1105 return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
1106
1107 return &((struct x86_perf_task_context *)ctx)->opt;
1108 }
1109
x86_pmu_has_lbr_callstack(void)1110 static inline bool x86_pmu_has_lbr_callstack(void)
1111 {
1112 return x86_pmu.lbr_sel_map &&
1113 x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
1114 }
1115
1116 DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
1117 DECLARE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1118
1119 int x86_perf_event_set_period(struct perf_event *event);
1120
1121 /*
1122 * Generalized hw caching related hw_event table, filled
1123 * in on a per model basis. A value of 0 means
1124 * 'not supported', -1 means 'hw_event makes no sense on
1125 * this CPU', any other value means the raw hw_event
1126 * ID.
1127 */
1128
1129 #define C(x) PERF_COUNT_HW_CACHE_##x
1130
1131 extern u64 __read_mostly hw_cache_event_ids
1132 [PERF_COUNT_HW_CACHE_MAX]
1133 [PERF_COUNT_HW_CACHE_OP_MAX]
1134 [PERF_COUNT_HW_CACHE_RESULT_MAX];
1135 extern u64 __read_mostly hw_cache_extra_regs
1136 [PERF_COUNT_HW_CACHE_MAX]
1137 [PERF_COUNT_HW_CACHE_OP_MAX]
1138 [PERF_COUNT_HW_CACHE_RESULT_MAX];
1139
1140 u64 x86_perf_event_update(struct perf_event *event);
1141
intel_pmu_topdown_event_update(struct perf_event * event,u64 * val)1142 static inline u64 intel_pmu_topdown_event_update(struct perf_event *event, u64 *val)
1143 {
1144 return x86_perf_event_update(event);
1145 }
1146 DECLARE_STATIC_CALL(intel_pmu_update_topdown_event, intel_pmu_topdown_event_update);
1147
x86_pmu_config_addr(int index)1148 static inline unsigned int x86_pmu_config_addr(int index)
1149 {
1150 return x86_pmu.eventsel + (x86_pmu.addr_offset ?
1151 x86_pmu.addr_offset(index, true) : index);
1152 }
1153
x86_pmu_event_addr(int index)1154 static inline unsigned int x86_pmu_event_addr(int index)
1155 {
1156 return x86_pmu.perfctr + (x86_pmu.addr_offset ?
1157 x86_pmu.addr_offset(index, false) : index);
1158 }
1159
x86_pmu_fixed_ctr_addr(int index)1160 static inline unsigned int x86_pmu_fixed_ctr_addr(int index)
1161 {
1162 return x86_pmu.fixedctr + (x86_pmu.addr_offset ?
1163 x86_pmu.addr_offset(index, false) : index);
1164 }
1165
x86_pmu_rdpmc_index(int index)1166 static inline int x86_pmu_rdpmc_index(int index)
1167 {
1168 return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
1169 }
1170
1171 bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
1172 unsigned long *fixed_cntr_mask);
1173
1174 int x86_add_exclusive(unsigned int what);
1175
1176 void x86_del_exclusive(unsigned int what);
1177
1178 int x86_reserve_hardware(void);
1179
1180 void x86_release_hardware(void);
1181
1182 int x86_pmu_max_precise(void);
1183
1184 void hw_perf_lbr_event_destroy(struct perf_event *event);
1185
1186 int x86_setup_perfctr(struct perf_event *event);
1187
1188 int x86_pmu_hw_config(struct perf_event *event);
1189
1190 void x86_pmu_disable_all(void);
1191
has_amd_brs(struct hw_perf_event * hwc)1192 static inline bool has_amd_brs(struct hw_perf_event *hwc)
1193 {
1194 return hwc->flags & PERF_X86_EVENT_AMD_BRS;
1195 }
1196
is_counter_pair(struct hw_perf_event * hwc)1197 static inline bool is_counter_pair(struct hw_perf_event *hwc)
1198 {
1199 return hwc->flags & PERF_X86_EVENT_PAIR;
1200 }
1201
__x86_pmu_enable_event(struct hw_perf_event * hwc,u64 enable_mask)1202 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
1203 u64 enable_mask)
1204 {
1205 u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
1206
1207 if (hwc->extra_reg.reg)
1208 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
1209
1210 /*
1211 * Add enabled Merge event on next counter
1212 * if large increment event being enabled on this counter
1213 */
1214 if (is_counter_pair(hwc))
1215 wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
1216
1217 wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
1218 }
1219
1220 void x86_pmu_enable_all(int added);
1221
1222 int perf_assign_events(struct event_constraint **constraints, int n,
1223 int wmin, int wmax, int gpmax, int *assign);
1224 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
1225
1226 void x86_pmu_stop(struct perf_event *event, int flags);
1227
x86_pmu_disable_event(struct perf_event * event)1228 static inline void x86_pmu_disable_event(struct perf_event *event)
1229 {
1230 u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
1231 struct hw_perf_event *hwc = &event->hw;
1232
1233 wrmsrl(hwc->config_base, hwc->config & ~disable_mask);
1234
1235 if (is_counter_pair(hwc))
1236 wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0);
1237 }
1238
1239 void x86_pmu_enable_event(struct perf_event *event);
1240
1241 int x86_pmu_handle_irq(struct pt_regs *regs);
1242
1243 void x86_pmu_show_pmu_cap(struct pmu *pmu);
1244
x86_pmu_num_counters(struct pmu * pmu)1245 static inline int x86_pmu_num_counters(struct pmu *pmu)
1246 {
1247 return hweight64(hybrid(pmu, cntr_mask64));
1248 }
1249
x86_pmu_max_num_counters(struct pmu * pmu)1250 static inline int x86_pmu_max_num_counters(struct pmu *pmu)
1251 {
1252 return fls64(hybrid(pmu, cntr_mask64));
1253 }
1254
x86_pmu_num_counters_fixed(struct pmu * pmu)1255 static inline int x86_pmu_num_counters_fixed(struct pmu *pmu)
1256 {
1257 return hweight64(hybrid(pmu, fixed_cntr_mask64));
1258 }
1259
x86_pmu_max_num_counters_fixed(struct pmu * pmu)1260 static inline int x86_pmu_max_num_counters_fixed(struct pmu *pmu)
1261 {
1262 return fls64(hybrid(pmu, fixed_cntr_mask64));
1263 }
1264
x86_pmu_get_event_config(struct perf_event * event)1265 static inline u64 x86_pmu_get_event_config(struct perf_event *event)
1266 {
1267 return event->attr.config & hybrid(event->pmu, config_mask);
1268 }
1269
1270 extern struct event_constraint emptyconstraint;
1271
1272 extern struct event_constraint unconstrained;
1273
kernel_ip(unsigned long ip)1274 static inline bool kernel_ip(unsigned long ip)
1275 {
1276 #ifdef CONFIG_X86_32
1277 return ip > PAGE_OFFSET;
1278 #else
1279 return (long)ip < 0;
1280 #endif
1281 }
1282
1283 /*
1284 * Not all PMUs provide the right context information to place the reported IP
1285 * into full context. Specifically segment registers are typically not
1286 * supplied.
1287 *
1288 * Assuming the address is a linear address (it is for IBS), we fake the CS and
1289 * vm86 mode using the known zero-based code segment and 'fix up' the registers
1290 * to reflect this.
1291 *
1292 * Intel PEBS/LBR appear to typically provide the effective address, nothing
1293 * much we can do about that but pray and treat it like a linear address.
1294 */
set_linear_ip(struct pt_regs * regs,unsigned long ip)1295 static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
1296 {
1297 regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
1298 if (regs->flags & X86_VM_MASK)
1299 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
1300 regs->ip = ip;
1301 }
1302
1303 /*
1304 * x86control flow change classification
1305 * x86control flow changes include branches, interrupts, traps, faults
1306 */
1307 enum {
1308 X86_BR_NONE = 0, /* unknown */
1309
1310 X86_BR_USER = 1 << 0, /* branch target is user */
1311 X86_BR_KERNEL = 1 << 1, /* branch target is kernel */
1312
1313 X86_BR_CALL = 1 << 2, /* call */
1314 X86_BR_RET = 1 << 3, /* return */
1315 X86_BR_SYSCALL = 1 << 4, /* syscall */
1316 X86_BR_SYSRET = 1 << 5, /* syscall return */
1317 X86_BR_INT = 1 << 6, /* sw interrupt */
1318 X86_BR_IRET = 1 << 7, /* return from interrupt */
1319 X86_BR_JCC = 1 << 8, /* conditional */
1320 X86_BR_JMP = 1 << 9, /* jump */
1321 X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */
1322 X86_BR_IND_CALL = 1 << 11,/* indirect calls */
1323 X86_BR_ABORT = 1 << 12,/* transaction abort */
1324 X86_BR_IN_TX = 1 << 13,/* in transaction */
1325 X86_BR_NO_TX = 1 << 14,/* not in transaction */
1326 X86_BR_ZERO_CALL = 1 << 15,/* zero length call */
1327 X86_BR_CALL_STACK = 1 << 16,/* call stack */
1328 X86_BR_IND_JMP = 1 << 17,/* indirect jump */
1329
1330 X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */
1331
1332 };
1333
1334 #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
1335 #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
1336
1337 #define X86_BR_ANY \
1338 (X86_BR_CALL |\
1339 X86_BR_RET |\
1340 X86_BR_SYSCALL |\
1341 X86_BR_SYSRET |\
1342 X86_BR_INT |\
1343 X86_BR_IRET |\
1344 X86_BR_JCC |\
1345 X86_BR_JMP |\
1346 X86_BR_IRQ |\
1347 X86_BR_ABORT |\
1348 X86_BR_IND_CALL |\
1349 X86_BR_IND_JMP |\
1350 X86_BR_ZERO_CALL)
1351
1352 #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
1353
1354 #define X86_BR_ANY_CALL \
1355 (X86_BR_CALL |\
1356 X86_BR_IND_CALL |\
1357 X86_BR_ZERO_CALL |\
1358 X86_BR_SYSCALL |\
1359 X86_BR_IRQ |\
1360 X86_BR_INT)
1361
1362 int common_branch_type(int type);
1363 int branch_type(unsigned long from, unsigned long to, int abort);
1364 int branch_type_fused(unsigned long from, unsigned long to, int abort,
1365 int *offset);
1366
1367 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
1368 ssize_t intel_event_sysfs_show(char *page, u64 config);
1369
1370 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1371 char *page);
1372 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1373 char *page);
1374 ssize_t events_hybrid_sysfs_show(struct device *dev,
1375 struct device_attribute *attr,
1376 char *page);
1377
fixed_counter_disabled(int i,struct pmu * pmu)1378 static inline bool fixed_counter_disabled(int i, struct pmu *pmu)
1379 {
1380 u64 intel_ctrl = hybrid(pmu, intel_ctrl);
1381
1382 return !(intel_ctrl >> (i + INTEL_PMC_IDX_FIXED));
1383 }
1384
1385 #ifdef CONFIG_CPU_SUP_AMD
1386
1387 int amd_pmu_init(void);
1388
1389 int amd_pmu_lbr_init(void);
1390 void amd_pmu_lbr_reset(void);
1391 void amd_pmu_lbr_read(void);
1392 void amd_pmu_lbr_add(struct perf_event *event);
1393 void amd_pmu_lbr_del(struct perf_event *event);
1394 void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
1395 struct task_struct *task, bool sched_in);
1396 void amd_pmu_lbr_enable_all(void);
1397 void amd_pmu_lbr_disable_all(void);
1398 int amd_pmu_lbr_hw_config(struct perf_event *event);
1399
__amd_pmu_lbr_disable(void)1400 static __always_inline void __amd_pmu_lbr_disable(void)
1401 {
1402 u64 dbg_ctl, dbg_extn_cfg;
1403
1404 rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
1405 wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
1406
1407 if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
1408 rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
1409 wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1410 }
1411 }
1412
1413 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1414
1415 #define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */
1416
1417 int amd_brs_init(void);
1418 void amd_brs_disable(void);
1419 void amd_brs_enable(void);
1420 void amd_brs_enable_all(void);
1421 void amd_brs_disable_all(void);
1422 void amd_brs_drain(void);
1423 void amd_brs_lopwr_init(void);
1424 int amd_brs_hw_config(struct perf_event *event);
1425 void amd_brs_reset(void);
1426
amd_pmu_brs_add(struct perf_event * event)1427 static inline void amd_pmu_brs_add(struct perf_event *event)
1428 {
1429 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1430
1431 perf_sched_cb_inc(event->pmu);
1432 cpuc->lbr_users++;
1433 /*
1434 * No need to reset BRS because it is reset
1435 * on brs_enable() and it is saturating
1436 */
1437 }
1438
amd_pmu_brs_del(struct perf_event * event)1439 static inline void amd_pmu_brs_del(struct perf_event *event)
1440 {
1441 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1442
1443 cpuc->lbr_users--;
1444 WARN_ON_ONCE(cpuc->lbr_users < 0);
1445
1446 perf_sched_cb_dec(event->pmu);
1447 }
1448
1449 void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
1450 struct task_struct *task, bool sched_in);
1451 #else
amd_brs_init(void)1452 static inline int amd_brs_init(void)
1453 {
1454 return 0;
1455 }
amd_brs_disable(void)1456 static inline void amd_brs_disable(void) {}
amd_brs_enable(void)1457 static inline void amd_brs_enable(void) {}
amd_brs_drain(void)1458 static inline void amd_brs_drain(void) {}
amd_brs_lopwr_init(void)1459 static inline void amd_brs_lopwr_init(void) {}
amd_brs_disable_all(void)1460 static inline void amd_brs_disable_all(void) {}
amd_brs_hw_config(struct perf_event * event)1461 static inline int amd_brs_hw_config(struct perf_event *event)
1462 {
1463 return 0;
1464 }
amd_brs_reset(void)1465 static inline void amd_brs_reset(void) {}
1466
amd_pmu_brs_add(struct perf_event * event)1467 static inline void amd_pmu_brs_add(struct perf_event *event)
1468 {
1469 }
1470
amd_pmu_brs_del(struct perf_event * event)1471 static inline void amd_pmu_brs_del(struct perf_event *event)
1472 {
1473 }
1474
amd_pmu_brs_sched_task(struct perf_event_pmu_context * pmu_ctx,struct task_struct * task,bool sched_in)1475 static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
1476 struct task_struct *task, bool sched_in)
1477 {
1478 }
1479
amd_brs_enable_all(void)1480 static inline void amd_brs_enable_all(void)
1481 {
1482 }
1483
1484 #endif
1485
1486 #else /* CONFIG_CPU_SUP_AMD */
1487
amd_pmu_init(void)1488 static inline int amd_pmu_init(void)
1489 {
1490 return 0;
1491 }
1492
amd_brs_init(void)1493 static inline int amd_brs_init(void)
1494 {
1495 return -EOPNOTSUPP;
1496 }
1497
amd_brs_drain(void)1498 static inline void amd_brs_drain(void)
1499 {
1500 }
1501
amd_brs_enable_all(void)1502 static inline void amd_brs_enable_all(void)
1503 {
1504 }
1505
amd_brs_disable_all(void)1506 static inline void amd_brs_disable_all(void)
1507 {
1508 }
1509 #endif /* CONFIG_CPU_SUP_AMD */
1510
is_pebs_pt(struct perf_event * event)1511 static inline int is_pebs_pt(struct perf_event *event)
1512 {
1513 return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT);
1514 }
1515
1516 #ifdef CONFIG_CPU_SUP_INTEL
1517
intel_pmu_has_bts_period(struct perf_event * event,u64 period)1518 static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
1519 {
1520 struct hw_perf_event *hwc = &event->hw;
1521 unsigned int hw_event, bts_event;
1522
1523 if (event->attr.freq)
1524 return false;
1525
1526 hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
1527 bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
1528
1529 return hw_event == bts_event && period == 1;
1530 }
1531
intel_pmu_has_bts(struct perf_event * event)1532 static inline bool intel_pmu_has_bts(struct perf_event *event)
1533 {
1534 struct hw_perf_event *hwc = &event->hw;
1535
1536 return intel_pmu_has_bts_period(event, hwc->sample_period);
1537 }
1538
__intel_pmu_pebs_disable_all(void)1539 static __always_inline void __intel_pmu_pebs_disable_all(void)
1540 {
1541 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1542 }
1543
__intel_pmu_arch_lbr_disable(void)1544 static __always_inline void __intel_pmu_arch_lbr_disable(void)
1545 {
1546 wrmsrl(MSR_ARCH_LBR_CTL, 0);
1547 }
1548
__intel_pmu_lbr_disable(void)1549 static __always_inline void __intel_pmu_lbr_disable(void)
1550 {
1551 u64 debugctl;
1552
1553 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1554 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1555 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1556 }
1557
1558 int intel_pmu_save_and_restart(struct perf_event *event);
1559
1560 struct event_constraint *
1561 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
1562 struct perf_event *event);
1563
1564 extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
1565 extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
1566
1567 int intel_pmu_init(void);
1568
1569 void init_debug_store_on_cpu(int cpu);
1570
1571 void fini_debug_store_on_cpu(int cpu);
1572
1573 void release_ds_buffers(void);
1574
1575 void reserve_ds_buffers(void);
1576
1577 void release_lbr_buffers(void);
1578
1579 void reserve_lbr_buffers(void);
1580
1581 extern struct event_constraint bts_constraint;
1582 extern struct event_constraint vlbr_constraint;
1583
1584 void intel_pmu_enable_bts(u64 config);
1585
1586 void intel_pmu_disable_bts(void);
1587
1588 int intel_pmu_drain_bts_buffer(void);
1589
1590 u64 grt_latency_data(struct perf_event *event, u64 status);
1591
1592 u64 cmt_latency_data(struct perf_event *event, u64 status);
1593
1594 u64 lnl_latency_data(struct perf_event *event, u64 status);
1595
1596 u64 arl_h_latency_data(struct perf_event *event, u64 status);
1597
1598 extern struct event_constraint intel_core2_pebs_event_constraints[];
1599
1600 extern struct event_constraint intel_atom_pebs_event_constraints[];
1601
1602 extern struct event_constraint intel_slm_pebs_event_constraints[];
1603
1604 extern struct event_constraint intel_glm_pebs_event_constraints[];
1605
1606 extern struct event_constraint intel_glp_pebs_event_constraints[];
1607
1608 extern struct event_constraint intel_grt_pebs_event_constraints[];
1609
1610 extern struct event_constraint intel_nehalem_pebs_event_constraints[];
1611
1612 extern struct event_constraint intel_westmere_pebs_event_constraints[];
1613
1614 extern struct event_constraint intel_snb_pebs_event_constraints[];
1615
1616 extern struct event_constraint intel_ivb_pebs_event_constraints[];
1617
1618 extern struct event_constraint intel_hsw_pebs_event_constraints[];
1619
1620 extern struct event_constraint intel_bdw_pebs_event_constraints[];
1621
1622 extern struct event_constraint intel_skl_pebs_event_constraints[];
1623
1624 extern struct event_constraint intel_icl_pebs_event_constraints[];
1625
1626 extern struct event_constraint intel_glc_pebs_event_constraints[];
1627
1628 extern struct event_constraint intel_lnc_pebs_event_constraints[];
1629
1630 struct event_constraint *intel_pebs_constraints(struct perf_event *event);
1631
1632 void intel_pmu_pebs_add(struct perf_event *event);
1633
1634 void intel_pmu_pebs_del(struct perf_event *event);
1635
1636 void intel_pmu_pebs_enable(struct perf_event *event);
1637
1638 void intel_pmu_pebs_disable(struct perf_event *event);
1639
1640 void intel_pmu_pebs_enable_all(void);
1641
1642 void intel_pmu_pebs_disable_all(void);
1643
1644 void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1645
1646 void intel_pmu_drain_pebs_buffer(void);
1647
1648 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
1649
1650 void intel_ds_init(void);
1651
1652 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
1653 struct cpu_hw_events *cpuc,
1654 struct perf_event *event);
1655
1656 void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
1657 struct task_struct *task, bool sched_in);
1658
1659 u64 lbr_from_signext_quirk_wr(u64 val);
1660
1661 void intel_pmu_lbr_reset(void);
1662
1663 void intel_pmu_lbr_reset_32(void);
1664
1665 void intel_pmu_lbr_reset_64(void);
1666
1667 void intel_pmu_lbr_add(struct perf_event *event);
1668
1669 void intel_pmu_lbr_del(struct perf_event *event);
1670
1671 void intel_pmu_lbr_enable_all(bool pmi);
1672
1673 void intel_pmu_lbr_disable_all(void);
1674
1675 void intel_pmu_lbr_read(void);
1676
1677 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
1678
1679 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
1680
1681 void intel_pmu_lbr_save(void *ctx);
1682
1683 void intel_pmu_lbr_restore(void *ctx);
1684
1685 void intel_pmu_lbr_init_core(void);
1686
1687 void intel_pmu_lbr_init_nhm(void);
1688
1689 void intel_pmu_lbr_init_atom(void);
1690
1691 void intel_pmu_lbr_init_slm(void);
1692
1693 void intel_pmu_lbr_init_snb(void);
1694
1695 void intel_pmu_lbr_init_hsw(void);
1696
1697 void intel_pmu_lbr_init_skl(void);
1698
1699 void intel_pmu_lbr_init_knl(void);
1700
1701 void intel_pmu_lbr_init(void);
1702
1703 void intel_pmu_arch_lbr_init(void);
1704
1705 void intel_pmu_pebs_data_source_nhm(void);
1706
1707 void intel_pmu_pebs_data_source_skl(bool pmem);
1708
1709 void intel_pmu_pebs_data_source_adl(void);
1710
1711 void intel_pmu_pebs_data_source_grt(void);
1712
1713 void intel_pmu_pebs_data_source_mtl(void);
1714
1715 void intel_pmu_pebs_data_source_arl_h(void);
1716
1717 void intel_pmu_pebs_data_source_cmt(void);
1718
1719 void intel_pmu_pebs_data_source_lnl(void);
1720
1721 int intel_pmu_setup_lbr_filter(struct perf_event *event);
1722
1723 void intel_pt_interrupt(void);
1724
1725 int intel_bts_interrupt(void);
1726
1727 void intel_bts_enable_local(void);
1728
1729 void intel_bts_disable_local(void);
1730
1731 int p4_pmu_init(void);
1732
1733 int p6_pmu_init(void);
1734
1735 int knc_pmu_init(void);
1736
is_ht_workaround_enabled(void)1737 static inline int is_ht_workaround_enabled(void)
1738 {
1739 return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
1740 }
1741
intel_pmu_pebs_mask(u64 cntr_mask)1742 static inline u64 intel_pmu_pebs_mask(u64 cntr_mask)
1743 {
1744 return MAX_PEBS_EVENTS_MASK & cntr_mask;
1745 }
1746
intel_pmu_max_num_pebs(struct pmu * pmu)1747 static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
1748 {
1749 static_assert(MAX_PEBS_EVENTS == 32);
1750 return fls((u32)hybrid(pmu, pebs_events_mask));
1751 }
1752
1753 #else /* CONFIG_CPU_SUP_INTEL */
1754
reserve_ds_buffers(void)1755 static inline void reserve_ds_buffers(void)
1756 {
1757 }
1758
release_ds_buffers(void)1759 static inline void release_ds_buffers(void)
1760 {
1761 }
1762
release_lbr_buffers(void)1763 static inline void release_lbr_buffers(void)
1764 {
1765 }
1766
reserve_lbr_buffers(void)1767 static inline void reserve_lbr_buffers(void)
1768 {
1769 }
1770
intel_pmu_init(void)1771 static inline int intel_pmu_init(void)
1772 {
1773 return 0;
1774 }
1775
intel_cpuc_prepare(struct cpu_hw_events * cpuc,int cpu)1776 static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
1777 {
1778 return 0;
1779 }
1780
intel_cpuc_finish(struct cpu_hw_events * cpuc)1781 static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
1782 {
1783 }
1784
is_ht_workaround_enabled(void)1785 static inline int is_ht_workaround_enabled(void)
1786 {
1787 return 0;
1788 }
1789 #endif /* CONFIG_CPU_SUP_INTEL */
1790
1791 #if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN))
1792 int zhaoxin_pmu_init(void);
1793 #else
zhaoxin_pmu_init(void)1794 static inline int zhaoxin_pmu_init(void)
1795 {
1796 return 0;
1797 }
1798 #endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/
1799