1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Branch Record Buffer Extension Driver.
4 *
5 * Copyright (C) 2022-2025 ARM Limited
6 *
7 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
8 */
9 #include <linux/types.h>
10 #include <linux/bitmap.h>
11 #include <linux/perf/arm_pmu.h>
12 #include "arm_brbe.h"
13
14 #define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \
15 BRBFCR_EL1_INDIRECT | \
16 BRBFCR_EL1_RTN | \
17 BRBFCR_EL1_INDCALL | \
18 BRBFCR_EL1_DIRCALL | \
19 BRBFCR_EL1_CONDDIR)
20
21 /*
22 * BRBTS_EL1 is currently not used for branch stack implementation
23 * purpose but BRBCR_ELx.TS needs to have a valid value from all
24 * available options. BRBCR_ELx_TS_VIRTUAL is selected for this.
25 */
26 #define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL)
27
28 /*
29 * BRBE Buffer Organization
30 *
31 * BRBE buffer is arranged as multiple banks of 32 branch record
32 * entries each. An individual branch record in a given bank could
33 * be accessed, after selecting the bank in BRBFCR_EL1.BANK and
34 * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with
35 * indices [0..31].
36 *
37 * Bank 0
38 *
39 * --------------------------------- ------
40 * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 |
41 * --------------------------------- ------
42 * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 |
43 * --------------------------------- ------
44 * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
45 * --------------------------------- ------
46 * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 |
47 * --------------------------------- ------
48 *
49 * Bank 1
50 *
51 * --------------------------------- ------
52 * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 |
53 * --------------------------------- ------
54 * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 |
55 * --------------------------------- ------
56 * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
57 * --------------------------------- ------
58 * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 |
59 * --------------------------------- ------
60 */
61 #define BRBE_BANK_MAX_ENTRIES 32
62
63 struct brbe_regset {
64 u64 brbsrc;
65 u64 brbtgt;
66 u64 brbinf;
67 };
68
69 #define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX)
70
71 struct brbe_hw_attr {
72 int brbe_version;
73 int brbe_cc;
74 int brbe_nr;
75 int brbe_format;
76 };
77
78 #define BRBE_REGN_CASE(n, case_macro) \
79 case n: case_macro(n); break
80
81 #define BRBE_REGN_SWITCH(x, case_macro) \
82 do { \
83 switch (x) { \
84 BRBE_REGN_CASE(0, case_macro); \
85 BRBE_REGN_CASE(1, case_macro); \
86 BRBE_REGN_CASE(2, case_macro); \
87 BRBE_REGN_CASE(3, case_macro); \
88 BRBE_REGN_CASE(4, case_macro); \
89 BRBE_REGN_CASE(5, case_macro); \
90 BRBE_REGN_CASE(6, case_macro); \
91 BRBE_REGN_CASE(7, case_macro); \
92 BRBE_REGN_CASE(8, case_macro); \
93 BRBE_REGN_CASE(9, case_macro); \
94 BRBE_REGN_CASE(10, case_macro); \
95 BRBE_REGN_CASE(11, case_macro); \
96 BRBE_REGN_CASE(12, case_macro); \
97 BRBE_REGN_CASE(13, case_macro); \
98 BRBE_REGN_CASE(14, case_macro); \
99 BRBE_REGN_CASE(15, case_macro); \
100 BRBE_REGN_CASE(16, case_macro); \
101 BRBE_REGN_CASE(17, case_macro); \
102 BRBE_REGN_CASE(18, case_macro); \
103 BRBE_REGN_CASE(19, case_macro); \
104 BRBE_REGN_CASE(20, case_macro); \
105 BRBE_REGN_CASE(21, case_macro); \
106 BRBE_REGN_CASE(22, case_macro); \
107 BRBE_REGN_CASE(23, case_macro); \
108 BRBE_REGN_CASE(24, case_macro); \
109 BRBE_REGN_CASE(25, case_macro); \
110 BRBE_REGN_CASE(26, case_macro); \
111 BRBE_REGN_CASE(27, case_macro); \
112 BRBE_REGN_CASE(28, case_macro); \
113 BRBE_REGN_CASE(29, case_macro); \
114 BRBE_REGN_CASE(30, case_macro); \
115 BRBE_REGN_CASE(31, case_macro); \
116 default: WARN(1, "Invalid BRB* index %d\n", x); \
117 } \
118 } while (0)
119
120 #define RETURN_READ_BRBSRCN(n) \
121 return read_sysreg_s(SYS_BRBSRC_EL1(n))
get_brbsrc_reg(int idx)122 static inline u64 get_brbsrc_reg(int idx)
123 {
124 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN);
125 return 0;
126 }
127
128 #define RETURN_READ_BRBTGTN(n) \
129 return read_sysreg_s(SYS_BRBTGT_EL1(n))
get_brbtgt_reg(int idx)130 static u64 get_brbtgt_reg(int idx)
131 {
132 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN);
133 return 0;
134 }
135
136 #define RETURN_READ_BRBINFN(n) \
137 return read_sysreg_s(SYS_BRBINF_EL1(n))
get_brbinf_reg(int idx)138 static u64 get_brbinf_reg(int idx)
139 {
140 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN);
141 return 0;
142 }
143
brbe_record_valid(u64 brbinf)144 static u64 brbe_record_valid(u64 brbinf)
145 {
146 return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf);
147 }
148
brbe_invalid(u64 brbinf)149 static bool brbe_invalid(u64 brbinf)
150 {
151 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE;
152 }
153
brbe_record_is_complete(u64 brbinf)154 static bool brbe_record_is_complete(u64 brbinf)
155 {
156 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL;
157 }
158
brbe_record_is_source_only(u64 brbinf)159 static bool brbe_record_is_source_only(u64 brbinf)
160 {
161 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE;
162 }
163
brbe_record_is_target_only(u64 brbinf)164 static bool brbe_record_is_target_only(u64 brbinf)
165 {
166 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET;
167 }
168
brbinf_get_in_tx(u64 brbinf)169 static int brbinf_get_in_tx(u64 brbinf)
170 {
171 return FIELD_GET(BRBINFx_EL1_T_MASK, brbinf);
172 }
173
brbinf_get_mispredict(u64 brbinf)174 static int brbinf_get_mispredict(u64 brbinf)
175 {
176 return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf);
177 }
178
brbinf_get_lastfailed(u64 brbinf)179 static int brbinf_get_lastfailed(u64 brbinf)
180 {
181 return FIELD_GET(BRBINFx_EL1_LASTFAILED_MASK, brbinf);
182 }
183
brbinf_get_cycles(u64 brbinf)184 static u16 brbinf_get_cycles(u64 brbinf)
185 {
186 u32 exp, mant, cycles;
187 /*
188 * Captured cycle count is unknown and hence
189 * should not be passed on to userspace.
190 */
191 if (brbinf & BRBINFx_EL1_CCU)
192 return 0;
193
194 exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf);
195 mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf);
196
197 if (!exp)
198 return mant;
199
200 cycles = (mant | 0x100) << (exp - 1);
201
202 return min(cycles, U16_MAX);
203 }
204
brbinf_get_type(u64 brbinf)205 static int brbinf_get_type(u64 brbinf)
206 {
207 return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf);
208 }
209
brbinf_get_el(u64 brbinf)210 static int brbinf_get_el(u64 brbinf)
211 {
212 return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf);
213 }
214
brbe_invalidate(void)215 void brbe_invalidate(void)
216 {
217 /* Ensure all branches before this point are recorded */
218 isb();
219 asm volatile(BRB_IALL_INSN);
220 /* Ensure all branch records are invalidated after this point */
221 isb();
222 }
223
valid_brbe_nr(int brbe_nr)224 static bool valid_brbe_nr(int brbe_nr)
225 {
226 return brbe_nr == BRBIDR0_EL1_NUMREC_8 ||
227 brbe_nr == BRBIDR0_EL1_NUMREC_16 ||
228 brbe_nr == BRBIDR0_EL1_NUMREC_32 ||
229 brbe_nr == BRBIDR0_EL1_NUMREC_64;
230 }
231
valid_brbe_cc(int brbe_cc)232 static bool valid_brbe_cc(int brbe_cc)
233 {
234 return brbe_cc == BRBIDR0_EL1_CC_20_BIT;
235 }
236
valid_brbe_format(int brbe_format)237 static bool valid_brbe_format(int brbe_format)
238 {
239 return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0;
240 }
241
valid_brbidr(u64 brbidr)242 static bool valid_brbidr(u64 brbidr)
243 {
244 int brbe_format, brbe_cc, brbe_nr;
245
246 brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr);
247 brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr);
248 brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr);
249
250 return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr);
251 }
252
valid_brbe_version(int brbe_version)253 static bool valid_brbe_version(int brbe_version)
254 {
255 return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP ||
256 brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1;
257 }
258
select_brbe_bank(int bank)259 static void select_brbe_bank(int bank)
260 {
261 u64 brbfcr;
262
263 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
264 brbfcr &= ~BRBFCR_EL1_BANK_MASK;
265 brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank);
266 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
267 /*
268 * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync
269 * between setting BANK and accessing branch records.
270 */
271 isb();
272 }
273
__read_brbe_regset(struct brbe_regset * entry,int idx)274 static bool __read_brbe_regset(struct brbe_regset *entry, int idx)
275 {
276 entry->brbinf = get_brbinf_reg(idx);
277
278 if (brbe_invalid(entry->brbinf))
279 return false;
280
281 entry->brbsrc = get_brbsrc_reg(idx);
282 entry->brbtgt = get_brbtgt_reg(idx);
283 return true;
284 }
285
286 /*
287 * Generic perf branch filters supported on BRBE
288 *
289 * New branch filters need to be evaluated whether they could be supported on
290 * BRBE. This ensures that such branch filters would not just be accepted, to
291 * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively
292 * supported only on platforms where kernel is in hyp mode.
293 */
294 #define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \
295 PERF_SAMPLE_BRANCH_IN_TX | \
296 PERF_SAMPLE_BRANCH_NO_TX | \
297 PERF_SAMPLE_BRANCH_CALL_STACK | \
298 PERF_SAMPLE_BRANCH_COUNTERS)
299
300 #define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \
301 PERF_SAMPLE_BRANCH_ANY_CALL | \
302 PERF_SAMPLE_BRANCH_ANY_RETURN | \
303 PERF_SAMPLE_BRANCH_IND_CALL | \
304 PERF_SAMPLE_BRANCH_COND | \
305 PERF_SAMPLE_BRANCH_IND_JUMP | \
306 PERF_SAMPLE_BRANCH_CALL)
307
308
309 #define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \
310 PERF_SAMPLE_BRANCH_KERNEL | \
311 PERF_SAMPLE_BRANCH_HV | \
312 BRBE_ALLOWED_BRANCH_TYPES | \
313 PERF_SAMPLE_BRANCH_NO_FLAGS | \
314 PERF_SAMPLE_BRANCH_NO_CYCLES | \
315 PERF_SAMPLE_BRANCH_TYPE_SAVE | \
316 PERF_SAMPLE_BRANCH_HW_INDEX | \
317 PERF_SAMPLE_BRANCH_PRIV_SAVE)
318
319 #define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \
320 BRBE_EXCLUDE_BRANCH_FILTERS)
321
322 /*
323 * BRBE supports the following functional branch type filters while
324 * generating branch records. These branch filters can be enabled,
325 * either individually or as a group i.e ORing multiple filters
326 * with each other.
327 *
328 * BRBFCR_EL1_CONDDIR - Conditional direct branch
329 * BRBFCR_EL1_DIRCALL - Direct call
330 * BRBFCR_EL1_INDCALL - Indirect call
331 * BRBFCR_EL1_INDIRECT - Indirect branch
332 * BRBFCR_EL1_DIRECT - Direct branch
333 * BRBFCR_EL1_RTN - Subroutine return
334 */
branch_type_to_brbfcr(int branch_type)335 static u64 branch_type_to_brbfcr(int branch_type)
336 {
337 u64 brbfcr = 0;
338
339 if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
340 brbfcr |= BRBFCR_EL1_BRANCH_FILTERS;
341 return brbfcr;
342 }
343
344 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
345 brbfcr |= BRBFCR_EL1_INDCALL;
346 brbfcr |= BRBFCR_EL1_DIRCALL;
347 }
348
349 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
350 brbfcr |= BRBFCR_EL1_RTN;
351
352 if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL)
353 brbfcr |= BRBFCR_EL1_INDCALL;
354
355 if (branch_type & PERF_SAMPLE_BRANCH_COND)
356 brbfcr |= BRBFCR_EL1_CONDDIR;
357
358 if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP)
359 brbfcr |= BRBFCR_EL1_INDIRECT;
360
361 if (branch_type & PERF_SAMPLE_BRANCH_CALL)
362 brbfcr |= BRBFCR_EL1_DIRCALL;
363
364 return brbfcr;
365 }
366
367 /*
368 * BRBE supports the following privilege mode filters while generating
369 * branch records.
370 *
371 * BRBCR_ELx_E0BRE - EL0 branch records
372 * BRBCR_ELx_ExBRE - EL1/EL2 branch records
373 *
374 * BRBE also supports the following additional functional branch type
375 * filters while generating branch records.
376 *
377 * BRBCR_ELx_EXCEPTION - Exception
378 * BRBCR_ELx_ERTN - Exception return
379 */
branch_type_to_brbcr(int branch_type)380 static u64 branch_type_to_brbcr(int branch_type)
381 {
382 u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS;
383
384 if (branch_type & PERF_SAMPLE_BRANCH_USER)
385 brbcr |= BRBCR_ELx_E0BRE;
386
387 /*
388 * When running in the hyp mode, writing into BRBCR_EL1
389 * actually writes into BRBCR_EL2 instead. Field E2BRE
390 * is also at the same position as E1BRE.
391 */
392 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL)
393 brbcr |= BRBCR_ELx_ExBRE;
394
395 if (branch_type & PERF_SAMPLE_BRANCH_HV) {
396 if (is_kernel_in_hyp_mode())
397 brbcr |= BRBCR_ELx_ExBRE;
398 }
399
400 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES))
401 brbcr |= BRBCR_ELx_CC;
402
403 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS))
404 brbcr |= BRBCR_ELx_MPRED;
405
406 /*
407 * The exception and exception return branches could be
408 * captured, irrespective of the perf event's privilege.
409 * If the perf event does not have enough privilege for
410 * a given exception level, then addresses which falls
411 * under that exception level will be reported as zero
412 * for the captured branch record, creating source only
413 * or target only records.
414 */
415 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) {
416 if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
417 brbcr |= BRBCR_ELx_EXCEPTION;
418 brbcr |= BRBCR_ELx_ERTN;
419 }
420
421 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL)
422 brbcr |= BRBCR_ELx_EXCEPTION;
423
424 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
425 brbcr |= BRBCR_ELx_ERTN;
426 }
427 return brbcr;
428 }
429
brbe_branch_attr_valid(struct perf_event * event)430 bool brbe_branch_attr_valid(struct perf_event *event)
431 {
432 u64 branch_type = event->attr.branch_sample_type;
433
434 /*
435 * Ensure both perf branch filter allowed and exclude
436 * masks are always in sync with the generic perf ABI.
437 */
438 BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1));
439
440 if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) {
441 pr_debug("requested branch filter not supported 0x%llx\n", branch_type);
442 return false;
443 }
444
445 /* Ensure at least 1 branch type is enabled */
446 if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) {
447 pr_debug("no branch type enabled 0x%llx\n", branch_type);
448 return false;
449 }
450
451 /*
452 * No branches are recorded in guests nor nVHE hypervisors, so
453 * excluding the host or both kernel and user is invalid.
454 *
455 * Ideally we'd just require exclude_guest and exclude_hv, but setting
456 * event filters with perf for kernel or user don't set exclude_guest.
457 * So effectively, exclude_guest and exclude_hv are ignored.
458 */
459 if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) {
460 pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type);
461 return false;
462 }
463
464 event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type);
465 event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type);
466
467 return true;
468 }
469
brbe_num_branch_records(const struct arm_pmu * armpmu)470 unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu)
471 {
472 return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr);
473 }
474
brbe_probe(struct arm_pmu * armpmu)475 void brbe_probe(struct arm_pmu *armpmu)
476 {
477 u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
478 u32 brbe;
479
480 brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT);
481 if (!valid_brbe_version(brbe))
482 return;
483
484 brbidr = read_sysreg_s(SYS_BRBIDR0_EL1);
485 if (!valid_brbidr(brbidr))
486 return;
487
488 armpmu->reg_brbidr = brbidr;
489 }
490
491 /*
492 * BRBE is assumed to be disabled/paused on entry
493 */
brbe_enable(const struct arm_pmu * arm_pmu)494 void brbe_enable(const struct arm_pmu *arm_pmu)
495 {
496 struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events);
497 u64 brbfcr = 0, brbcr = 0;
498
499 /*
500 * Discard existing records to avoid a discontinuity, e.g. records
501 * missed during handling an overflow.
502 */
503 brbe_invalidate();
504
505 /*
506 * Merge the permitted branch filters of all events.
507 */
508 for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) {
509 struct perf_event *event = cpuc->events[i];
510
511 if (event && has_branch_stack(event)) {
512 brbfcr |= event->hw.branch_reg.config;
513 brbcr |= event->hw.extra_reg.config;
514 }
515 }
516
517 /*
518 * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP
519 * controls freezing the branch records on counter overflow rather than
520 * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to).
521 * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1
522 * and EL0 recording disabled for guests.
523 *
524 * As BRBCR_EL1 CC and MPRED bits also need to match, use the same
525 * value for both registers just masking the exception levels.
526 */
527 if (is_kernel_in_hyp_mode())
528 write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12);
529 write_sysreg_s(brbcr, SYS_BRBCR_EL1);
530 /* Ensure BRBCR_ELx settings take effect before unpausing */
531 isb();
532
533 /* Finally write SYS_BRBFCR_EL to unpause BRBE */
534 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
535 /* Synchronization in PMCR write ensures ordering WRT PMU enabling */
536 }
537
brbe_disable(void)538 void brbe_disable(void)
539 {
540 /*
541 * No need for synchronization here as synchronization in PMCR write
542 * ensures ordering and in the interrupt handler this is a NOP as
543 * we're already paused.
544 */
545 write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
546 write_sysreg_s(0, SYS_BRBCR_EL1);
547 }
548
549 static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = {
550 [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 },
551 [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 },
552 [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 },
553 [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 },
554 [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 },
555 [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 },
556 [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 },
557 [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 },
558 [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 },
559 [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 },
560 [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 },
561 [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN },
562 [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST },
563 [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA },
564 };
565
brbe_set_perf_entry_type(struct perf_branch_entry * entry,u64 brbinf)566 static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf)
567 {
568 int brbe_type = brbinf_get_type(brbinf);
569
570 if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) {
571 const int *br_type = brbe_type_to_perf_type_map[brbe_type];
572
573 entry->type = br_type[0];
574 entry->new_type = br_type[1];
575 }
576 }
577
brbinf_get_perf_priv(u64 brbinf)578 static int brbinf_get_perf_priv(u64 brbinf)
579 {
580 int brbe_el = brbinf_get_el(brbinf);
581
582 switch (brbe_el) {
583 case BRBINFx_EL1_EL_EL0:
584 return PERF_BR_PRIV_USER;
585 case BRBINFx_EL1_EL_EL1:
586 return PERF_BR_PRIV_KERNEL;
587 case BRBINFx_EL1_EL_EL2:
588 if (is_kernel_in_hyp_mode())
589 return PERF_BR_PRIV_KERNEL;
590 return PERF_BR_PRIV_HV;
591 default:
592 pr_warn_once("%d - unknown branch privilege captured\n", brbe_el);
593 return PERF_BR_PRIV_UNKNOWN;
594 }
595 }
596
perf_entry_from_brbe_regset(int index,struct perf_branch_entry * entry,const struct perf_event * event)597 static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry,
598 const struct perf_event *event)
599 {
600 struct brbe_regset bregs;
601 u64 brbinf;
602
603 if (!__read_brbe_regset(&bregs, index))
604 return false;
605
606 brbinf = bregs.brbinf;
607 perf_clear_branch_entry_bitfields(entry);
608 if (brbe_record_is_complete(brbinf)) {
609 entry->from = bregs.brbsrc;
610 entry->to = bregs.brbtgt;
611 } else if (brbe_record_is_source_only(brbinf)) {
612 entry->from = bregs.brbsrc;
613 entry->to = 0;
614 } else if (brbe_record_is_target_only(brbinf)) {
615 entry->from = 0;
616 entry->to = bregs.brbtgt;
617 }
618
619 brbe_set_perf_entry_type(entry, brbinf);
620
621 if (!branch_sample_no_cycles(event))
622 entry->cycles = brbinf_get_cycles(brbinf);
623
624 if (!branch_sample_no_flags(event)) {
625 /* Mispredict info is available for source only and complete branch records. */
626 if (!brbe_record_is_target_only(brbinf)) {
627 entry->mispred = brbinf_get_mispredict(brbinf);
628 entry->predicted = !entry->mispred;
629 }
630
631 /*
632 * Currently TME feature is neither implemented in any hardware
633 * nor it is being supported in the kernel. Just warn here once
634 * if TME related information shows up rather unexpectedly.
635 */
636 if (brbinf_get_lastfailed(brbinf) || brbinf_get_in_tx(brbinf))
637 pr_warn_once("Unknown transaction states\n");
638 }
639
640 /*
641 * Branch privilege level is available for target only and complete
642 * branch records.
643 */
644 if (!brbe_record_is_source_only(brbinf))
645 entry->priv = brbinf_get_perf_priv(brbinf);
646
647 return true;
648 }
649
650 #define PERF_BR_ARM64_ALL ( \
651 BIT(PERF_BR_COND) | \
652 BIT(PERF_BR_UNCOND) | \
653 BIT(PERF_BR_IND) | \
654 BIT(PERF_BR_CALL) | \
655 BIT(PERF_BR_IND_CALL) | \
656 BIT(PERF_BR_RET))
657
658 #define PERF_BR_ARM64_ALL_KERNEL ( \
659 BIT(PERF_BR_SYSCALL) | \
660 BIT(PERF_BR_IRQ) | \
661 BIT(PERF_BR_SERROR) | \
662 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \
663 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \
664 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST))
665
prepare_event_branch_type_mask(u64 branch_sample,unsigned long * event_type_mask)666 static void prepare_event_branch_type_mask(u64 branch_sample,
667 unsigned long *event_type_mask)
668 {
669 if (branch_sample & PERF_SAMPLE_BRANCH_ANY) {
670 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
671 bitmap_from_u64(event_type_mask,
672 BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL |
673 PERF_BR_ARM64_ALL_KERNEL);
674 else
675 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL);
676 return;
677 }
678
679 bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX);
680
681 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) {
682 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
683 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL);
684
685 set_bit(PERF_BR_CALL, event_type_mask);
686 set_bit(PERF_BR_IND_CALL, event_type_mask);
687 }
688
689 if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP)
690 set_bit(PERF_BR_IND, event_type_mask);
691
692 if (branch_sample & PERF_SAMPLE_BRANCH_COND)
693 set_bit(PERF_BR_COND, event_type_mask);
694
695 if (branch_sample & PERF_SAMPLE_BRANCH_CALL)
696 set_bit(PERF_BR_CALL, event_type_mask);
697
698 if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL)
699 set_bit(PERF_BR_IND_CALL, event_type_mask);
700
701 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) {
702 set_bit(PERF_BR_RET, event_type_mask);
703
704 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
705 set_bit(PERF_BR_ERET, event_type_mask);
706 }
707 }
708
709 /*
710 * BRBE is configured with an OR of permissions from all events, so there may
711 * be events which have to be dropped or events where just the source or target
712 * address has to be zeroed.
713 */
filter_branch_privilege(struct perf_branch_entry * entry,u64 branch_sample_type)714 static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type)
715 {
716 bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4);
717 bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4);
718 bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) ||
719 (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV)));
720
721 /* We can only have a half record if permissions have not been expanded */
722 if (!entry->from || !entry->to)
723 return true;
724
725 /*
726 * If record is within a single exception level, just need to either
727 * drop or keep the entire record.
728 */
729 if (from_user == to_user)
730 return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) ||
731 ((entry->priv == PERF_BR_PRIV_USER) &&
732 (branch_sample_type & PERF_SAMPLE_BRANCH_USER));
733
734 /*
735 * Record is across exception levels, mask addresses for the exception
736 * level we're not capturing.
737 */
738 if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) {
739 if (from_user)
740 entry->from = 0;
741 if (to_user)
742 entry->to = 0;
743 }
744
745 if (exclude_kernel) {
746 if (!from_user)
747 entry->from = 0;
748 if (!to_user)
749 entry->to = 0;
750 }
751
752 return true;
753 }
754
filter_branch_type(struct perf_branch_entry * entry,const unsigned long * event_type_mask)755 static bool filter_branch_type(struct perf_branch_entry *entry,
756 const unsigned long *event_type_mask)
757 {
758 if (entry->type == PERF_BR_EXTEND_ABI)
759 return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask);
760 else
761 return test_bit(entry->type, event_type_mask);
762 }
763
filter_branch_record(struct perf_branch_entry * entry,u64 branch_sample,const unsigned long * event_type_mask)764 static bool filter_branch_record(struct perf_branch_entry *entry,
765 u64 branch_sample,
766 const unsigned long *event_type_mask)
767 {
768 return filter_branch_type(entry, event_type_mask) &&
769 filter_branch_privilege(entry, branch_sample);
770 }
771
brbe_read_filtered_entries(struct perf_branch_stack * branch_stack,const struct perf_event * event)772 void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack,
773 const struct perf_event *event)
774 {
775 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
776 int nr_hw = brbe_num_branch_records(cpu_pmu);
777 int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES);
778 int nr_filtered = 0;
779 u64 branch_sample_type = event->attr.branch_sample_type;
780 DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX);
781
782 prepare_event_branch_type_mask(branch_sample_type, event_type_mask);
783
784 for (int bank = 0; bank < nr_banks; bank++) {
785 int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES);
786 int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES);
787
788 select_brbe_bank(bank);
789
790 for (int i = 0; i < nr_this_bank; i++) {
791 struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered];
792
793 if (!perf_entry_from_brbe_regset(i, pbe, event))
794 goto done;
795
796 if (!filter_branch_record(pbe, branch_sample_type, event_type_mask))
797 continue;
798
799 nr_filtered++;
800 }
801 }
802
803 done:
804 branch_stack->nr = nr_filtered;
805 }
806