1 /*
2 * Test the ARM Performance Monitors Unit (PMU).
3 *
4 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
5 * Copyright (C) 2016, Red Hat Inc, Wei Huang <wei@redhat.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU Lesser General Public License version 2.1 and
9 * only version 2.1 as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
14 * for more details.
15 */
16 #include "libcflat.h"
17 #include "errata.h"
18 #include "asm/barrier.h"
19 #include "asm/sysreg.h"
20 #include "asm/processor.h"
21 #include <bitops.h>
22 #include <asm/gic.h>
23
24 #define PMU_PMCR_E (1 << 0)
25 #define PMU_PMCR_P (1 << 1)
26 #define PMU_PMCR_C (1 << 2)
27 #define PMU_PMCR_D (1 << 3)
28 #define PMU_PMCR_X (1 << 4)
29 #define PMU_PMCR_DP (1 << 5)
30 #define PMU_PMCR_LC (1 << 6)
31 #define PMU_PMCR_LP (1 << 7)
32 #define PMU_PMCR_N_SHIFT 11
33 #define PMU_PMCR_N_MASK 0x1f
34 #define PMU_PMCR_ID_SHIFT 16
35 #define PMU_PMCR_ID_MASK 0xff
36 #define PMU_PMCR_IMP_SHIFT 24
37 #define PMU_PMCR_IMP_MASK 0xff
38
39 #define PMU_CYCLE_IDX 31
40
41 #define NR_SAMPLES 10
42
43 /* Some PMU events */
44 #define SW_INCR 0x0
45 #define INST_RETIRED 0x8
46 #define CPU_CYCLES 0x11
47 #define MEM_ACCESS 0x13
48 #define INST_PREC 0x1B
49 #define STALL_FRONTEND 0x23
50 #define STALL_BACKEND 0x24
51 #define CHAIN 0x1E
52
53 #define COMMON_EVENTS_LOW 0x0
54 #define COMMON_EVENTS_HIGH 0x3F
55 #define EXT_COMMON_EVENTS_LOW 0x4000
56 #define EXT_COMMON_EVENTS_HIGH 0x403F
57
58 #define ALL_SET_32 0x00000000FFFFFFFFULL
59 #define ALL_SET_64 0xFFFFFFFFFFFFFFFFULL
60
61 #define ALL_SET(__overflow_at_64bits) \
62 (__overflow_at_64bits ? ALL_SET_64 : ALL_SET_32)
63
64 #define ALL_CLEAR 0x0000000000000000ULL
65 #define PRE_OVERFLOW_32 0x00000000FFFFFFF0ULL
66 #define PRE_OVERFLOW_64 0xFFFFFFFFFFFFFFF0ULL
67 #define COUNT 250
68 #define MARGIN 100
69 #define COUNT_INT 1000
70 /*
71 * PRE_OVERFLOW2 is set so that 1st @COUNT iterations do not
72 * produce 32b overflow and 2nd @COUNT iterations do. To accommodate
73 * for some observed variability we take into account a given @MARGIN
74 */
75 #define PRE_OVERFLOW2_32 (ALL_SET_32 - COUNT - MARGIN)
76 #define PRE_OVERFLOW2_64 (ALL_SET_64 - COUNT - MARGIN)
77
78 #define PRE_OVERFLOW2(__overflow_at_64bits) \
79 (__overflow_at_64bits ? PRE_OVERFLOW2_64 : PRE_OVERFLOW2_32)
80
81 #define PRE_OVERFLOW(__overflow_at_64bits) \
82 (__overflow_at_64bits ? PRE_OVERFLOW_64 : PRE_OVERFLOW_32)
83
84 #define PMU_PPI 23
85
86 struct pmu {
87 unsigned int version;
88 unsigned int nb_implemented_counters;
89 uint32_t pmcr_ro;
90 };
91
92 struct pmu_stats {
93 unsigned long bitmap;
94 uint32_t interrupts[32];
95 bool unexpected;
96 };
97
98 static struct pmu pmu;
99
100 #if defined(__arm__)
101 #define ID_DFR0_PERFMON_SHIFT 24
102 #define ID_DFR0_PERFMON_MASK 0xf
103
104 #define ID_DFR0_PMU_NOTIMPL 0b0000
105 #define ID_DFR0_PMU_V1 0b0001
106 #define ID_DFR0_PMU_V2 0b0010
107 #define ID_DFR0_PMU_V3 0b0011
108 #define ID_DFR0_PMU_V3_8_1 0b0100
109 #define ID_DFR0_PMU_V3_8_4 0b0101
110 #define ID_DFR0_PMU_V3_8_5 0b0110
111 #define ID_DFR0_PMU_IMPDEF 0b1111
112
113 #define PMCR __ACCESS_CP15(c9, 0, c12, 0)
114 #define ID_DFR0 __ACCESS_CP15(c0, 0, c1, 2)
115 #define PMSELR __ACCESS_CP15(c9, 0, c12, 5)
116 #define PMXEVTYPER __ACCESS_CP15(c9, 0, c13, 1)
117 #define PMCNTENSET __ACCESS_CP15(c9, 0, c12, 1)
118 #define PMCNTENCLR __ACCESS_CP15(c9, 0, c12, 2)
119 #define PMOVSR __ACCESS_CP15(c9, 0, c12, 3)
120 #define PMCCNTR32 __ACCESS_CP15(c9, 0, c13, 0)
121 #define PMINTENCLR __ACCESS_CP15(c9, 0, c14, 2)
122 #define PMCCNTR64 __ACCESS_CP15_64(0, c9)
123
get_id_dfr0(void)124 static inline uint32_t get_id_dfr0(void) { return read_sysreg(ID_DFR0); }
get_pmcr(void)125 static inline uint32_t get_pmcr(void) { return read_sysreg(PMCR); }
set_pmcr(uint32_t v)126 static inline void set_pmcr(uint32_t v) { write_sysreg(v, PMCR); }
set_pmcntenset(uint32_t v)127 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, PMCNTENSET); }
128
get_pmu_version(void)129 static inline uint8_t get_pmu_version(void)
130 {
131 return (get_id_dfr0() >> ID_DFR0_PERFMON_SHIFT) & ID_DFR0_PERFMON_MASK;
132 }
133
get_pmccntr(void)134 static inline uint64_t get_pmccntr(void)
135 {
136 return read_sysreg(PMCCNTR32);
137 }
138
set_pmccntr(uint64_t value)139 static inline void set_pmccntr(uint64_t value)
140 {
141 write_sysreg(value & 0xffffffff, PMCCNTR32);
142 }
143
144 /* PMCCFILTR is an obsolete name for PMXEVTYPER31 in ARMv7 */
set_pmccfiltr(uint32_t value)145 static inline void set_pmccfiltr(uint32_t value)
146 {
147 write_sysreg(PMU_CYCLE_IDX, PMSELR);
148 write_sysreg(value, PMXEVTYPER);
149 isb();
150 }
151
152 /*
153 * Extra instructions inserted by the compiler would be difficult to compensate
154 * for, so hand assemble everything between, and including, the PMCR accesses
155 * to start and stop counting. isb instructions were inserted to make sure
156 * pmccntr read after this function returns the exact instructions executed in
157 * the controlled block. Total instrs = isb + mcr + 2*loop = 2 + 2*loop.
158 */
precise_instrs_loop(int loop,uint32_t pmcr)159 static inline void precise_instrs_loop(int loop, uint32_t pmcr)
160 {
161 asm volatile(
162 " mcr p15, 0, %[pmcr], c9, c12, 0\n"
163 " isb\n"
164 "1: subs %[loop], %[loop], #1\n"
165 " bgt 1b\n"
166 " mcr p15, 0, %[z], c9, c12, 0\n"
167 " isb\n"
168 : [loop] "+r" (loop)
169 : [pmcr] "r" (pmcr), [z] "r" (0)
170 : "cc");
171 }
172
pmu_reset(void)173 static void pmu_reset(void)
174 {
175 /* reset all counters, counting disabled at PMCR level*/
176 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P);
177 /* Disable all counters */
178 write_sysreg(ALL_SET_32, PMCNTENCLR);
179 /* clear overflow reg */
180 write_sysreg(ALL_SET_32, PMOVSR);
181 /* disable overflow interrupts on all counters */
182 write_sysreg(ALL_SET_32, PMINTENCLR);
183 isb();
184 }
185
186 /* event counter tests only implemented for aarch64 */
test_event_introspection(void)187 static void test_event_introspection(void) {}
test_event_counter_config(void)188 static void test_event_counter_config(void) {}
test_basic_event_count(bool overflow_at_64bits)189 static void test_basic_event_count(bool overflow_at_64bits) {}
test_mem_access_reliability(bool overflow_at_64bits)190 static void test_mem_access_reliability(bool overflow_at_64bits) {}
test_mem_access(bool overflow_at_64bits)191 static void test_mem_access(bool overflow_at_64bits) {}
test_sw_incr(bool overflow_at_64bits)192 static void test_sw_incr(bool overflow_at_64bits) {}
test_chained_counters(bool unused)193 static void test_chained_counters(bool unused) {}
test_chained_sw_incr(bool unused)194 static void test_chained_sw_incr(bool unused) {}
test_chain_promotion(bool unused)195 static void test_chain_promotion(bool unused) {}
test_overflow_interrupt(bool overflow_at_64bits)196 static void test_overflow_interrupt(bool overflow_at_64bits) {}
197
198 #elif defined(__aarch64__)
199 #define ID_AA64DFR0_PERFMON_SHIFT 8
200 #define ID_AA64DFR0_PERFMON_MASK 0xf
201
202 #define ID_DFR0_PMU_NOTIMPL 0b0000
203 #define ID_DFR0_PMU_V3 0b0001
204 #define ID_DFR0_PMU_V3_8_1 0b0100
205 #define ID_DFR0_PMU_V3_8_4 0b0101
206 #define ID_DFR0_PMU_V3_8_5 0b0110
207 #define ID_DFR0_PMU_IMPDEF 0b1111
208
get_id_aa64dfr0(void)209 static inline uint32_t get_id_aa64dfr0(void) { return read_sysreg(id_aa64dfr0_el1); }
get_pmcr(void)210 static inline uint32_t get_pmcr(void) { return read_sysreg(pmcr_el0); }
set_pmcr(uint32_t v)211 static inline void set_pmcr(uint32_t v) { write_sysreg(v, pmcr_el0); }
get_pmccntr(void)212 static inline uint64_t get_pmccntr(void) { return read_sysreg(pmccntr_el0); }
set_pmccntr(uint64_t v)213 static inline void set_pmccntr(uint64_t v) { write_sysreg(v, pmccntr_el0); }
set_pmcntenset(uint32_t v)214 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, pmcntenset_el0); }
set_pmccfiltr(uint32_t v)215 static inline void set_pmccfiltr(uint32_t v) { write_sysreg(v, pmccfiltr_el0); }
216
get_pmu_version(void)217 static inline uint8_t get_pmu_version(void)
218 {
219 uint8_t ver = (get_id_aa64dfr0() >> ID_AA64DFR0_PERFMON_SHIFT) & ID_AA64DFR0_PERFMON_MASK;
220 return ver;
221 }
222
223 /*
224 * Extra instructions inserted by the compiler would be difficult to compensate
225 * for, so hand assemble everything between, and including, the PMCR accesses
226 * to start and stop counting. isb instructions are inserted to make sure
227 * pmccntr read after this function returns the exact instructions executed
228 * in the controlled block. Total instrs = isb + msr + 2*loop = 2 + 2*loop.
229 */
precise_instrs_loop(int loop,uint32_t pmcr)230 static inline void precise_instrs_loop(int loop, uint32_t pmcr)
231 {
232 uint64_t pmcr64 = pmcr;
233 asm volatile(
234 " msr pmcr_el0, %[pmcr]\n"
235 " isb\n"
236 "1: subs %w[loop], %w[loop], #1\n"
237 " b.gt 1b\n"
238 " msr pmcr_el0, xzr\n"
239 " isb\n"
240 : [loop] "+r" (loop)
241 : [pmcr] "r" (pmcr64)
242 : "cc");
243 }
244
245 #define PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7)
246 #define PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1)
247 #define PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2)
248
249 #define PMEVTYPER_EXCLUDE_EL1 BIT(31)
250 #define PMEVTYPER_EXCLUDE_EL0 BIT(30)
251
is_event_supported(uint32_t n,bool warn)252 static bool is_event_supported(uint32_t n, bool warn)
253 {
254 uint64_t pmceid0 = read_sysreg(pmceid0_el0);
255 uint64_t pmceid1 = read_sysreg_s(PMCEID1_EL0);
256 bool supported;
257 uint64_t reg;
258
259 /*
260 * The low 32-bits of PMCEID0/1 respectively describe
261 * event support for events 0-31/32-63. Their High
262 * 32-bits describe support for extended events
263 * starting at 0x4000, using the same split.
264 */
265 assert((n >= COMMON_EVENTS_LOW && n <= COMMON_EVENTS_HIGH) ||
266 (n >= EXT_COMMON_EVENTS_LOW && n <= EXT_COMMON_EVENTS_HIGH));
267
268 if (n <= COMMON_EVENTS_HIGH)
269 reg = lower_32_bits(pmceid0) | ((u64)lower_32_bits(pmceid1) << 32);
270 else
271 reg = upper_32_bits(pmceid0) | ((u64)upper_32_bits(pmceid1) << 32);
272
273 supported = reg & (1UL << (n & 0x3F));
274
275 if (!supported && warn)
276 report_info("event 0x%x is not supported", n);
277 return supported;
278 }
279
test_event_introspection(void)280 static void test_event_introspection(void)
281 {
282 bool required_events;
283
284 if (!pmu.nb_implemented_counters) {
285 report_skip("No event counter, skip ...");
286 return;
287 }
288
289 /* PMUv3 requires an implementation includes some common events */
290 required_events = is_event_supported(SW_INCR, true) &&
291 is_event_supported(CPU_CYCLES, true) &&
292 (is_event_supported(INST_RETIRED, true) ||
293 is_event_supported(INST_PREC, true));
294
295 if (pmu.version >= ID_DFR0_PMU_V3_8_1) {
296 required_events = required_events &&
297 is_event_supported(STALL_FRONTEND, true) &&
298 is_event_supported(STALL_BACKEND, true);
299 }
300
301 report(required_events, "Check required events are implemented");
302 }
303
304 /*
305 * Extra instructions inserted by the compiler would be difficult to compensate
306 * for, so hand assemble everything between, and including, the PMCR accesses
307 * to start and stop counting. isb instructions are inserted to make sure
308 * pmccntr read after this function returns the exact instructions executed
309 * in the controlled block. Loads @loop times the data at @address into x9.
310 */
mem_access_loop(void * addr,long loop,uint32_t pmcr)311 static void mem_access_loop(void *addr, long loop, uint32_t pmcr)
312 {
313 uint64_t pmcr64 = pmcr;
314 asm volatile(
315 " dsb ish\n"
316 " msr pmcr_el0, %[pmcr]\n"
317 " isb\n"
318 " dsb ish\n"
319 " mov x10, %[loop]\n"
320 "1: sub x10, x10, #1\n"
321 " ldr x9, [%[addr]]\n"
322 " cmp x10, #0x0\n"
323 " b.gt 1b\n"
324 " dsb ish\n"
325 " msr pmcr_el0, xzr\n"
326 " isb\n"
327 :
328 : [addr] "r" (addr), [pmcr] "r" (pmcr64), [loop] "r" (loop)
329 : "x9", "x10", "cc");
330 }
331
332 static volatile struct pmu_stats pmu_stats;
333
irq_handler(struct pt_regs * regs)334 static void irq_handler(struct pt_regs *regs)
335 {
336 uint32_t irqstat, irqnr;
337
338 irqstat = gic_read_iar();
339 irqnr = gic_iar_irqnr(irqstat);
340
341 if (irqnr == PMU_PPI) {
342 unsigned long overflows = read_sysreg(pmovsclr_el0);
343 int i;
344
345 for (i = 0; i < 32; i++) {
346 if (test_and_clear_bit(i, &overflows)) {
347 pmu_stats.interrupts[i]++;
348 pmu_stats.bitmap |= 1 << i;
349 }
350 }
351 write_sysreg(ALL_SET_32, pmovsclr_el0);
352 isb();
353 } else {
354 pmu_stats.unexpected = true;
355 }
356 gic_write_eoir(irqstat);
357 }
358
pmu_reset_stats(void)359 static void pmu_reset_stats(void)
360 {
361 int i;
362
363 for (i = 0; i < 32; i++)
364 pmu_stats.interrupts[i] = 0;
365
366 pmu_stats.bitmap = 0;
367 pmu_stats.unexpected = false;
368 }
369
pmu_reset(void)370 static void pmu_reset(void)
371 {
372 /* reset all counters, counting disabled at PMCR level*/
373 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P);
374 /* Disable all counters */
375 write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0);
376 /* clear overflow reg */
377 write_sysreg(ALL_SET_32, pmovsclr_el0);
378 /* disable overflow interrupts on all counters */
379 write_sysreg(ALL_SET_32, pmintenclr_el1);
380 pmu_reset_stats();
381 isb();
382 }
383
test_event_counter_config(void)384 static void test_event_counter_config(void)
385 {
386 int i;
387
388 if (!pmu.nb_implemented_counters) {
389 report_skip("No event counter, skip ...");
390 return;
391 }
392
393 pmu_reset();
394
395 /*
396 * Test setting through PMESELR/PMXEVTYPER and PMEVTYPERn read,
397 * select counter 0
398 */
399 write_sysreg(0, PMSELR_EL0);
400 /* program this counter to count unsupported event */
401 write_sysreg(0xEA, PMXEVTYPER_EL0);
402 write_sysreg(0xdeadbeef, PMXEVCNTR_EL0);
403 report((read_regn_el0(pmevtyper, 0) & 0xFFF) == 0xEA,
404 "PMESELR/PMXEVTYPER/PMEVTYPERn");
405 report((read_regn_el0(pmevcntr, 0) == 0xdeadbeef),
406 "PMESELR/PMXEVCNTR/PMEVCNTRn");
407
408 /* try to configure an unsupported event within the range [0x0, 0x3F] */
409 for (i = 0; i <= 0x3F; i++) {
410 if (!is_event_supported(i, false))
411 break;
412 }
413 if (i > 0x3F) {
414 report_skip("pmevtyper: all events within [0x0, 0x3F] are supported");
415 return;
416 }
417
418 /* select counter 0 */
419 write_sysreg(0, PMSELR_EL0);
420 /* program this counter to count unsupported event */
421 write_sysreg(i, PMXEVCNTR_EL0);
422 /* read the counter value */
423 read_sysreg(PMXEVCNTR_EL0);
424 report(read_sysreg(PMXEVCNTR_EL0) == i,
425 "read of a counter programmed with unsupported event");
426 }
427
satisfy_prerequisites(uint32_t * events,unsigned int nb_events)428 static bool satisfy_prerequisites(uint32_t *events, unsigned int nb_events)
429 {
430 int i;
431
432 if (pmu.nb_implemented_counters < nb_events) {
433 report_skip("Skip test as number of counters is too small (%d)",
434 pmu.nb_implemented_counters);
435 return false;
436 }
437
438 for (i = 0; i < nb_events; i++) {
439 if (!is_event_supported(events[i], false)) {
440 report_skip("Skip test as event 0x%x is not supported",
441 events[i]);
442 return false;
443 }
444 }
445 return true;
446 }
447
pmevcntr_mask(void)448 static uint64_t pmevcntr_mask(void)
449 {
450 /*
451 * Bits [63:0] are always incremented for 64-bit counters,
452 * even if the PMU is configured to generate an overflow at
453 * bits [31:0]
454 *
455 * For more details see the AArch64.IncrementEventCounter()
456 * pseudo-code in the ARM ARM DDI 0487I.a, section J1.1.1.
457 */
458 if (pmu.version >= ID_DFR0_PMU_V3_8_5)
459 return ~0;
460
461 return (uint32_t)~0;
462 }
463
check_overflow_prerequisites(bool overflow_at_64bits)464 static bool check_overflow_prerequisites(bool overflow_at_64bits)
465 {
466 if (overflow_at_64bits && pmu.version < ID_DFR0_PMU_V3_8_5) {
467 report_skip("Skip test as 64 overflows need FEAT_PMUv3p5");
468 return false;
469 }
470
471 return true;
472 }
473
test_basic_event_count(bool overflow_at_64bits)474 static void test_basic_event_count(bool overflow_at_64bits)
475 {
476 uint32_t implemented_counter_mask, non_implemented_counter_mask;
477 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
478 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
479 uint32_t events[] = {CPU_CYCLES, INST_RETIRED};
480 uint32_t counter_mask;
481
482 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
483 !check_overflow_prerequisites(overflow_at_64bits))
484 return;
485
486 implemented_counter_mask = BIT(pmu.nb_implemented_counters) - 1;
487 non_implemented_counter_mask = ~(BIT(31) | implemented_counter_mask);
488 counter_mask = implemented_counter_mask | non_implemented_counter_mask;
489
490 write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
491 write_regn_el0(pmevtyper, 1, INST_RETIRED | PMEVTYPER_EXCLUDE_EL0);
492
493 /* disable all counters */
494 write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0);
495 report(!read_sysreg_s(PMCNTENCLR_EL0) && !read_sysreg_s(PMCNTENSET_EL0),
496 "pmcntenclr: disable all counters");
497
498 /*
499 * clear cycle and all event counters and allow counter enablement
500 * through PMCNTENSET. LC is RES1.
501 */
502 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P | pmcr_lp);
503 isb();
504 report(get_pmcr() == (pmu.pmcr_ro | PMU_PMCR_LC | pmcr_lp), "pmcr: reset counters");
505
506 /* Preset counter #0 to pre overflow value to trigger an overflow */
507 write_regn_el0(pmevcntr, 0, pre_overflow);
508 report(read_regn_el0(pmevcntr, 0) == pre_overflow,
509 "counter #0 preset to pre-overflow value");
510 report(!read_regn_el0(pmevcntr, 1), "counter #1 is 0");
511
512 /*
513 * Enable all implemented counters and also attempt to enable
514 * not supported counters. Counting still is disabled by !PMCR.E
515 */
516 write_sysreg_s(counter_mask, PMCNTENSET_EL0);
517
518 /* check only those implemented are enabled */
519 report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) &&
520 (read_sysreg_s(PMCNTENSET_EL0) == implemented_counter_mask),
521 "pmcntenset: enabled implemented_counters");
522
523 /* Disable all counters but counters #0 and #1 */
524 write_sysreg_s(~0x3, PMCNTENCLR_EL0);
525 report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) &&
526 (read_sysreg_s(PMCNTENSET_EL0) == 0x3),
527 "pmcntenset: just enabled #0 and #1");
528
529 /* clear overflow register */
530 write_sysreg(ALL_SET_32, pmovsclr_el0);
531 report(!read_sysreg(pmovsclr_el0), "check overflow reg is 0");
532
533 /* disable overflow interrupts on all counters*/
534 write_sysreg(ALL_SET_32, pmintenclr_el1);
535 report(!read_sysreg(pmintenclr_el1),
536 "pmintenclr_el1=0, all interrupts disabled");
537
538 /* enable overflow interrupts on all event counters */
539 write_sysreg(implemented_counter_mask | non_implemented_counter_mask,
540 pmintenset_el1);
541 report(read_sysreg(pmintenset_el1) == implemented_counter_mask,
542 "overflow interrupts enabled on all implemented counters");
543
544 /* Set PMCR.E, execute asm code and unset PMCR.E */
545 precise_instrs_loop(20, pmu.pmcr_ro | PMU_PMCR_E);
546
547 report_info("counter #0 is 0x%lx (CPU_CYCLES)",
548 read_regn_el0(pmevcntr, 0));
549 report_info("counter #1 is 0x%lx (INST_RETIRED)",
550 read_regn_el0(pmevcntr, 1));
551
552 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0));
553 report(read_sysreg(pmovsclr_el0) == 0x1,
554 "check overflow happened on #0 only");
555 }
556
test_mem_access(bool overflow_at_64bits)557 static void test_mem_access(bool overflow_at_64bits)
558 {
559 void *addr = malloc(PAGE_SIZE);
560 uint32_t events[] = {MEM_ACCESS, MEM_ACCESS};
561 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
562 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
563
564 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
565 !check_overflow_prerequisites(overflow_at_64bits))
566 return;
567
568 pmu_reset();
569
570 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
571 write_regn_el0(pmevtyper, 1, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
572 write_sysreg_s(0x3, PMCNTENSET_EL0);
573 isb();
574 mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
575 report_info("counter #0 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 0));
576 report_info("counter #1 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 1));
577 /* We may measure more than 20 mem access depending on the core */
578 report((read_regn_el0(pmevcntr, 0) == read_regn_el0(pmevcntr, 1)) &&
579 (read_regn_el0(pmevcntr, 0) >= 20) && !read_sysreg(pmovsclr_el0),
580 "Ran 20 mem accesses");
581
582 pmu_reset();
583
584 write_regn_el0(pmevcntr, 0, pre_overflow);
585 write_regn_el0(pmevcntr, 1, pre_overflow);
586 write_sysreg_s(0x3, PMCNTENSET_EL0);
587 isb();
588 mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
589 report(read_sysreg(pmovsclr_el0) == 0x3,
590 "Ran 20 mem accesses with expected overflows on both counters");
591 report_info("cnt#0=0x%lx cnt#1=0x%lx overflow=0x%lx",
592 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1),
593 read_sysreg(pmovsclr_el0));
594 }
595
test_sw_incr(bool overflow_at_64bits)596 static void test_sw_incr(bool overflow_at_64bits)
597 {
598 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
599 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
600 uint32_t events[] = {SW_INCR, SW_INCR};
601 uint64_t cntr0 = (pre_overflow + 100) & pmevcntr_mask();
602 int i;
603
604 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
605 !check_overflow_prerequisites(overflow_at_64bits))
606 return;
607
608 pmu_reset();
609
610 write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
611 write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
612 /* enable counters #0 and #1 */
613 write_sysreg_s(0x3, PMCNTENSET_EL0);
614
615 write_regn_el0(pmevcntr, 0, pre_overflow);
616 isb();
617
618 for (i = 0; i < 100; i++)
619 write_sysreg(0x1, pmswinc_el0);
620
621 isb();
622 report_info("SW_INCR counter #0 has value 0x%lx", read_regn_el0(pmevcntr, 0));
623 report(read_regn_el0(pmevcntr, 0) == pre_overflow,
624 "PWSYNC does not increment if PMCR.E is unset");
625
626 pmu_reset();
627
628 write_regn_el0(pmevcntr, 0, pre_overflow);
629 write_sysreg_s(0x3, PMCNTENSET_EL0);
630 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
631 isb();
632
633 for (i = 0; i < 100; i++)
634 write_sysreg(0x3, pmswinc_el0);
635
636 isb();
637 report(read_regn_el0(pmevcntr, 0) == cntr0, "counter #0 after + 100 SW_INCR");
638 report(read_regn_el0(pmevcntr, 1) == 100, "counter #1 after + 100 SW_INCR");
639 report_info("counter values after 100 SW_INCR #0=0x%lx #1=0x%lx",
640 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1));
641 report(read_sysreg(pmovsclr_el0) == 0x1,
642 "overflow on counter #0 after 100 SW_INCR");
643 }
644
enable_chain_counter(int even)645 static void enable_chain_counter(int even)
646 {
647 write_sysreg_s(BIT(even + 1), PMCNTENSET_EL0); /* Enable the high counter first */
648 isb();
649 write_sysreg_s(BIT(even), PMCNTENSET_EL0); /* Enable the low counter */
650 isb();
651 }
652
disable_chain_counter(int even)653 static void disable_chain_counter(int even)
654 {
655 write_sysreg_s(BIT(even), PMCNTENCLR_EL0); /* Disable the low counter first*/
656 isb();
657 write_sysreg_s(BIT(even + 1), PMCNTENCLR_EL0); /* Disable the high counter */
658 isb();
659 }
660
test_chained_counters(bool unused)661 static void test_chained_counters(bool unused)
662 {
663 uint32_t events[] = {CPU_CYCLES, CHAIN};
664 uint64_t all_set = pmevcntr_mask();
665
666 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)))
667 return;
668
669 pmu_reset();
670
671 write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
672 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
673 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
674 enable_chain_counter(0);
675
676 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E);
677
678 report(read_regn_el0(pmevcntr, 1) == 1, "CHAIN counter #1 incremented");
679 report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #1");
680
681 /* test 64b overflow */
682
683 pmu_reset();
684
685 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
686 write_regn_el0(pmevcntr, 1, 0x1);
687 enable_chain_counter(0);
688 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E);
689 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0));
690 report(read_regn_el0(pmevcntr, 1) == 2, "CHAIN counter #1 set to 2");
691 report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #2");
692
693 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
694 write_regn_el0(pmevcntr, 1, all_set);
695
696 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E);
697 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0));
698 report(read_regn_el0(pmevcntr, 1) == 0, "CHAIN counter #1 wrapped");
699 report(read_sysreg(pmovsclr_el0) == 0x3, "overflow on even and odd counters");
700 }
701
test_chained_sw_incr(bool unused)702 static void test_chained_sw_incr(bool unused)
703 {
704 uint32_t events[] = {SW_INCR, CHAIN};
705 uint64_t cntr0 = (PRE_OVERFLOW_32 + 100) & pmevcntr_mask();
706 uint64_t cntr1 = (ALL_SET_32 + 1) & pmevcntr_mask();
707 int i;
708
709 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)))
710 return;
711
712 pmu_reset();
713
714 write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
715 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
716 enable_chain_counter(0);
717
718 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
719 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E);
720 isb();
721
722 for (i = 0; i < 100; i++)
723 write_sysreg(0x1, pmswinc_el0);
724
725 isb();
726 report((read_sysreg(pmovsclr_el0) == 0x1) &&
727 (read_regn_el0(pmevcntr, 1) == 1),
728 "overflow and chain counter incremented after 100 SW_INCR/CHAIN");
729 report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0),
730 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1));
731
732 /* 64b SW_INCR and overflow on CHAIN counter*/
733 pmu_reset();
734
735 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
736 write_regn_el0(pmevcntr, 1, ALL_SET_32);
737 enable_chain_counter(0);
738 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E);
739 isb();
740
741 for (i = 0; i < 100; i++)
742 write_sysreg(0x1, pmswinc_el0);
743
744 isb();
745 report((read_sysreg(pmovsclr_el0) == 0x3) &&
746 (read_regn_el0(pmevcntr, 0) == cntr0) &&
747 (read_regn_el0(pmevcntr, 1) == cntr1),
748 "expected overflows and values after 100 SW_INCR/CHAIN");
749 report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0),
750 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1));
751 }
752 #define PRINT_REGS(__s) \
753 report_info("%s #1=0x%lx #0=0x%lx overflow=0x%lx", __s, \
754 read_regn_el0(pmevcntr, 1), \
755 read_regn_el0(pmevcntr, 0), \
756 read_sysreg(pmovsclr_el0))
757
758 /*
759 * This test checks that a mem access loop featuring COUNT accesses
760 * does not overflow with an init value of PRE_OVERFLOW2. It also
761 * records the min/max access count to see how much the counting
762 * is (un)reliable
763 */
test_mem_access_reliability(bool overflow_at_64bits)764 static void test_mem_access_reliability(bool overflow_at_64bits)
765 {
766 uint32_t events[] = {MEM_ACCESS};
767 void *addr = malloc(PAGE_SIZE);
768 uint64_t cntr_val, num_events, max = 0, min = pmevcntr_mask();
769 uint64_t pre_overflow2 = PRE_OVERFLOW2(overflow_at_64bits);
770 uint64_t all_set = ALL_SET(overflow_at_64bits);
771 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
772 bool overflow = false;
773
774 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
775 !check_overflow_prerequisites(overflow_at_64bits))
776 return;
777
778 pmu_reset();
779 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
780 for (int i = 0; i < 100; i++) {
781 pmu_reset();
782 write_regn_el0(pmevcntr, 0, pre_overflow2);
783 write_sysreg_s(0x1, PMCNTENSET_EL0);
784 isb();
785 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
786 cntr_val = read_regn_el0(pmevcntr, 0);
787 if (cntr_val >= pre_overflow2) {
788 num_events = cntr_val - pre_overflow2;
789 } else {
790 /* unexpected counter overflow */
791 num_events = cntr_val + all_set - pre_overflow2;
792 overflow = true;
793 report_info("iter=%d num_events=%ld min=%ld max=%ld overflow!!!",
794 i, num_events, min, max);
795 }
796 /* record extreme value */
797 max = MAX(num_events, max);
798 min = MIN(num_events, min);
799 }
800 report_info("overflow=%d min=%ld max=%ld expected=%d acceptable margin=%d",
801 overflow, min, max, COUNT, MARGIN);
802 report(!overflow, "mem_access count is reliable");
803 }
804
test_chain_promotion(bool unused)805 static void test_chain_promotion(bool unused)
806 {
807 uint32_t events[] = {MEM_ACCESS, CHAIN};
808 void *addr = malloc(PAGE_SIZE);
809
810 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)))
811 return;
812
813 /* Only enable CHAIN counter */
814 report_prefix_push("subtest1");
815 pmu_reset();
816 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
817 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
818 write_sysreg_s(0x2, PMCNTENSET_EL0);
819 isb();
820
821 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
822 PRINT_REGS("post");
823 report(!read_regn_el0(pmevcntr, 0),
824 "chain counter not counting if even counter is disabled");
825 report_prefix_pop();
826
827 /* Only enable even counter */
828 report_prefix_push("subtest2");
829 pmu_reset();
830 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
831 write_sysreg_s(0x1, PMCNTENSET_EL0);
832 isb();
833
834 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
835 PRINT_REGS("post");
836 report(!read_regn_el0(pmevcntr, 1) && (read_sysreg(pmovsclr_el0) == 0x1),
837 "odd counter did not increment on overflow if disabled");
838 report_prefix_pop();
839
840 /* 1st COUNT with CHAIN enabled, next COUNT with CHAIN disabled */
841 report_prefix_push("subtest3");
842 pmu_reset();
843 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
844 enable_chain_counter(0);
845 PRINT_REGS("init");
846
847 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
848 PRINT_REGS("After 1st loop");
849
850 /* disable the CHAIN event */
851 disable_chain_counter(0);
852 write_sysreg_s(0x1, PMCNTENSET_EL0); /* Enable the low counter */
853 isb();
854 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
855 PRINT_REGS("After 2nd loop");
856 report(read_sysreg(pmovsclr_el0) == 0x1,
857 "should have triggered an overflow on #0");
858 report(!read_regn_el0(pmevcntr, 1),
859 "CHAIN counter #1 shouldn't have incremented");
860 report_prefix_pop();
861
862 /* 1st COUNT with CHAIN disabled, next COUNT with CHAIN enabled */
863
864 report_prefix_push("subtest4");
865 pmu_reset();
866 write_sysreg_s(0x1, PMCNTENSET_EL0);
867 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
868 isb();
869 PRINT_REGS("init");
870
871 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
872 PRINT_REGS("After 1st loop");
873
874 /* Disable the low counter first and enable the chain counter */
875 write_sysreg_s(0x1, PMCNTENCLR_EL0);
876 isb();
877 enable_chain_counter(0);
878
879 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
880
881 PRINT_REGS("After 2nd loop");
882
883 report((read_regn_el0(pmevcntr, 1) == 1) &&
884 (read_sysreg(pmovsclr_el0) == 0x1),
885 "CHAIN counter enabled: CHAIN counter was incremented and overflow");
886 report_prefix_pop();
887
888 /* start as MEM_ACCESS/CPU_CYCLES and move to CHAIN/MEM_ACCESS */
889 report_prefix_push("subtest5");
890 pmu_reset();
891 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
892 write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
893 write_sysreg_s(0x3, PMCNTENSET_EL0);
894 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
895 isb();
896 PRINT_REGS("init");
897
898 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
899 PRINT_REGS("After 1st loop");
900
901 /* 0 becomes CHAINED */
902 write_sysreg_s(0x3, PMCNTENCLR_EL0);
903 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
904 write_regn_el0(pmevcntr, 1, 0x0);
905 enable_chain_counter(0);
906
907 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
908 PRINT_REGS("After 2nd loop");
909
910 report((read_regn_el0(pmevcntr, 1) == 1) &&
911 (read_sysreg(pmovsclr_el0) == 0x1),
912 "32b->64b: CHAIN counter incremented and overflow");
913 report_prefix_pop();
914
915 /* start as CHAIN/MEM_ACCESS and move to MEM_ACCESS/CPU_CYCLES */
916 report_prefix_push("subtest6");
917 pmu_reset();
918 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
919 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
920 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
921 enable_chain_counter(0);
922 PRINT_REGS("init");
923
924 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
925 PRINT_REGS("After 1st loop");
926
927 disable_chain_counter(0);
928 write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
929 write_sysreg_s(0x3, PMCNTENSET_EL0);
930
931 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
932 PRINT_REGS("After 2nd loop");
933 report(read_sysreg(pmovsclr_el0) == 1,
934 "overflow is expected on counter 0");
935 report_prefix_pop();
936 }
937
expect_interrupts(uint32_t bitmap)938 static bool expect_interrupts(uint32_t bitmap)
939 {
940 int i;
941
942 if (pmu_stats.bitmap ^ bitmap || pmu_stats.unexpected)
943 return false;
944
945 for (i = 0; i < 32; i++) {
946 if (test_and_clear_bit(i, &pmu_stats.bitmap))
947 if (pmu_stats.interrupts[i] != 1)
948 return false;
949 }
950 return true;
951 }
952
test_overflow_interrupt(bool overflow_at_64bits)953 static void test_overflow_interrupt(bool overflow_at_64bits)
954 {
955 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
956 uint64_t all_set = pmevcntr_mask();
957 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
958 uint32_t events[] = {MEM_ACCESS, SW_INCR};
959 void *addr = malloc(PAGE_SIZE);
960 int i;
961
962 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
963 !check_overflow_prerequisites(overflow_at_64bits))
964 return;
965
966 gic_enable_defaults();
967 install_irq_handler(EL1H_IRQ, irq_handler);
968 local_irq_enable();
969 gic_enable_irq(23);
970
971 pmu_reset();
972
973 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
974 write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
975 write_sysreg_s(0x3, PMCNTENSET_EL0);
976 write_regn_el0(pmevcntr, 0, pre_overflow);
977 write_regn_el0(pmevcntr, 1, pre_overflow);
978 isb();
979
980 /* interrupts are disabled (PMINTENSET_EL1 == 0) */
981
982 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
983 report(expect_interrupts(0), "no overflow interrupt after preset");
984
985 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
986 isb();
987
988 for (i = 0; i < 100; i++)
989 write_sysreg(0x2, pmswinc_el0);
990
991 isb();
992 set_pmcr(pmu.pmcr_ro);
993 isb();
994 report(expect_interrupts(0), "no overflow interrupt after counting");
995
996 /* enable interrupts (PMINTENSET_EL1 <= ALL_SET_32) */
997
998 pmu_reset_stats();
999
1000 write_regn_el0(pmevcntr, 0, pre_overflow);
1001 write_regn_el0(pmevcntr, 1, pre_overflow);
1002 write_sysreg(ALL_SET_32, pmovsclr_el0);
1003 write_sysreg(ALL_SET_32, pmintenset_el1);
1004 isb();
1005
1006 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1007
1008 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1009 isb();
1010
1011 for (i = 0; i < 100; i++)
1012 write_sysreg(0x3, pmswinc_el0);
1013
1014 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro);
1015 report_info("overflow=0x%lx", read_sysreg(pmovsclr_el0));
1016 report(expect_interrupts(0x3),
1017 "overflow interrupts expected on #0 and #1");
1018
1019 /*
1020 * promote to 64-b:
1021 *
1022 * This only applies to the !overflow_at_64bits case, as
1023 * overflow_at_64bits doesn't implement CHAIN events. The
1024 * overflow_at_64bits case just checks that chained counters are
1025 * not incremented when PMCR.LP == 1.
1026 */
1027
1028 pmu_reset_stats();
1029
1030 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
1031 write_regn_el0(pmevcntr, 0, pre_overflow);
1032 isb();
1033 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1034 report(expect_interrupts(0x1), "expect overflow interrupt");
1035
1036 /* overflow on odd counter */
1037 pmu_reset_stats();
1038 write_regn_el0(pmevcntr, 0, pre_overflow);
1039 write_regn_el0(pmevcntr, 1, all_set);
1040 isb();
1041 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1042 if (overflow_at_64bits) {
1043 report(expect_interrupts(0x1),
1044 "expect overflow interrupt on even counter");
1045 report(read_regn_el0(pmevcntr, 1) == all_set,
1046 "Odd counter did not change");
1047 } else {
1048 report(expect_interrupts(0x3),
1049 "expect overflow interrupt on even and odd counter");
1050 report(read_regn_el0(pmevcntr, 1) != all_set,
1051 "Odd counter wrapped");
1052 }
1053 }
1054 #endif
1055
1056 /*
1057 * Ensure that the cycle counter progresses between back-to-back reads.
1058 */
check_cycles_increase(void)1059 static bool check_cycles_increase(void)
1060 {
1061 bool success = true;
1062
1063 /* init before event access, this test only cares about cycle count */
1064 pmu_reset();
1065 set_pmcntenset(1 << PMU_CYCLE_IDX);
1066 set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */
1067
1068 set_pmcr(get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E);
1069 isb();
1070
1071 for (int i = 0; i < NR_SAMPLES; i++) {
1072 uint64_t a, b;
1073
1074 a = get_pmccntr();
1075 b = get_pmccntr();
1076
1077 if (a >= b) {
1078 printf("Read %"PRId64" then %"PRId64".\n", a, b);
1079 success = false;
1080 break;
1081 }
1082 }
1083
1084 set_pmcr(get_pmcr() & ~PMU_PMCR_E);
1085 isb();
1086
1087 return success;
1088 }
1089
1090 /*
1091 * Execute a known number of guest instructions. Only even instruction counts
1092 * greater than or equal to 4 are supported by the in-line assembly code. The
1093 * control register (PMCR_EL0) is initialized with the provided value (allowing
1094 * for example for the cycle counter or event counters to be reset). At the end
1095 * of the exact instruction loop, zero is written to PMCR_EL0 to disable
1096 * counting, allowing the cycle counter or event counters to be read at the
1097 * leisure of the calling code.
1098 */
measure_instrs(int num,uint32_t pmcr)1099 static void measure_instrs(int num, uint32_t pmcr)
1100 {
1101 int loop = (num - 2) / 2;
1102
1103 assert(num >= 4 && ((num - 2) % 2 == 0));
1104 precise_instrs_loop(loop, pmcr);
1105 }
1106
1107 /*
1108 * Measure cycle counts for various known instruction counts. Ensure that the
1109 * cycle counter progresses (similar to check_cycles_increase() but with more
1110 * instructions and using reset and stop controls). If supplied a positive,
1111 * nonzero CPI parameter, it also strictly checks that every measurement matches
1112 * it. Strict CPI checking is used to test -icount mode.
1113 */
check_cpi(int cpi)1114 static bool check_cpi(int cpi)
1115 {
1116 uint32_t pmcr = get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E;
1117
1118 /* init before event access, this test only cares about cycle count */
1119 pmu_reset();
1120 set_pmcntenset(1 << PMU_CYCLE_IDX);
1121 set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */
1122
1123 if (cpi > 0)
1124 printf("Checking for CPI=%d.\n", cpi);
1125 printf("instrs : cycles0 cycles1 ...\n");
1126
1127 for (unsigned int i = 4; i < 300; i += 32) {
1128 uint64_t avg, sum = 0;
1129
1130 printf("%4d:", i);
1131 for (int j = 0; j < NR_SAMPLES; j++) {
1132 uint64_t cycles;
1133
1134 set_pmccntr(0);
1135 measure_instrs(i, pmcr);
1136 cycles = get_pmccntr();
1137 printf(" %4"PRId64"", cycles);
1138
1139 if (!cycles) {
1140 printf("\ncycles not incrementing!\n");
1141 return false;
1142 } else if (cpi > 0 && cycles != i * cpi) {
1143 printf("\nunexpected cycle count received!\n");
1144 return false;
1145 } else if ((cycles >> 32) != 0) {
1146 /* The cycles taken by the loop above should
1147 * fit in 32 bits easily. We check the upper
1148 * 32 bits of the cycle counter to make sure
1149 * there is no supprise. */
1150 printf("\ncycle count bigger than 32bit!\n");
1151 return false;
1152 }
1153
1154 sum += cycles;
1155 }
1156 avg = sum / NR_SAMPLES;
1157 printf(" avg=%-4"PRId64" %s=%-3"PRId64"\n", avg,
1158 (avg >= i) ? "cpi" : "ipc",
1159 (avg >= i) ? avg / i : i / avg);
1160 }
1161
1162 return true;
1163 }
1164
pmccntr64_test(void)1165 static void pmccntr64_test(void)
1166 {
1167 #ifdef __arm__
1168 if (pmu.version == ID_DFR0_PMU_V3) {
1169 if (ERRATA(9e3f7a296940)) {
1170 write_sysreg(0xdead, PMCCNTR64);
1171 report(read_sysreg(PMCCNTR64) == 0xdead, "pmccntr64");
1172 } else
1173 report_skip("Skipping unsafe pmccntr64 test. Set ERRATA_9e3f7a296940=y to enable.");
1174 }
1175 #endif
1176 }
1177
1178 /* Return FALSE if no PMU found, otherwise return TRUE */
pmu_probe(void)1179 static bool pmu_probe(void)
1180 {
1181 uint32_t pmcr;
1182 uint8_t implementer;
1183
1184 pmu.version = get_pmu_version();
1185 if (pmu.version == ID_DFR0_PMU_NOTIMPL || pmu.version == ID_DFR0_PMU_IMPDEF)
1186 return false;
1187
1188 report_info("PMU version: 0x%x", pmu.version);
1189
1190 pmcr = get_pmcr();
1191 implementer = (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK;
1192 report_info("PMU implementer/ID code: %#"PRIx32"(\"%c\")/%#"PRIx32,
1193 (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK,
1194 implementer ? implementer : ' ',
1195 (pmcr >> PMU_PMCR_ID_SHIFT) & PMU_PMCR_ID_MASK);
1196
1197 /* store read-only and RES0 fields of the PMCR bottom-half*/
1198 pmu.pmcr_ro = pmcr & 0xFFFFFF00;
1199 pmu.nb_implemented_counters =
1200 (pmcr >> PMU_PMCR_N_SHIFT) & PMU_PMCR_N_MASK;
1201 report_info("Implements %d event counters",
1202 pmu.nb_implemented_counters);
1203
1204 return true;
1205 }
1206
run_test(const char * name,const char * prefix,void (* test)(bool),void * arg)1207 static void run_test(const char *name, const char *prefix,
1208 void (*test)(bool), void *arg)
1209 {
1210 report_prefix_push(name);
1211 report_prefix_push(prefix);
1212
1213 test(arg);
1214
1215 report_prefix_pop();
1216 report_prefix_pop();
1217 }
1218
run_event_test(char * name,void (* test)(bool),bool overflow_at_64bits)1219 static void run_event_test(char *name, void (*test)(bool),
1220 bool overflow_at_64bits)
1221 {
1222 const char *prefix = overflow_at_64bits ? "64-bit overflows"
1223 : "32-bit overflows";
1224
1225 run_test(name, prefix, test, (void *)overflow_at_64bits);
1226 }
1227
main(int argc,char * argv[])1228 int main(int argc, char *argv[])
1229 {
1230 int cpi = 0;
1231
1232 if (!pmu_probe()) {
1233 printf("No PMU found, test skipped...\n");
1234 return report_summary();
1235 }
1236
1237 if (argc < 2)
1238 report_abort("no test specified");
1239
1240 report_prefix_push("pmu");
1241
1242 if (strcmp(argv[1], "cycle-counter") == 0) {
1243 report_prefix_push(argv[1]);
1244 if (argc > 2)
1245 cpi = atol(argv[2]);
1246 report(check_cycles_increase(),
1247 "Monotonically increasing cycle count");
1248 report(check_cpi(cpi), "Cycle/instruction ratio");
1249 pmccntr64_test();
1250 report_prefix_pop();
1251 } else if (strcmp(argv[1], "pmu-event-introspection") == 0) {
1252 report_prefix_push(argv[1]);
1253 test_event_introspection();
1254 report_prefix_pop();
1255 } else if (strcmp(argv[1], "pmu-event-counter-config") == 0) {
1256 report_prefix_push(argv[1]);
1257 test_event_counter_config();
1258 report_prefix_pop();
1259 } else if (strcmp(argv[1], "pmu-basic-event-count") == 0) {
1260 run_event_test(argv[1], test_basic_event_count, false);
1261 run_event_test(argv[1], test_basic_event_count, true);
1262 } else if (strcmp(argv[1], "pmu-mem-access-reliability") == 0) {
1263 run_event_test(argv[1], test_mem_access_reliability, false);
1264 run_event_test(argv[1], test_mem_access_reliability, true);
1265 } else if (strcmp(argv[1], "pmu-mem-access") == 0) {
1266 run_event_test(argv[1], test_mem_access, false);
1267 run_event_test(argv[1], test_mem_access, true);
1268 } else if (strcmp(argv[1], "pmu-sw-incr") == 0) {
1269 run_event_test(argv[1], test_sw_incr, false);
1270 run_event_test(argv[1], test_sw_incr, true);
1271 } else if (strcmp(argv[1], "pmu-chained-counters") == 0) {
1272 run_event_test(argv[1], test_chained_counters, false);
1273 } else if (strcmp(argv[1], "pmu-chained-sw-incr") == 0) {
1274 run_event_test(argv[1], test_chained_sw_incr, false);
1275 } else if (strcmp(argv[1], "pmu-chain-promotion") == 0) {
1276 run_event_test(argv[1], test_chain_promotion, false);
1277 } else if (strcmp(argv[1], "pmu-overflow-interrupt") == 0) {
1278 run_event_test(argv[1], test_overflow_interrupt, false);
1279 run_event_test(argv[1], test_overflow_interrupt, true);
1280 } else {
1281 report_abort("Unknown sub-test '%s'", argv[1]);
1282 }
1283
1284 return report_summary();
1285 }
1286