xref: /kvm-unit-tests/arm/pmu.c (revision f77fb696cfd0e4a5562cdca189be557946bf522f)
1 /*
2  * Test the ARM Performance Monitors Unit (PMU).
3  *
4  * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
5  * Copyright (C) 2016, Red Hat Inc, Wei Huang <wei@redhat.com>
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms of the GNU Lesser General Public License version 2.1 and
9  * only version 2.1 as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
14  * for more details.
15  */
16 #include "libcflat.h"
17 #include "errata.h"
18 #include "asm/barrier.h"
19 #include "asm/sysreg.h"
20 #include "asm/processor.h"
21 #include <bitops.h>
22 #include <asm/gic.h>
23 
24 #define PMU_PMCR_E         (1 << 0)
25 #define PMU_PMCR_P         (1 << 1)
26 #define PMU_PMCR_C         (1 << 2)
27 #define PMU_PMCR_D         (1 << 3)
28 #define PMU_PMCR_X         (1 << 4)
29 #define PMU_PMCR_DP        (1 << 5)
30 #define PMU_PMCR_LC        (1 << 6)
31 #define PMU_PMCR_LP        (1 << 7)
32 #define PMU_PMCR_N_SHIFT   11
33 #define PMU_PMCR_N_MASK    0x1f
34 #define PMU_PMCR_ID_SHIFT  16
35 #define PMU_PMCR_ID_MASK   0xff
36 #define PMU_PMCR_IMP_SHIFT 24
37 #define PMU_PMCR_IMP_MASK  0xff
38 
39 #define PMU_CYCLE_IDX      31
40 
41 #define NR_SAMPLES 10
42 
43 /* Some PMU events */
44 #define SW_INCR			0x0
45 #define INST_RETIRED		0x8
46 #define CPU_CYCLES		0x11
47 #define MEM_ACCESS		0x13
48 #define INST_PREC		0x1B
49 #define STALL_FRONTEND		0x23
50 #define STALL_BACKEND		0x24
51 #define CHAIN			0x1E
52 
53 #define COMMON_EVENTS_LOW	0x0
54 #define COMMON_EVENTS_HIGH	0x3F
55 #define EXT_COMMON_EVENTS_LOW	0x4000
56 #define EXT_COMMON_EVENTS_HIGH	0x403F
57 
58 #define ALL_SET_32		0x00000000FFFFFFFFULL
59 #define ALL_SET_64		0xFFFFFFFFFFFFFFFFULL
60 
61 #define ALL_SET(__overflow_at_64bits)				\
62 	(__overflow_at_64bits ? ALL_SET_64 : ALL_SET_32)
63 
64 #define ALL_CLEAR		0x0000000000000000ULL
65 #define PRE_OVERFLOW_32		0x00000000FFFFFFF0ULL
66 #define PRE_OVERFLOW_64		0xFFFFFFFFFFFFFFF0ULL
67 #define COUNT 250
68 #define MARGIN 100
69 #define COUNT_INT 1000
70 /*
71  * PRE_OVERFLOW2 is set so that 1st @COUNT iterations do not
72  * produce 32b overflow and 2nd @COUNT iterations do. To accommodate
73  * for some observed variability we take into account a given @MARGIN
74  */
75 #define PRE_OVERFLOW2_32		(ALL_SET_32 - COUNT - MARGIN)
76 #define PRE_OVERFLOW2_64		(ALL_SET_64 - COUNT - MARGIN)
77 
78 #define PRE_OVERFLOW2(__overflow_at_64bits)				\
79 	(__overflow_at_64bits ? PRE_OVERFLOW2_64 : PRE_OVERFLOW2_32)
80 
81 #define PRE_OVERFLOW(__overflow_at_64bits)				\
82 	(__overflow_at_64bits ? PRE_OVERFLOW_64 : PRE_OVERFLOW_32)
83 
84 #define PMU_PPI			23
85 
86 struct pmu {
87 	unsigned int version;
88 	unsigned int nb_implemented_counters;
89 	uint32_t pmcr_ro;
90 };
91 
92 struct pmu_stats {
93 	unsigned long bitmap;
94 	uint32_t interrupts[32];
95 	bool unexpected;
96 };
97 
98 static struct pmu pmu;
99 
100 #if defined(__arm__)
101 #define ID_DFR0_PERFMON_SHIFT 24
102 #define ID_DFR0_PERFMON_MASK  0xf
103 
104 #define ID_DFR0_PMU_NOTIMPL	0b0000
105 #define ID_DFR0_PMU_V1		0b0001
106 #define ID_DFR0_PMU_V2		0b0010
107 #define ID_DFR0_PMU_V3		0b0011
108 #define ID_DFR0_PMU_V3_8_1	0b0100
109 #define ID_DFR0_PMU_V3_8_4	0b0101
110 #define ID_DFR0_PMU_V3_8_5	0b0110
111 #define ID_DFR0_PMU_IMPDEF	0b1111
112 
113 #define PMCR         __ACCESS_CP15(c9, 0, c12, 0)
114 #define ID_DFR0      __ACCESS_CP15(c0, 0, c1, 2)
115 #define PMSELR       __ACCESS_CP15(c9, 0, c12, 5)
116 #define PMXEVTYPER   __ACCESS_CP15(c9, 0, c13, 1)
117 #define PMCNTENSET   __ACCESS_CP15(c9, 0, c12, 1)
118 #define PMCNTENCLR   __ACCESS_CP15(c9, 0, c12, 2)
119 #define PMOVSR       __ACCESS_CP15(c9, 0, c12, 3)
120 #define PMCCNTR32    __ACCESS_CP15(c9, 0, c13, 0)
121 #define PMINTENCLR   __ACCESS_CP15(c9, 0, c14, 2)
122 #define PMCCNTR64    __ACCESS_CP15_64(0, c9)
123 
get_id_dfr0(void)124 static inline uint32_t get_id_dfr0(void) { return read_sysreg(ID_DFR0); }
get_pmcr(void)125 static inline uint32_t get_pmcr(void) { return read_sysreg(PMCR); }
set_pmcr(uint32_t v)126 static inline void set_pmcr(uint32_t v) { write_sysreg(v, PMCR); }
set_pmcntenset(uint32_t v)127 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, PMCNTENSET); }
128 
get_pmu_version(void)129 static inline uint8_t get_pmu_version(void)
130 {
131 	return (get_id_dfr0() >> ID_DFR0_PERFMON_SHIFT) & ID_DFR0_PERFMON_MASK;
132 }
133 
get_pmccntr(void)134 static inline uint64_t get_pmccntr(void)
135 {
136 	return read_sysreg(PMCCNTR32);
137 }
138 
set_pmccntr(uint64_t value)139 static inline void set_pmccntr(uint64_t value)
140 {
141 	write_sysreg(value & 0xffffffff, PMCCNTR32);
142 }
143 
144 /* PMCCFILTR is an obsolete name for PMXEVTYPER31 in ARMv7 */
set_pmccfiltr(uint32_t value)145 static inline void set_pmccfiltr(uint32_t value)
146 {
147 	write_sysreg(PMU_CYCLE_IDX, PMSELR);
148 	write_sysreg(value, PMXEVTYPER);
149 	isb();
150 }
151 
152 /*
153  * Extra instructions inserted by the compiler would be difficult to compensate
154  * for, so hand assemble everything between, and including, the PMCR accesses
155  * to start and stop counting. isb instructions were inserted to make sure
156  * pmccntr read after this function returns the exact instructions executed in
157  * the controlled block. Total instrs = isb + mcr + 2*loop = 2 + 2*loop.
158  */
precise_instrs_loop(int loop,uint32_t pmcr)159 static inline void precise_instrs_loop(int loop, uint32_t pmcr)
160 {
161 	asm volatile(
162 	"	mcr	p15, 0, %[pmcr], c9, c12, 0\n"
163 	"	isb\n"
164 	"1:	subs	%[loop], %[loop], #1\n"
165 	"	bgt	1b\n"
166 	"	mcr	p15, 0, %[z], c9, c12, 0\n"
167 	"	isb\n"
168 	: [loop] "+r" (loop)
169 	: [pmcr] "r" (pmcr), [z] "r" (0)
170 	: "cc");
171 }
172 
pmu_reset(void)173 static void pmu_reset(void)
174 {
175 	/* reset all counters, counting disabled at PMCR level*/
176 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P);
177 	/* Disable all counters */
178 	write_sysreg(ALL_SET_32, PMCNTENCLR);
179 	/* clear overflow reg */
180 	write_sysreg(ALL_SET_32, PMOVSR);
181 	/* disable overflow interrupts on all counters */
182 	write_sysreg(ALL_SET_32, PMINTENCLR);
183 	isb();
184 }
185 
186 /* event counter tests only implemented for aarch64 */
test_event_introspection(void)187 static void test_event_introspection(void) {}
test_event_counter_config(void)188 static void test_event_counter_config(void) {}
test_basic_event_count(bool overflow_at_64bits)189 static void test_basic_event_count(bool overflow_at_64bits) {}
test_mem_access_reliability(bool overflow_at_64bits)190 static void test_mem_access_reliability(bool overflow_at_64bits) {}
test_mem_access(bool overflow_at_64bits)191 static void test_mem_access(bool overflow_at_64bits) {}
test_sw_incr(bool overflow_at_64bits)192 static void test_sw_incr(bool overflow_at_64bits) {}
test_chained_counters(bool unused)193 static void test_chained_counters(bool unused) {}
test_chained_sw_incr(bool unused)194 static void test_chained_sw_incr(bool unused) {}
test_chain_promotion(bool unused)195 static void test_chain_promotion(bool unused) {}
test_overflow_interrupt(bool overflow_at_64bits)196 static void test_overflow_interrupt(bool overflow_at_64bits) {}
197 
198 #elif defined(__aarch64__)
199 #define ID_AA64DFR0_PERFMON_SHIFT 8
200 #define ID_AA64DFR0_PERFMON_MASK  0xf
201 
202 #define ID_DFR0_PMU_NOTIMPL	0b0000
203 #define ID_DFR0_PMU_V3		0b0001
204 #define ID_DFR0_PMU_V3_8_1	0b0100
205 #define ID_DFR0_PMU_V3_8_4	0b0101
206 #define ID_DFR0_PMU_V3_8_5	0b0110
207 #define ID_DFR0_PMU_IMPDEF	0b1111
208 
get_id_aa64dfr0(void)209 static inline uint32_t get_id_aa64dfr0(void) { return read_sysreg(id_aa64dfr0_el1); }
get_pmcr(void)210 static inline uint32_t get_pmcr(void) { return read_sysreg(pmcr_el0); }
set_pmcr(uint32_t v)211 static inline void set_pmcr(uint32_t v) { write_sysreg(v, pmcr_el0); }
get_pmccntr(void)212 static inline uint64_t get_pmccntr(void) { return read_sysreg(pmccntr_el0); }
set_pmccntr(uint64_t v)213 static inline void set_pmccntr(uint64_t v) { write_sysreg(v, pmccntr_el0); }
set_pmcntenset(uint32_t v)214 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, pmcntenset_el0); }
set_pmccfiltr(uint32_t v)215 static inline void set_pmccfiltr(uint32_t v) { write_sysreg(v, pmccfiltr_el0); }
216 
get_pmu_version(void)217 static inline uint8_t get_pmu_version(void)
218 {
219 	uint8_t ver = (get_id_aa64dfr0() >> ID_AA64DFR0_PERFMON_SHIFT) & ID_AA64DFR0_PERFMON_MASK;
220 	return ver;
221 }
222 
223 /*
224  * Extra instructions inserted by the compiler would be difficult to compensate
225  * for, so hand assemble everything between, and including, the PMCR accesses
226  * to start and stop counting. isb instructions are inserted to make sure
227  * pmccntr read after this function returns the exact instructions executed
228  * in the controlled block. Total instrs = isb + msr + 2*loop = 2 + 2*loop.
229  */
precise_instrs_loop(int loop,uint32_t pmcr)230 static inline void precise_instrs_loop(int loop, uint32_t pmcr)
231 {
232 	uint64_t pmcr64 = pmcr;
233 	asm volatile(
234 	"	msr	pmcr_el0, %[pmcr]\n"
235 	"	isb\n"
236 	"1:	subs	%w[loop], %w[loop], #1\n"
237 	"	b.gt	1b\n"
238 	"	msr	pmcr_el0, xzr\n"
239 	"	isb\n"
240 	: [loop] "+r" (loop)
241 	: [pmcr] "r" (pmcr64)
242 	: "cc");
243 }
244 
245 #define PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7)
246 #define PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1)
247 #define PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2)
248 
249 #define PMEVTYPER_EXCLUDE_EL1 BIT(31)
250 #define PMEVTYPER_EXCLUDE_EL0 BIT(30)
251 
is_event_supported(uint32_t n,bool warn)252 static bool is_event_supported(uint32_t n, bool warn)
253 {
254 	uint64_t pmceid0 = read_sysreg(pmceid0_el0);
255 	uint64_t pmceid1 = read_sysreg_s(PMCEID1_EL0);
256 	bool supported;
257 	uint64_t reg;
258 
259 	/*
260 	 * The low 32-bits of PMCEID0/1 respectively describe
261 	 * event support for events 0-31/32-63. Their High
262 	 * 32-bits describe support for extended events
263 	 * starting at 0x4000, using the same split.
264 	 */
265 	assert((n >= COMMON_EVENTS_LOW  && n <= COMMON_EVENTS_HIGH) ||
266 	       (n >= EXT_COMMON_EVENTS_LOW && n <= EXT_COMMON_EVENTS_HIGH));
267 
268 	if (n <= COMMON_EVENTS_HIGH)
269 		reg = lower_32_bits(pmceid0) | ((u64)lower_32_bits(pmceid1) << 32);
270 	else
271 		reg = upper_32_bits(pmceid0) | ((u64)upper_32_bits(pmceid1) << 32);
272 
273 	supported =  reg & (1UL << (n & 0x3F));
274 
275 	if (!supported && warn)
276 		report_info("event 0x%x is not supported", n);
277 	return supported;
278 }
279 
test_event_introspection(void)280 static void test_event_introspection(void)
281 {
282 	bool required_events;
283 
284 	if (!pmu.nb_implemented_counters) {
285 		report_skip("No event counter, skip ...");
286 		return;
287 	}
288 
289 	/* PMUv3 requires an implementation includes some common events */
290 	required_events = is_event_supported(SW_INCR, true) &&
291 			  is_event_supported(CPU_CYCLES, true) &&
292 			  (is_event_supported(INST_RETIRED, true) ||
293 			   is_event_supported(INST_PREC, true));
294 
295 	if (pmu.version >= ID_DFR0_PMU_V3_8_1) {
296 		required_events = required_events &&
297 				  is_event_supported(STALL_FRONTEND, true) &&
298 				  is_event_supported(STALL_BACKEND, true);
299 	}
300 
301 	report(required_events, "Check required events are implemented");
302 }
303 
304 /*
305  * Extra instructions inserted by the compiler would be difficult to compensate
306  * for, so hand assemble everything between, and including, the PMCR accesses
307  * to start and stop counting. isb instructions are inserted to make sure
308  * pmccntr read after this function returns the exact instructions executed
309  * in the controlled block. Loads @loop times the data at @address into x9.
310  */
mem_access_loop(void * addr,long loop,uint32_t pmcr)311 static void mem_access_loop(void *addr, long loop, uint32_t pmcr)
312 {
313 	uint64_t pmcr64 = pmcr;
314 asm volatile(
315 	"       dsb     ish\n"
316 	"       msr     pmcr_el0, %[pmcr]\n"
317 	"       isb\n"
318 	"       dsb     ish\n"
319 	"       mov     x10, %[loop]\n"
320 	"1:     sub     x10, x10, #1\n"
321 	"       ldr	x9, [%[addr]]\n"
322 	"       cmp     x10, #0x0\n"
323 	"       b.gt    1b\n"
324 	"       dsb     ish\n"
325 	"       msr     pmcr_el0, xzr\n"
326 	"       isb\n"
327 	:
328 	: [addr] "r" (addr), [pmcr] "r" (pmcr64), [loop] "r" (loop)
329 	: "x9", "x10", "cc");
330 }
331 
332 static volatile struct pmu_stats pmu_stats;
333 
irq_handler(struct pt_regs * regs)334 static void irq_handler(struct pt_regs *regs)
335 {
336 	uint32_t irqstat, irqnr;
337 
338 	irqstat = gic_read_iar();
339 	irqnr = gic_iar_irqnr(irqstat);
340 
341 	if (irqnr == PMU_PPI) {
342 		unsigned long overflows = read_sysreg(pmovsclr_el0);
343 		int i;
344 
345 		for (i = 0; i < 32; i++) {
346 			if (test_and_clear_bit(i, &overflows)) {
347 				pmu_stats.interrupts[i]++;
348 				pmu_stats.bitmap |= 1 << i;
349 			}
350 		}
351 		write_sysreg(ALL_SET_32, pmovsclr_el0);
352 		isb();
353 	} else {
354 		pmu_stats.unexpected = true;
355 	}
356 	gic_write_eoir(irqstat);
357 }
358 
pmu_reset_stats(void)359 static void pmu_reset_stats(void)
360 {
361 	int i;
362 
363 	for (i = 0; i < 32; i++)
364 		pmu_stats.interrupts[i] = 0;
365 
366 	pmu_stats.bitmap = 0;
367 	pmu_stats.unexpected = false;
368 }
369 
pmu_reset(void)370 static void pmu_reset(void)
371 {
372 	/* reset all counters, counting disabled at PMCR level*/
373 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P);
374 	/* Disable all counters */
375 	write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0);
376 	/* clear overflow reg */
377 	write_sysreg(ALL_SET_32, pmovsclr_el0);
378 	/* disable overflow interrupts on all counters */
379 	write_sysreg(ALL_SET_32, pmintenclr_el1);
380 	pmu_reset_stats();
381 	isb();
382 }
383 
test_event_counter_config(void)384 static void test_event_counter_config(void)
385 {
386 	int i;
387 
388 	if (!pmu.nb_implemented_counters) {
389 		report_skip("No event counter, skip ...");
390 		return;
391 	}
392 
393 	pmu_reset();
394 
395 	/*
396 	 * Test setting through PMESELR/PMXEVTYPER and PMEVTYPERn read,
397 	 * select counter 0
398 	 */
399 	write_sysreg(0, PMSELR_EL0);
400 	/* program this counter to count unsupported event */
401 	write_sysreg(0xEA, PMXEVTYPER_EL0);
402 	write_sysreg(0xdeadbeef, PMXEVCNTR_EL0);
403 	report((read_regn_el0(pmevtyper, 0) & 0xFFF) == 0xEA,
404 		"PMESELR/PMXEVTYPER/PMEVTYPERn");
405 	report((read_regn_el0(pmevcntr, 0) == 0xdeadbeef),
406 		"PMESELR/PMXEVCNTR/PMEVCNTRn");
407 
408 	/* try to configure an unsupported event within the range [0x0, 0x3F] */
409 	for (i = 0; i <= 0x3F; i++) {
410 		if (!is_event_supported(i, false))
411 			break;
412 	}
413 	if (i > 0x3F) {
414 		report_skip("pmevtyper: all events within [0x0, 0x3F] are supported");
415 		return;
416 	}
417 
418 	/* select counter 0 */
419 	write_sysreg(0, PMSELR_EL0);
420 	/* program this counter to count unsupported event */
421 	write_sysreg(i, PMXEVCNTR_EL0);
422 	/* read the counter value */
423 	read_sysreg(PMXEVCNTR_EL0);
424 	report(read_sysreg(PMXEVCNTR_EL0) == i,
425 		"read of a counter programmed with unsupported event");
426 }
427 
satisfy_prerequisites(uint32_t * events,unsigned int nb_events)428 static bool satisfy_prerequisites(uint32_t *events, unsigned int nb_events)
429 {
430 	int i;
431 
432 	if (pmu.nb_implemented_counters < nb_events) {
433 		report_skip("Skip test as number of counters is too small (%d)",
434 			    pmu.nb_implemented_counters);
435 		return false;
436 	}
437 
438 	for (i = 0; i < nb_events; i++) {
439 		if (!is_event_supported(events[i], false)) {
440 			report_skip("Skip test as event 0x%x is not supported",
441 				    events[i]);
442 			return false;
443 		}
444 	}
445 	return true;
446 }
447 
pmevcntr_mask(void)448 static uint64_t pmevcntr_mask(void)
449 {
450 	/*
451 	 * Bits [63:0] are always incremented for 64-bit counters,
452 	 * even if the PMU is configured to generate an overflow at
453 	 * bits [31:0]
454 	 *
455 	 * For more details see the AArch64.IncrementEventCounter()
456 	 * pseudo-code in the ARM ARM DDI 0487I.a, section J1.1.1.
457 	 */
458 	if (pmu.version >= ID_DFR0_PMU_V3_8_5)
459 		return ~0;
460 
461 	return (uint32_t)~0;
462 }
463 
check_overflow_prerequisites(bool overflow_at_64bits)464 static bool check_overflow_prerequisites(bool overflow_at_64bits)
465 {
466 	if (overflow_at_64bits && pmu.version < ID_DFR0_PMU_V3_8_5) {
467 		report_skip("Skip test as 64 overflows need FEAT_PMUv3p5");
468 		return false;
469 	}
470 
471 	return true;
472 }
473 
test_basic_event_count(bool overflow_at_64bits)474 static void test_basic_event_count(bool overflow_at_64bits)
475 {
476 	uint32_t implemented_counter_mask, non_implemented_counter_mask;
477 	uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
478 	uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
479 	uint32_t events[] = {CPU_CYCLES, INST_RETIRED};
480 	uint32_t counter_mask;
481 
482 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
483 	    !check_overflow_prerequisites(overflow_at_64bits))
484 		return;
485 
486 	implemented_counter_mask = BIT(pmu.nb_implemented_counters) - 1;
487 	non_implemented_counter_mask = ~(BIT(31) | implemented_counter_mask);
488 	counter_mask = implemented_counter_mask | non_implemented_counter_mask;
489 
490 	write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
491 	write_regn_el0(pmevtyper, 1, INST_RETIRED | PMEVTYPER_EXCLUDE_EL0);
492 
493 	/* disable all counters */
494 	write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0);
495 	report(!read_sysreg_s(PMCNTENCLR_EL0) && !read_sysreg_s(PMCNTENSET_EL0),
496 		"pmcntenclr: disable all counters");
497 
498 	/*
499 	 * clear cycle and all event counters and allow counter enablement
500 	 * through PMCNTENSET. LC is RES1.
501 	 */
502 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P | pmcr_lp);
503 	isb();
504 	report(get_pmcr() == (pmu.pmcr_ro | PMU_PMCR_LC | pmcr_lp), "pmcr: reset counters");
505 
506 	/* Preset counter #0 to pre overflow value to trigger an overflow */
507 	write_regn_el0(pmevcntr, 0, pre_overflow);
508 	report(read_regn_el0(pmevcntr, 0) == pre_overflow,
509 		"counter #0 preset to pre-overflow value");
510 	report(!read_regn_el0(pmevcntr, 1), "counter #1 is 0");
511 
512 	/*
513 	 * Enable all implemented counters and also attempt to enable
514 	 * not supported counters. Counting still is disabled by !PMCR.E
515 	 */
516 	write_sysreg_s(counter_mask, PMCNTENSET_EL0);
517 
518 	/* check only those implemented are enabled */
519 	report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) &&
520 		(read_sysreg_s(PMCNTENSET_EL0) == implemented_counter_mask),
521 		"pmcntenset: enabled implemented_counters");
522 
523 	/* Disable all counters but counters #0 and #1 */
524 	write_sysreg_s(~0x3, PMCNTENCLR_EL0);
525 	report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) &&
526 		(read_sysreg_s(PMCNTENSET_EL0) == 0x3),
527 		"pmcntenset: just enabled #0 and #1");
528 
529 	/* clear overflow register */
530 	write_sysreg(ALL_SET_32, pmovsclr_el0);
531 	report(!read_sysreg(pmovsclr_el0), "check overflow reg is 0");
532 
533 	/* disable overflow interrupts on all counters*/
534 	write_sysreg(ALL_SET_32, pmintenclr_el1);
535 	report(!read_sysreg(pmintenclr_el1),
536 		"pmintenclr_el1=0, all interrupts disabled");
537 
538 	/* enable overflow interrupts on all event counters */
539 	write_sysreg(implemented_counter_mask | non_implemented_counter_mask,
540 		     pmintenset_el1);
541 	report(read_sysreg(pmintenset_el1) == implemented_counter_mask,
542 		"overflow interrupts enabled on all implemented counters");
543 
544 	/* Set PMCR.E, execute asm code and unset PMCR.E */
545 	precise_instrs_loop(20, pmu.pmcr_ro | PMU_PMCR_E);
546 
547 	report_info("counter #0 is 0x%lx (CPU_CYCLES)",
548 		    read_regn_el0(pmevcntr, 0));
549 	report_info("counter #1 is 0x%lx (INST_RETIRED)",
550 		    read_regn_el0(pmevcntr, 1));
551 
552 	report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0));
553 	report(read_sysreg(pmovsclr_el0) == 0x1,
554 		"check overflow happened on #0 only");
555 }
556 
test_mem_access(bool overflow_at_64bits)557 static void test_mem_access(bool overflow_at_64bits)
558 {
559 	void *addr = malloc(PAGE_SIZE);
560 	uint32_t events[] = {MEM_ACCESS, MEM_ACCESS};
561 	uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
562 	uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
563 
564 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
565 	    !check_overflow_prerequisites(overflow_at_64bits))
566 		return;
567 
568 	pmu_reset();
569 
570 	write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
571 	write_regn_el0(pmevtyper, 1, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
572 	write_sysreg_s(0x3, PMCNTENSET_EL0);
573 	isb();
574 	mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
575 	report_info("counter #0 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 0));
576 	report_info("counter #1 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 1));
577 	/* We may measure more than 20 mem access depending on the core */
578 	report((read_regn_el0(pmevcntr, 0) == read_regn_el0(pmevcntr, 1)) &&
579 	       (read_regn_el0(pmevcntr, 0) >= 20) && !read_sysreg(pmovsclr_el0),
580 	       "Ran 20 mem accesses");
581 
582 	pmu_reset();
583 
584 	write_regn_el0(pmevcntr, 0, pre_overflow);
585 	write_regn_el0(pmevcntr, 1, pre_overflow);
586 	write_sysreg_s(0x3, PMCNTENSET_EL0);
587 	isb();
588 	mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
589 	report(read_sysreg(pmovsclr_el0) == 0x3,
590 	       "Ran 20 mem accesses with expected overflows on both counters");
591 	report_info("cnt#0=0x%lx cnt#1=0x%lx overflow=0x%lx",
592 			read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1),
593 			read_sysreg(pmovsclr_el0));
594 }
595 
test_sw_incr(bool overflow_at_64bits)596 static void test_sw_incr(bool overflow_at_64bits)
597 {
598 	uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
599 	uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
600 	uint32_t events[] = {SW_INCR, SW_INCR};
601 	uint64_t cntr0 = (pre_overflow + 100) & pmevcntr_mask();
602 	int i;
603 
604 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
605 	    !check_overflow_prerequisites(overflow_at_64bits))
606 		return;
607 
608 	pmu_reset();
609 
610 	write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
611 	write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
612 	/* enable counters #0 and #1 */
613 	write_sysreg_s(0x3, PMCNTENSET_EL0);
614 
615 	write_regn_el0(pmevcntr, 0, pre_overflow);
616 	isb();
617 
618 	for (i = 0; i < 100; i++)
619 		write_sysreg(0x1, pmswinc_el0);
620 
621 	isb();
622 	report_info("SW_INCR counter #0 has value 0x%lx", read_regn_el0(pmevcntr, 0));
623 	report(read_regn_el0(pmevcntr, 0) == pre_overflow,
624 		"PWSYNC does not increment if PMCR.E is unset");
625 
626 	pmu_reset();
627 
628 	write_regn_el0(pmevcntr, 0, pre_overflow);
629 	write_sysreg_s(0x3, PMCNTENSET_EL0);
630 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
631 	isb();
632 
633 	for (i = 0; i < 100; i++)
634 		write_sysreg(0x3, pmswinc_el0);
635 
636 	isb();
637 	report(read_regn_el0(pmevcntr, 0) == cntr0, "counter #0 after + 100 SW_INCR");
638 	report(read_regn_el0(pmevcntr, 1) == 100, "counter #1 after + 100 SW_INCR");
639 	report_info("counter values after 100 SW_INCR #0=0x%lx #1=0x%lx",
640 		    read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1));
641 	report(read_sysreg(pmovsclr_el0) == 0x1,
642 		"overflow on counter #0 after 100 SW_INCR");
643 }
644 
enable_chain_counter(int even)645 static void enable_chain_counter(int even)
646 {
647 	write_sysreg_s(BIT(even + 1), PMCNTENSET_EL0); /* Enable the high counter first */
648 	isb();
649 	write_sysreg_s(BIT(even), PMCNTENSET_EL0); /* Enable the low counter */
650 	isb();
651 }
652 
disable_chain_counter(int even)653 static void disable_chain_counter(int even)
654 {
655 	write_sysreg_s(BIT(even), PMCNTENCLR_EL0); /* Disable the low counter first*/
656 	isb();
657 	write_sysreg_s(BIT(even + 1), PMCNTENCLR_EL0); /* Disable the high counter */
658 	isb();
659 }
660 
test_chained_counters(bool unused)661 static void test_chained_counters(bool unused)
662 {
663 	uint32_t events[] = {CPU_CYCLES, CHAIN};
664 	uint64_t all_set = pmevcntr_mask();
665 
666 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)))
667 		return;
668 
669 	pmu_reset();
670 
671 	write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
672 	write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
673 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
674 	enable_chain_counter(0);
675 
676 	precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E);
677 
678 	report(read_regn_el0(pmevcntr, 1) == 1, "CHAIN counter #1 incremented");
679 	report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #1");
680 
681 	/* test 64b overflow */
682 
683 	pmu_reset();
684 
685 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
686 	write_regn_el0(pmevcntr, 1, 0x1);
687 	enable_chain_counter(0);
688 	precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E);
689 	report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0));
690 	report(read_regn_el0(pmevcntr, 1) == 2, "CHAIN counter #1 set to 2");
691 	report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #2");
692 
693 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
694 	write_regn_el0(pmevcntr, 1, all_set);
695 
696 	precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E);
697 	report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0));
698 	report(read_regn_el0(pmevcntr, 1) == 0, "CHAIN counter #1 wrapped");
699 	report(read_sysreg(pmovsclr_el0) == 0x3, "overflow on even and odd counters");
700 }
701 
test_chained_sw_incr(bool unused)702 static void test_chained_sw_incr(bool unused)
703 {
704 	uint32_t events[] = {SW_INCR, CHAIN};
705 	uint64_t cntr0 = (PRE_OVERFLOW_32 + 100) & pmevcntr_mask();
706 	uint64_t cntr1 = (ALL_SET_32 + 1) & pmevcntr_mask();
707 	int i;
708 
709 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)))
710 		return;
711 
712 	pmu_reset();
713 
714 	write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
715 	write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
716 	enable_chain_counter(0);
717 
718 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
719 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_E);
720 	isb();
721 
722 	for (i = 0; i < 100; i++)
723 		write_sysreg(0x1, pmswinc_el0);
724 
725 	isb();
726 	report((read_sysreg(pmovsclr_el0) == 0x1) &&
727 		(read_regn_el0(pmevcntr, 1) == 1),
728 		"overflow and chain counter incremented after 100 SW_INCR/CHAIN");
729 	report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0),
730 		    read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1));
731 
732 	/* 64b SW_INCR and overflow on CHAIN counter*/
733 	pmu_reset();
734 
735 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
736 	write_regn_el0(pmevcntr, 1, ALL_SET_32);
737 	enable_chain_counter(0);
738 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_E);
739 	isb();
740 
741 	for (i = 0; i < 100; i++)
742 		write_sysreg(0x1, pmswinc_el0);
743 
744 	isb();
745 	report((read_sysreg(pmovsclr_el0) == 0x3) &&
746 	       (read_regn_el0(pmevcntr, 0) == cntr0) &&
747 	       (read_regn_el0(pmevcntr, 1) == cntr1),
748 	       "expected overflows and values after 100 SW_INCR/CHAIN");
749 	report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0),
750 		    read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1));
751 }
752 #define PRINT_REGS(__s) \
753 	report_info("%s #1=0x%lx #0=0x%lx overflow=0x%lx", __s, \
754 		    read_regn_el0(pmevcntr, 1), \
755 		    read_regn_el0(pmevcntr, 0), \
756 		    read_sysreg(pmovsclr_el0))
757 
758 /*
759  * This test checks that a mem access loop featuring COUNT accesses
760  * does not overflow with an init value of PRE_OVERFLOW2. It also
761  * records the min/max access count to see how much the counting
762  * is (un)reliable
763  */
test_mem_access_reliability(bool overflow_at_64bits)764 static void test_mem_access_reliability(bool overflow_at_64bits)
765 {
766 	uint32_t events[] = {MEM_ACCESS};
767 	void *addr = malloc(PAGE_SIZE);
768 	uint64_t cntr_val, num_events, max = 0, min = pmevcntr_mask();
769 	uint64_t pre_overflow2 = PRE_OVERFLOW2(overflow_at_64bits);
770 	uint64_t all_set = ALL_SET(overflow_at_64bits);
771 	uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
772 	bool overflow = false;
773 
774 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
775 	    !check_overflow_prerequisites(overflow_at_64bits))
776 		return;
777 
778 	pmu_reset();
779 	write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
780 	for (int i = 0; i < 100; i++) {
781 		pmu_reset();
782 		write_regn_el0(pmevcntr, 0, pre_overflow2);
783 		write_sysreg_s(0x1, PMCNTENSET_EL0);
784 		isb();
785 		mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
786 		cntr_val = read_regn_el0(pmevcntr, 0);
787 		if (cntr_val >= pre_overflow2) {
788 			num_events = cntr_val - pre_overflow2;
789 		} else {
790 			/* unexpected counter overflow */
791 			num_events = cntr_val + all_set - pre_overflow2;
792 			overflow = true;
793 			report_info("iter=%d num_events=%ld min=%ld max=%ld overflow!!!",
794 				    i, num_events, min, max);
795 		}
796 		/* record extreme value */
797 		max = MAX(num_events, max);
798 		min = MIN(num_events, min);
799 	}
800 	report_info("overflow=%d min=%ld max=%ld expected=%d acceptable margin=%d",
801 		    overflow, min, max, COUNT, MARGIN);
802 	report(!overflow, "mem_access count is reliable");
803 }
804 
test_chain_promotion(bool unused)805 static void test_chain_promotion(bool unused)
806 {
807 	uint32_t events[] = {MEM_ACCESS, CHAIN};
808 	void *addr = malloc(PAGE_SIZE);
809 
810 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)))
811 		return;
812 
813 	/* Only enable CHAIN counter */
814 	report_prefix_push("subtest1");
815 	pmu_reset();
816 	write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
817 	write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
818 	write_sysreg_s(0x2, PMCNTENSET_EL0);
819 	isb();
820 
821 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
822 	PRINT_REGS("post");
823 	report(!read_regn_el0(pmevcntr, 0),
824 		"chain counter not counting if even counter is disabled");
825 	report_prefix_pop();
826 
827 	/* Only enable even counter */
828 	report_prefix_push("subtest2");
829 	pmu_reset();
830 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32);
831 	write_sysreg_s(0x1, PMCNTENSET_EL0);
832 	isb();
833 
834 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
835 	PRINT_REGS("post");
836 	report(!read_regn_el0(pmevcntr, 1) && (read_sysreg(pmovsclr_el0) == 0x1),
837 		"odd counter did not increment on overflow if disabled");
838 	report_prefix_pop();
839 
840 	/* 1st COUNT with CHAIN enabled, next COUNT with CHAIN disabled */
841 	report_prefix_push("subtest3");
842 	pmu_reset();
843 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
844 	enable_chain_counter(0);
845 	PRINT_REGS("init");
846 
847 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
848 	PRINT_REGS("After 1st loop");
849 
850 	/* disable the CHAIN event */
851 	disable_chain_counter(0);
852 	write_sysreg_s(0x1, PMCNTENSET_EL0); /* Enable the low counter */
853 	isb();
854 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
855 	PRINT_REGS("After 2nd loop");
856 	report(read_sysreg(pmovsclr_el0) == 0x1,
857 		"should have triggered an overflow on #0");
858 	report(!read_regn_el0(pmevcntr, 1),
859 		"CHAIN counter #1 shouldn't have incremented");
860 	report_prefix_pop();
861 
862 	/* 1st COUNT with CHAIN disabled, next COUNT with CHAIN enabled */
863 
864 	report_prefix_push("subtest4");
865 	pmu_reset();
866 	write_sysreg_s(0x1, PMCNTENSET_EL0);
867 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
868 	isb();
869 	PRINT_REGS("init");
870 
871 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
872 	PRINT_REGS("After 1st loop");
873 
874 	/* Disable the low counter first and enable the chain counter */
875 	write_sysreg_s(0x1, PMCNTENCLR_EL0);
876 	isb();
877 	enable_chain_counter(0);
878 
879 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
880 
881 	PRINT_REGS("After 2nd loop");
882 
883 	report((read_regn_el0(pmevcntr, 1) == 1) &&
884 		(read_sysreg(pmovsclr_el0) == 0x1),
885 		"CHAIN counter enabled: CHAIN counter was incremented and overflow");
886 	report_prefix_pop();
887 
888 	/* start as MEM_ACCESS/CPU_CYCLES and move to CHAIN/MEM_ACCESS */
889 	report_prefix_push("subtest5");
890 	pmu_reset();
891 	write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
892 	write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
893 	write_sysreg_s(0x3, PMCNTENSET_EL0);
894 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
895 	isb();
896 	PRINT_REGS("init");
897 
898 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
899 	PRINT_REGS("After 1st loop");
900 
901 	/* 0 becomes CHAINED */
902 	write_sysreg_s(0x3, PMCNTENCLR_EL0);
903 	write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
904 	write_regn_el0(pmevcntr, 1, 0x0);
905 	enable_chain_counter(0);
906 
907 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
908 	PRINT_REGS("After 2nd loop");
909 
910 	report((read_regn_el0(pmevcntr, 1) == 1) &&
911 		(read_sysreg(pmovsclr_el0) == 0x1),
912 		"32b->64b: CHAIN counter incremented and overflow");
913 	report_prefix_pop();
914 
915 	/* start as CHAIN/MEM_ACCESS and move to MEM_ACCESS/CPU_CYCLES */
916 	report_prefix_push("subtest6");
917 	pmu_reset();
918 	write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
919 	write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
920 	write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32);
921 	enable_chain_counter(0);
922 	PRINT_REGS("init");
923 
924 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
925 	PRINT_REGS("After 1st loop");
926 
927 	disable_chain_counter(0);
928 	write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0);
929 	write_sysreg_s(0x3, PMCNTENSET_EL0);
930 
931 	mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E);
932 	PRINT_REGS("After 2nd loop");
933 	report(read_sysreg(pmovsclr_el0) == 1,
934 		"overflow is expected on counter 0");
935 	report_prefix_pop();
936 }
937 
expect_interrupts(uint32_t bitmap)938 static bool expect_interrupts(uint32_t bitmap)
939 {
940 	int i;
941 
942 	if (pmu_stats.bitmap ^ bitmap || pmu_stats.unexpected)
943 		return false;
944 
945 	for (i = 0; i < 32; i++) {
946 		if (test_and_clear_bit(i, &pmu_stats.bitmap))
947 			if (pmu_stats.interrupts[i] != 1)
948 				return false;
949 	}
950 	return true;
951 }
952 
test_overflow_interrupt(bool overflow_at_64bits)953 static void test_overflow_interrupt(bool overflow_at_64bits)
954 {
955 	uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits);
956 	uint64_t all_set = pmevcntr_mask();
957 	uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0;
958 	uint32_t events[] = {MEM_ACCESS, SW_INCR};
959 	void *addr = malloc(PAGE_SIZE);
960 	int i;
961 
962 	if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) ||
963 	    !check_overflow_prerequisites(overflow_at_64bits))
964 		return;
965 
966 	gic_enable_defaults();
967 	install_irq_handler(EL1H_IRQ, irq_handler);
968 	local_irq_enable();
969 	gic_enable_irq(23);
970 
971 	pmu_reset();
972 
973 	write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0);
974 	write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0);
975 	write_sysreg_s(0x3, PMCNTENSET_EL0);
976 	write_regn_el0(pmevcntr, 0, pre_overflow);
977 	write_regn_el0(pmevcntr, 1, pre_overflow);
978 	isb();
979 
980 	/* interrupts are disabled (PMINTENSET_EL1 == 0) */
981 
982 	mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
983 	report(expect_interrupts(0), "no overflow interrupt after preset");
984 
985 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
986 	isb();
987 
988 	for (i = 0; i < 100; i++)
989 		write_sysreg(0x2, pmswinc_el0);
990 
991 	isb();
992 	set_pmcr(pmu.pmcr_ro);
993 	isb();
994 	report(expect_interrupts(0), "no overflow interrupt after counting");
995 
996 	/* enable interrupts (PMINTENSET_EL1 <= ALL_SET_32) */
997 
998 	pmu_reset_stats();
999 
1000 	write_regn_el0(pmevcntr, 0, pre_overflow);
1001 	write_regn_el0(pmevcntr, 1, pre_overflow);
1002 	write_sysreg(ALL_SET_32, pmovsclr_el0);
1003 	write_sysreg(ALL_SET_32, pmintenset_el1);
1004 	isb();
1005 
1006 	mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1007 
1008 	set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1009 	isb();
1010 
1011 	for (i = 0; i < 100; i++)
1012 		write_sysreg(0x3, pmswinc_el0);
1013 
1014 	mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro);
1015 	report_info("overflow=0x%lx", read_sysreg(pmovsclr_el0));
1016 	report(expect_interrupts(0x3),
1017 		"overflow interrupts expected on #0 and #1");
1018 
1019 	/*
1020 	 * promote to 64-b:
1021 	 *
1022 	 * This only applies to the !overflow_at_64bits case, as
1023 	 * overflow_at_64bits doesn't implement CHAIN events. The
1024 	 * overflow_at_64bits case just checks that chained counters are
1025 	 * not incremented when PMCR.LP == 1.
1026 	 */
1027 
1028 	pmu_reset_stats();
1029 
1030 	write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0);
1031 	write_regn_el0(pmevcntr, 0, pre_overflow);
1032 	isb();
1033 	mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1034 	report(expect_interrupts(0x1), "expect overflow interrupt");
1035 
1036 	/* overflow on odd counter */
1037 	pmu_reset_stats();
1038 	write_regn_el0(pmevcntr, 0, pre_overflow);
1039 	write_regn_el0(pmevcntr, 1, all_set);
1040 	isb();
1041 	mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp);
1042 	if (overflow_at_64bits) {
1043 		report(expect_interrupts(0x1),
1044 		       "expect overflow interrupt on even counter");
1045 		report(read_regn_el0(pmevcntr, 1) == all_set,
1046 		       "Odd counter did not change");
1047 	} else {
1048 		report(expect_interrupts(0x3),
1049 		       "expect overflow interrupt on even and odd counter");
1050 		report(read_regn_el0(pmevcntr, 1) != all_set,
1051 		       "Odd counter wrapped");
1052 	}
1053 }
1054 #endif
1055 
1056 /*
1057  * Ensure that the cycle counter progresses between back-to-back reads.
1058  */
check_cycles_increase(void)1059 static bool check_cycles_increase(void)
1060 {
1061 	bool success = true;
1062 
1063 	/* init before event access, this test only cares about cycle count */
1064 	pmu_reset();
1065 	set_pmcntenset(1 << PMU_CYCLE_IDX);
1066 	set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */
1067 
1068 	set_pmcr(get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E);
1069 	isb();
1070 
1071 	for (int i = 0; i < NR_SAMPLES; i++) {
1072 		uint64_t a, b;
1073 
1074 		a = get_pmccntr();
1075 		b = get_pmccntr();
1076 
1077 		if (a >= b) {
1078 			printf("Read %"PRId64" then %"PRId64".\n", a, b);
1079 			success = false;
1080 			break;
1081 		}
1082 	}
1083 
1084 	set_pmcr(get_pmcr() & ~PMU_PMCR_E);
1085 	isb();
1086 
1087 	return success;
1088 }
1089 
1090 /*
1091  * Execute a known number of guest instructions. Only even instruction counts
1092  * greater than or equal to 4 are supported by the in-line assembly code. The
1093  * control register (PMCR_EL0) is initialized with the provided value (allowing
1094  * for example for the cycle counter or event counters to be reset). At the end
1095  * of the exact instruction loop, zero is written to PMCR_EL0 to disable
1096  * counting, allowing the cycle counter or event counters to be read at the
1097  * leisure of the calling code.
1098  */
measure_instrs(int num,uint32_t pmcr)1099 static void measure_instrs(int num, uint32_t pmcr)
1100 {
1101 	int loop = (num - 2) / 2;
1102 
1103 	assert(num >= 4 && ((num - 2) % 2 == 0));
1104 	precise_instrs_loop(loop, pmcr);
1105 }
1106 
1107 /*
1108  * Measure cycle counts for various known instruction counts. Ensure that the
1109  * cycle counter progresses (similar to check_cycles_increase() but with more
1110  * instructions and using reset and stop controls). If supplied a positive,
1111  * nonzero CPI parameter, it also strictly checks that every measurement matches
1112  * it. Strict CPI checking is used to test -icount mode.
1113  */
check_cpi(int cpi)1114 static bool check_cpi(int cpi)
1115 {
1116 	uint32_t pmcr = get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E;
1117 
1118 	/* init before event access, this test only cares about cycle count */
1119 	pmu_reset();
1120 	set_pmcntenset(1 << PMU_CYCLE_IDX);
1121 	set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */
1122 
1123 	if (cpi > 0)
1124 		printf("Checking for CPI=%d.\n", cpi);
1125 	printf("instrs : cycles0 cycles1 ...\n");
1126 
1127 	for (unsigned int i = 4; i < 300; i += 32) {
1128 		uint64_t avg, sum = 0;
1129 
1130 		printf("%4d:", i);
1131 		for (int j = 0; j < NR_SAMPLES; j++) {
1132 			uint64_t cycles;
1133 
1134 			set_pmccntr(0);
1135 			measure_instrs(i, pmcr);
1136 			cycles = get_pmccntr();
1137 			printf(" %4"PRId64"", cycles);
1138 
1139 			if (!cycles) {
1140 				printf("\ncycles not incrementing!\n");
1141 				return false;
1142 			} else if (cpi > 0 && cycles != i * cpi) {
1143 				printf("\nunexpected cycle count received!\n");
1144 				return false;
1145 			} else if ((cycles >> 32) != 0) {
1146 				/* The cycles taken by the loop above should
1147 				 * fit in 32 bits easily. We check the upper
1148 				 * 32 bits of the cycle counter to make sure
1149 				 * there is no supprise. */
1150 				printf("\ncycle count bigger than 32bit!\n");
1151 				return false;
1152 			}
1153 
1154 			sum += cycles;
1155 		}
1156 		avg = sum / NR_SAMPLES;
1157 		printf(" avg=%-4"PRId64" %s=%-3"PRId64"\n", avg,
1158 		       (avg >= i) ? "cpi" : "ipc",
1159 		       (avg >= i) ? avg / i : i / avg);
1160 	}
1161 
1162 	return true;
1163 }
1164 
pmccntr64_test(void)1165 static void pmccntr64_test(void)
1166 {
1167 #ifdef __arm__
1168 	if (pmu.version == ID_DFR0_PMU_V3) {
1169 		if (ERRATA(9e3f7a296940)) {
1170 			write_sysreg(0xdead, PMCCNTR64);
1171 			report(read_sysreg(PMCCNTR64) == 0xdead, "pmccntr64");
1172 		} else
1173 			report_skip("Skipping unsafe pmccntr64 test. Set ERRATA_9e3f7a296940=y to enable.");
1174 	}
1175 #endif
1176 }
1177 
1178 /* Return FALSE if no PMU found, otherwise return TRUE */
pmu_probe(void)1179 static bool pmu_probe(void)
1180 {
1181 	uint32_t pmcr;
1182 	uint8_t implementer;
1183 
1184 	pmu.version = get_pmu_version();
1185 	if (pmu.version == ID_DFR0_PMU_NOTIMPL || pmu.version == ID_DFR0_PMU_IMPDEF)
1186 		return false;
1187 
1188 	report_info("PMU version: 0x%x", pmu.version);
1189 
1190 	pmcr = get_pmcr();
1191 	implementer = (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK;
1192 	report_info("PMU implementer/ID code: %#"PRIx32"(\"%c\")/%#"PRIx32,
1193 		    (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK,
1194 		    implementer ? implementer : ' ',
1195 		    (pmcr >> PMU_PMCR_ID_SHIFT) & PMU_PMCR_ID_MASK);
1196 
1197 	/* store read-only and RES0 fields of the PMCR bottom-half*/
1198 	pmu.pmcr_ro = pmcr & 0xFFFFFF00;
1199 	pmu.nb_implemented_counters =
1200 		(pmcr >> PMU_PMCR_N_SHIFT) & PMU_PMCR_N_MASK;
1201 	report_info("Implements %d event counters",
1202 		    pmu.nb_implemented_counters);
1203 
1204 	return true;
1205 }
1206 
run_test(const char * name,const char * prefix,void (* test)(bool),void * arg)1207 static void run_test(const char *name, const char *prefix,
1208 		     void (*test)(bool), void *arg)
1209 {
1210 	report_prefix_push(name);
1211 	report_prefix_push(prefix);
1212 
1213 	test(arg);
1214 
1215 	report_prefix_pop();
1216 	report_prefix_pop();
1217 }
1218 
run_event_test(char * name,void (* test)(bool),bool overflow_at_64bits)1219 static void run_event_test(char *name, void (*test)(bool),
1220 			   bool overflow_at_64bits)
1221 {
1222 	const char *prefix = overflow_at_64bits ? "64-bit overflows"
1223 						: "32-bit overflows";
1224 
1225 	run_test(name, prefix, test, (void *)overflow_at_64bits);
1226 }
1227 
main(int argc,char * argv[])1228 int main(int argc, char *argv[])
1229 {
1230 	int cpi = 0;
1231 
1232 	if (!pmu_probe()) {
1233 		printf("No PMU found, test skipped...\n");
1234 		return report_summary();
1235 	}
1236 
1237 	if (argc < 2)
1238 		report_abort("no test specified");
1239 
1240 	report_prefix_push("pmu");
1241 
1242 	if (strcmp(argv[1], "cycle-counter") == 0) {
1243 		report_prefix_push(argv[1]);
1244 		if (argc > 2)
1245 			cpi = atol(argv[2]);
1246 		report(check_cycles_increase(),
1247 		       "Monotonically increasing cycle count");
1248 		report(check_cpi(cpi), "Cycle/instruction ratio");
1249 		pmccntr64_test();
1250 		report_prefix_pop();
1251 	} else if (strcmp(argv[1], "pmu-event-introspection") == 0) {
1252 		report_prefix_push(argv[1]);
1253 		test_event_introspection();
1254 		report_prefix_pop();
1255 	} else if (strcmp(argv[1], "pmu-event-counter-config") == 0) {
1256 		report_prefix_push(argv[1]);
1257 		test_event_counter_config();
1258 		report_prefix_pop();
1259 	} else if (strcmp(argv[1], "pmu-basic-event-count") == 0) {
1260 		run_event_test(argv[1], test_basic_event_count, false);
1261 		run_event_test(argv[1], test_basic_event_count, true);
1262 	} else if (strcmp(argv[1], "pmu-mem-access-reliability") == 0) {
1263 		run_event_test(argv[1], test_mem_access_reliability, false);
1264 		run_event_test(argv[1], test_mem_access_reliability, true);
1265 	} else if (strcmp(argv[1], "pmu-mem-access") == 0) {
1266 		run_event_test(argv[1], test_mem_access, false);
1267 		run_event_test(argv[1], test_mem_access, true);
1268 	} else if (strcmp(argv[1], "pmu-sw-incr") == 0) {
1269 		run_event_test(argv[1], test_sw_incr, false);
1270 		run_event_test(argv[1], test_sw_incr, true);
1271 	} else if (strcmp(argv[1], "pmu-chained-counters") == 0) {
1272 		run_event_test(argv[1], test_chained_counters, false);
1273 	} else if (strcmp(argv[1], "pmu-chained-sw-incr") == 0) {
1274 		run_event_test(argv[1], test_chained_sw_incr, false);
1275 	} else if (strcmp(argv[1], "pmu-chain-promotion") == 0) {
1276 		run_event_test(argv[1], test_chain_promotion, false);
1277 	} else if (strcmp(argv[1], "pmu-overflow-interrupt") == 0) {
1278 		run_event_test(argv[1], test_overflow_interrupt, false);
1279 		run_event_test(argv[1], test_overflow_interrupt, true);
1280 	} else {
1281 		report_abort("Unknown sub-test '%s'", argv[1]);
1282 	}
1283 
1284 	return report_summary();
1285 }
1286