1 /* 2 * Test the ARM Performance Monitors Unit (PMU). 3 * 4 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. 5 * Copyright (C) 2016, Red Hat Inc, Wei Huang <wei@redhat.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU Lesser General Public License version 2.1 and 9 * only version 2.1 as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License 14 * for more details. 15 */ 16 #include "libcflat.h" 17 #include "errata.h" 18 #include "asm/barrier.h" 19 #include "asm/sysreg.h" 20 #include "asm/processor.h" 21 #include <bitops.h> 22 #include <asm/gic.h> 23 24 #define PMU_PMCR_E (1 << 0) 25 #define PMU_PMCR_P (1 << 1) 26 #define PMU_PMCR_C (1 << 2) 27 #define PMU_PMCR_D (1 << 3) 28 #define PMU_PMCR_X (1 << 4) 29 #define PMU_PMCR_DP (1 << 5) 30 #define PMU_PMCR_LC (1 << 6) 31 #define PMU_PMCR_LP (1 << 7) 32 #define PMU_PMCR_N_SHIFT 11 33 #define PMU_PMCR_N_MASK 0x1f 34 #define PMU_PMCR_ID_SHIFT 16 35 #define PMU_PMCR_ID_MASK 0xff 36 #define PMU_PMCR_IMP_SHIFT 24 37 #define PMU_PMCR_IMP_MASK 0xff 38 39 #define PMU_CYCLE_IDX 31 40 41 #define NR_SAMPLES 10 42 43 /* Some PMU events */ 44 #define SW_INCR 0x0 45 #define INST_RETIRED 0x8 46 #define CPU_CYCLES 0x11 47 #define MEM_ACCESS 0x13 48 #define INST_PREC 0x1B 49 #define STALL_FRONTEND 0x23 50 #define STALL_BACKEND 0x24 51 #define CHAIN 0x1E 52 53 #define COMMON_EVENTS_LOW 0x0 54 #define COMMON_EVENTS_HIGH 0x3F 55 #define EXT_COMMON_EVENTS_LOW 0x4000 56 #define EXT_COMMON_EVENTS_HIGH 0x403F 57 58 #define ALL_SET_32 0x00000000FFFFFFFFULL 59 #define ALL_SET_64 0xFFFFFFFFFFFFFFFFULL 60 61 #define ALL_SET(__overflow_at_64bits) \ 62 (__overflow_at_64bits ? ALL_SET_64 : ALL_SET_32) 63 64 #define ALL_CLEAR 0x0000000000000000ULL 65 #define PRE_OVERFLOW_32 0x00000000FFFFFFF0ULL 66 #define PRE_OVERFLOW_64 0xFFFFFFFFFFFFFFF0ULL 67 #define COUNT 250 68 #define MARGIN 100 69 #define COUNT_INT 1000 70 /* 71 * PRE_OVERFLOW2 is set so that 1st @COUNT iterations do not 72 * produce 32b overflow and 2nd @COUNT iterations do. To accommodate 73 * for some observed variability we take into account a given @MARGIN 74 */ 75 #define PRE_OVERFLOW2_32 (ALL_SET_32 - COUNT - MARGIN) 76 #define PRE_OVERFLOW2_64 (ALL_SET_64 - COUNT - MARGIN) 77 78 #define PRE_OVERFLOW2(__overflow_at_64bits) \ 79 (__overflow_at_64bits ? PRE_OVERFLOW2_64 : PRE_OVERFLOW2_32) 80 81 #define PRE_OVERFLOW(__overflow_at_64bits) \ 82 (__overflow_at_64bits ? PRE_OVERFLOW_64 : PRE_OVERFLOW_32) 83 84 #define PMU_PPI 23 85 86 struct pmu { 87 unsigned int version; 88 unsigned int nb_implemented_counters; 89 uint32_t pmcr_ro; 90 }; 91 92 struct pmu_stats { 93 unsigned long bitmap; 94 uint32_t interrupts[32]; 95 bool unexpected; 96 }; 97 98 static struct pmu pmu; 99 100 #if defined(__arm__) 101 #define ID_DFR0_PERFMON_SHIFT 24 102 #define ID_DFR0_PERFMON_MASK 0xf 103 104 #define ID_DFR0_PMU_NOTIMPL 0b0000 105 #define ID_DFR0_PMU_V1 0b0001 106 #define ID_DFR0_PMU_V2 0b0010 107 #define ID_DFR0_PMU_V3 0b0011 108 #define ID_DFR0_PMU_V3_8_1 0b0100 109 #define ID_DFR0_PMU_V3_8_4 0b0101 110 #define ID_DFR0_PMU_V3_8_5 0b0110 111 #define ID_DFR0_PMU_IMPDEF 0b1111 112 113 #define PMCR __ACCESS_CP15(c9, 0, c12, 0) 114 #define ID_DFR0 __ACCESS_CP15(c0, 0, c1, 2) 115 #define PMSELR __ACCESS_CP15(c9, 0, c12, 5) 116 #define PMXEVTYPER __ACCESS_CP15(c9, 0, c13, 1) 117 #define PMCNTENSET __ACCESS_CP15(c9, 0, c12, 1) 118 #define PMCNTENCLR __ACCESS_CP15(c9, 0, c12, 2) 119 #define PMOVSR __ACCESS_CP15(c9, 0, c12, 3) 120 #define PMCCNTR32 __ACCESS_CP15(c9, 0, c13, 0) 121 #define PMINTENCLR __ACCESS_CP15(c9, 0, c14, 2) 122 #define PMCCNTR64 __ACCESS_CP15_64(0, c9) 123 124 static inline uint32_t get_id_dfr0(void) { return read_sysreg(ID_DFR0); } 125 static inline uint32_t get_pmcr(void) { return read_sysreg(PMCR); } 126 static inline void set_pmcr(uint32_t v) { write_sysreg(v, PMCR); } 127 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, PMCNTENSET); } 128 129 static inline uint8_t get_pmu_version(void) 130 { 131 return (get_id_dfr0() >> ID_DFR0_PERFMON_SHIFT) & ID_DFR0_PERFMON_MASK; 132 } 133 134 static inline uint64_t get_pmccntr(void) 135 { 136 return read_sysreg(PMCCNTR32); 137 } 138 139 static inline void set_pmccntr(uint64_t value) 140 { 141 write_sysreg(value & 0xffffffff, PMCCNTR32); 142 } 143 144 /* PMCCFILTR is an obsolete name for PMXEVTYPER31 in ARMv7 */ 145 static inline void set_pmccfiltr(uint32_t value) 146 { 147 write_sysreg(PMU_CYCLE_IDX, PMSELR); 148 write_sysreg(value, PMXEVTYPER); 149 isb(); 150 } 151 152 /* 153 * Extra instructions inserted by the compiler would be difficult to compensate 154 * for, so hand assemble everything between, and including, the PMCR accesses 155 * to start and stop counting. isb instructions were inserted to make sure 156 * pmccntr read after this function returns the exact instructions executed in 157 * the controlled block. Total instrs = isb + mcr + 2*loop = 2 + 2*loop. 158 */ 159 static inline void precise_instrs_loop(int loop, uint32_t pmcr) 160 { 161 asm volatile( 162 " mcr p15, 0, %[pmcr], c9, c12, 0\n" 163 " isb\n" 164 "1: subs %[loop], %[loop], #1\n" 165 " bgt 1b\n" 166 " mcr p15, 0, %[z], c9, c12, 0\n" 167 " isb\n" 168 : [loop] "+r" (loop) 169 : [pmcr] "r" (pmcr), [z] "r" (0) 170 : "cc"); 171 } 172 173 static void pmu_reset(void) 174 { 175 /* reset all counters, counting disabled at PMCR level*/ 176 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P); 177 /* Disable all counters */ 178 write_sysreg(ALL_SET_32, PMCNTENCLR); 179 /* clear overflow reg */ 180 write_sysreg(ALL_SET_32, PMOVSR); 181 /* disable overflow interrupts on all counters */ 182 write_sysreg(ALL_SET_32, PMINTENCLR); 183 isb(); 184 } 185 186 /* event counter tests only implemented for aarch64 */ 187 static void test_event_introspection(void) {} 188 static void test_event_counter_config(void) {} 189 static void test_basic_event_count(bool overflow_at_64bits) {} 190 static void test_mem_access_reliability(bool overflow_at_64bits) {} 191 static void test_mem_access(bool overflow_at_64bits) {} 192 static void test_sw_incr(bool overflow_at_64bits) {} 193 static void test_chained_counters(bool unused) {} 194 static void test_chained_sw_incr(bool unused) {} 195 static void test_chain_promotion(bool unused) {} 196 static void test_overflow_interrupt(bool overflow_at_64bits) {} 197 198 #elif defined(__aarch64__) 199 #define ID_AA64DFR0_PERFMON_SHIFT 8 200 #define ID_AA64DFR0_PERFMON_MASK 0xf 201 202 #define ID_DFR0_PMU_NOTIMPL 0b0000 203 #define ID_DFR0_PMU_V3 0b0001 204 #define ID_DFR0_PMU_V3_8_1 0b0100 205 #define ID_DFR0_PMU_V3_8_4 0b0101 206 #define ID_DFR0_PMU_V3_8_5 0b0110 207 #define ID_DFR0_PMU_IMPDEF 0b1111 208 209 static inline uint32_t get_id_aa64dfr0(void) { return read_sysreg(id_aa64dfr0_el1); } 210 static inline uint32_t get_pmcr(void) { return read_sysreg(pmcr_el0); } 211 static inline void set_pmcr(uint32_t v) { write_sysreg(v, pmcr_el0); } 212 static inline uint64_t get_pmccntr(void) { return read_sysreg(pmccntr_el0); } 213 static inline void set_pmccntr(uint64_t v) { write_sysreg(v, pmccntr_el0); } 214 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, pmcntenset_el0); } 215 static inline void set_pmccfiltr(uint32_t v) { write_sysreg(v, pmccfiltr_el0); } 216 217 static inline uint8_t get_pmu_version(void) 218 { 219 uint8_t ver = (get_id_aa64dfr0() >> ID_AA64DFR0_PERFMON_SHIFT) & ID_AA64DFR0_PERFMON_MASK; 220 return ver; 221 } 222 223 /* 224 * Extra instructions inserted by the compiler would be difficult to compensate 225 * for, so hand assemble everything between, and including, the PMCR accesses 226 * to start and stop counting. isb instructions are inserted to make sure 227 * pmccntr read after this function returns the exact instructions executed 228 * in the controlled block. Total instrs = isb + msr + 2*loop = 2 + 2*loop. 229 */ 230 static inline void precise_instrs_loop(int loop, uint32_t pmcr) 231 { 232 uint64_t pmcr64 = pmcr; 233 asm volatile( 234 " msr pmcr_el0, %[pmcr]\n" 235 " isb\n" 236 "1: subs %w[loop], %w[loop], #1\n" 237 " b.gt 1b\n" 238 " msr pmcr_el0, xzr\n" 239 " isb\n" 240 : [loop] "+r" (loop) 241 : [pmcr] "r" (pmcr64) 242 : "cc"); 243 } 244 245 #define PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7) 246 #define PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1) 247 #define PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2) 248 249 #define PMEVTYPER_EXCLUDE_EL1 BIT(31) 250 #define PMEVTYPER_EXCLUDE_EL0 BIT(30) 251 252 static bool is_event_supported(uint32_t n, bool warn) 253 { 254 uint64_t pmceid0 = read_sysreg(pmceid0_el0); 255 uint64_t pmceid1 = read_sysreg_s(PMCEID1_EL0); 256 bool supported; 257 uint64_t reg; 258 259 /* 260 * The low 32-bits of PMCEID0/1 respectively describe 261 * event support for events 0-31/32-63. Their High 262 * 32-bits describe support for extended events 263 * starting at 0x4000, using the same split. 264 */ 265 assert((n >= COMMON_EVENTS_LOW && n <= COMMON_EVENTS_HIGH) || 266 (n >= EXT_COMMON_EVENTS_LOW && n <= EXT_COMMON_EVENTS_HIGH)); 267 268 if (n <= COMMON_EVENTS_HIGH) 269 reg = lower_32_bits(pmceid0) | ((u64)lower_32_bits(pmceid1) << 32); 270 else 271 reg = upper_32_bits(pmceid0) | ((u64)upper_32_bits(pmceid1) << 32); 272 273 supported = reg & (1UL << (n & 0x3F)); 274 275 if (!supported && warn) 276 report_info("event 0x%x is not supported", n); 277 return supported; 278 } 279 280 static void test_event_introspection(void) 281 { 282 bool required_events; 283 284 if (!pmu.nb_implemented_counters) { 285 report_skip("No event counter, skip ..."); 286 return; 287 } 288 289 /* PMUv3 requires an implementation includes some common events */ 290 required_events = is_event_supported(SW_INCR, true) && 291 is_event_supported(CPU_CYCLES, true) && 292 (is_event_supported(INST_RETIRED, true) || 293 is_event_supported(INST_PREC, true)); 294 295 if (pmu.version >= ID_DFR0_PMU_V3_8_1) { 296 required_events = required_events && 297 is_event_supported(STALL_FRONTEND, true) && 298 is_event_supported(STALL_BACKEND, true); 299 } 300 301 report(required_events, "Check required events are implemented"); 302 } 303 304 /* 305 * Extra instructions inserted by the compiler would be difficult to compensate 306 * for, so hand assemble everything between, and including, the PMCR accesses 307 * to start and stop counting. isb instructions are inserted to make sure 308 * pmccntr read after this function returns the exact instructions executed 309 * in the controlled block. Loads @loop times the data at @address into x9. 310 */ 311 static void mem_access_loop(void *addr, long loop, uint32_t pmcr) 312 { 313 uint64_t pmcr64 = pmcr; 314 asm volatile( 315 " dsb ish\n" 316 " msr pmcr_el0, %[pmcr]\n" 317 " isb\n" 318 " dsb ish\n" 319 " mov x10, %[loop]\n" 320 "1: sub x10, x10, #1\n" 321 " ldr x9, [%[addr]]\n" 322 " cmp x10, #0x0\n" 323 " b.gt 1b\n" 324 " dsb ish\n" 325 " msr pmcr_el0, xzr\n" 326 " isb\n" 327 : 328 : [addr] "r" (addr), [pmcr] "r" (pmcr64), [loop] "r" (loop) 329 : "x9", "x10", "cc"); 330 } 331 332 static volatile struct pmu_stats pmu_stats; 333 334 static void irq_handler(struct pt_regs *regs) 335 { 336 uint32_t irqstat, irqnr; 337 338 irqstat = gic_read_iar(); 339 irqnr = gic_iar_irqnr(irqstat); 340 341 if (irqnr == PMU_PPI) { 342 unsigned long overflows = read_sysreg(pmovsclr_el0); 343 int i; 344 345 for (i = 0; i < 32; i++) { 346 if (test_and_clear_bit(i, &overflows)) { 347 pmu_stats.interrupts[i]++; 348 pmu_stats.bitmap |= 1 << i; 349 } 350 } 351 write_sysreg(ALL_SET_32, pmovsclr_el0); 352 isb(); 353 } else { 354 pmu_stats.unexpected = true; 355 } 356 gic_write_eoir(irqstat); 357 } 358 359 static void pmu_reset_stats(void) 360 { 361 int i; 362 363 for (i = 0; i < 32; i++) 364 pmu_stats.interrupts[i] = 0; 365 366 pmu_stats.bitmap = 0; 367 pmu_stats.unexpected = false; 368 } 369 370 static void pmu_reset(void) 371 { 372 /* reset all counters, counting disabled at PMCR level*/ 373 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P); 374 /* Disable all counters */ 375 write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0); 376 /* clear overflow reg */ 377 write_sysreg(ALL_SET_32, pmovsclr_el0); 378 /* disable overflow interrupts on all counters */ 379 write_sysreg(ALL_SET_32, pmintenclr_el1); 380 pmu_reset_stats(); 381 isb(); 382 } 383 384 static void test_event_counter_config(void) 385 { 386 int i; 387 388 if (!pmu.nb_implemented_counters) { 389 report_skip("No event counter, skip ..."); 390 return; 391 } 392 393 pmu_reset(); 394 395 /* 396 * Test setting through PMESELR/PMXEVTYPER and PMEVTYPERn read, 397 * select counter 0 398 */ 399 write_sysreg(1, PMSELR_EL0); 400 /* program this counter to count unsupported event */ 401 write_sysreg(0xEA, PMXEVTYPER_EL0); 402 write_sysreg(0xdeadbeef, PMXEVCNTR_EL0); 403 report((read_regn_el0(pmevtyper, 1) & 0xFFF) == 0xEA, 404 "PMESELR/PMXEVTYPER/PMEVTYPERn"); 405 report((read_regn_el0(pmevcntr, 1) == 0xdeadbeef), 406 "PMESELR/PMXEVCNTR/PMEVCNTRn"); 407 408 /* try to configure an unsupported event within the range [0x0, 0x3F] */ 409 for (i = 0; i <= 0x3F; i++) { 410 if (!is_event_supported(i, false)) 411 break; 412 } 413 if (i > 0x3F) { 414 report_skip("pmevtyper: all events within [0x0, 0x3F] are supported"); 415 return; 416 } 417 418 /* select counter 0 */ 419 write_sysreg(0, PMSELR_EL0); 420 /* program this counter to count unsupported event */ 421 write_sysreg(i, PMXEVCNTR_EL0); 422 /* read the counter value */ 423 read_sysreg(PMXEVCNTR_EL0); 424 report(read_sysreg(PMXEVCNTR_EL0) == i, 425 "read of a counter programmed with unsupported event"); 426 } 427 428 static bool satisfy_prerequisites(uint32_t *events, unsigned int nb_events) 429 { 430 int i; 431 432 if (pmu.nb_implemented_counters < nb_events) { 433 report_skip("Skip test as number of counters is too small (%d)", 434 pmu.nb_implemented_counters); 435 return false; 436 } 437 438 for (i = 0; i < nb_events; i++) { 439 if (!is_event_supported(events[i], false)) { 440 report_skip("Skip test as event 0x%x is not supported", 441 events[i]); 442 return false; 443 } 444 } 445 return true; 446 } 447 448 static uint64_t pmevcntr_mask(void) 449 { 450 /* 451 * Bits [63:0] are always incremented for 64-bit counters, 452 * even if the PMU is configured to generate an overflow at 453 * bits [31:0] 454 * 455 * For more details see the AArch64.IncrementEventCounter() 456 * pseudo-code in the ARM ARM DDI 0487I.a, section J1.1.1. 457 */ 458 if (pmu.version >= ID_DFR0_PMU_V3_8_5) 459 return ~0; 460 461 return (uint32_t)~0; 462 } 463 464 static bool check_overflow_prerequisites(bool overflow_at_64bits) 465 { 466 if (overflow_at_64bits && pmu.version < ID_DFR0_PMU_V3_8_5) { 467 report_skip("Skip test as 64 overflows need FEAT_PMUv3p5"); 468 return false; 469 } 470 471 return true; 472 } 473 474 static void test_basic_event_count(bool overflow_at_64bits) 475 { 476 uint32_t implemented_counter_mask, non_implemented_counter_mask; 477 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 478 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 479 uint32_t events[] = {CPU_CYCLES, INST_RETIRED}; 480 uint32_t counter_mask; 481 482 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 483 !check_overflow_prerequisites(overflow_at_64bits)) 484 return; 485 486 implemented_counter_mask = BIT(pmu.nb_implemented_counters) - 1; 487 non_implemented_counter_mask = ~(BIT(31) | implemented_counter_mask); 488 counter_mask = implemented_counter_mask | non_implemented_counter_mask; 489 490 write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 491 write_regn_el0(pmevtyper, 1, INST_RETIRED | PMEVTYPER_EXCLUDE_EL0); 492 493 /* disable all counters */ 494 write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0); 495 report(!read_sysreg_s(PMCNTENCLR_EL0) && !read_sysreg_s(PMCNTENSET_EL0), 496 "pmcntenclr: disable all counters"); 497 498 /* 499 * clear cycle and all event counters and allow counter enablement 500 * through PMCNTENSET. LC is RES1. 501 */ 502 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P | pmcr_lp); 503 isb(); 504 report(get_pmcr() == (pmu.pmcr_ro | PMU_PMCR_LC | pmcr_lp), "pmcr: reset counters"); 505 506 /* Preset counter #0 to pre overflow value to trigger an overflow */ 507 write_regn_el0(pmevcntr, 0, pre_overflow); 508 report(read_regn_el0(pmevcntr, 0) == pre_overflow, 509 "counter #0 preset to pre-overflow value"); 510 report(!read_regn_el0(pmevcntr, 1), "counter #1 is 0"); 511 512 /* 513 * Enable all implemented counters and also attempt to enable 514 * not supported counters. Counting still is disabled by !PMCR.E 515 */ 516 write_sysreg_s(counter_mask, PMCNTENSET_EL0); 517 518 /* check only those implemented are enabled */ 519 report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) && 520 (read_sysreg_s(PMCNTENSET_EL0) == implemented_counter_mask), 521 "pmcntenset: enabled implemented_counters"); 522 523 /* Disable all counters but counters #0 and #1 */ 524 write_sysreg_s(~0x3, PMCNTENCLR_EL0); 525 report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) && 526 (read_sysreg_s(PMCNTENSET_EL0) == 0x3), 527 "pmcntenset: just enabled #0 and #1"); 528 529 /* clear overflow register */ 530 write_sysreg(ALL_SET_32, pmovsclr_el0); 531 report(!read_sysreg(pmovsclr_el0), "check overflow reg is 0"); 532 533 /* disable overflow interrupts on all counters*/ 534 write_sysreg(ALL_SET_32, pmintenclr_el1); 535 report(!read_sysreg(pmintenclr_el1), 536 "pmintenclr_el1=0, all interrupts disabled"); 537 538 /* enable overflow interrupts on all event counters */ 539 write_sysreg(implemented_counter_mask | non_implemented_counter_mask, 540 pmintenset_el1); 541 report(read_sysreg(pmintenset_el1) == implemented_counter_mask, 542 "overflow interrupts enabled on all implemented counters"); 543 544 /* Set PMCR.E, execute asm code and unset PMCR.E */ 545 precise_instrs_loop(20, pmu.pmcr_ro | PMU_PMCR_E); 546 547 report_info("counter #0 is 0x%lx (CPU_CYCLES)", 548 read_regn_el0(pmevcntr, 0)); 549 report_info("counter #1 is 0x%lx (INST_RETIRED)", 550 read_regn_el0(pmevcntr, 1)); 551 552 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0)); 553 report(read_sysreg(pmovsclr_el0) == 0x1, 554 "check overflow happened on #0 only"); 555 } 556 557 static void test_mem_access(bool overflow_at_64bits) 558 { 559 void *addr = malloc(PAGE_SIZE); 560 uint32_t events[] = {MEM_ACCESS, MEM_ACCESS}; 561 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 562 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 563 564 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 565 !check_overflow_prerequisites(overflow_at_64bits)) 566 return; 567 568 pmu_reset(); 569 570 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 571 write_regn_el0(pmevtyper, 1, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 572 write_sysreg_s(0x3, PMCNTENSET_EL0); 573 isb(); 574 mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 575 report_info("counter #0 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 0)); 576 report_info("counter #1 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 1)); 577 /* We may measure more than 20 mem access depending on the core */ 578 report((read_regn_el0(pmevcntr, 0) == read_regn_el0(pmevcntr, 1)) && 579 (read_regn_el0(pmevcntr, 0) >= 20) && !read_sysreg(pmovsclr_el0), 580 "Ran 20 mem accesses"); 581 582 pmu_reset(); 583 584 write_regn_el0(pmevcntr, 0, pre_overflow); 585 write_regn_el0(pmevcntr, 1, pre_overflow); 586 write_sysreg_s(0x3, PMCNTENSET_EL0); 587 isb(); 588 mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 589 report(read_sysreg(pmovsclr_el0) == 0x3, 590 "Ran 20 mem accesses with expected overflows on both counters"); 591 report_info("cnt#0=0x%lx cnt#1=0x%lx overflow=0x%lx", 592 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1), 593 read_sysreg(pmovsclr_el0)); 594 } 595 596 static void test_sw_incr(bool overflow_at_64bits) 597 { 598 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 599 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 600 uint32_t events[] = {SW_INCR, SW_INCR}; 601 uint64_t cntr0 = (pre_overflow + 100) & pmevcntr_mask(); 602 int i; 603 604 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 605 !check_overflow_prerequisites(overflow_at_64bits)) 606 return; 607 608 pmu_reset(); 609 610 write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 611 write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 612 /* enable counters #0 and #1 */ 613 write_sysreg_s(0x3, PMCNTENSET_EL0); 614 615 write_regn_el0(pmevcntr, 0, pre_overflow); 616 isb(); 617 618 for (i = 0; i < 100; i++) 619 write_sysreg(0x1, pmswinc_el0); 620 621 isb(); 622 report_info("SW_INCR counter #0 has value 0x%lx", read_regn_el0(pmevcntr, 0)); 623 report(read_regn_el0(pmevcntr, 0) == pre_overflow, 624 "PWSYNC does not increment if PMCR.E is unset"); 625 626 pmu_reset(); 627 628 write_regn_el0(pmevcntr, 0, pre_overflow); 629 write_sysreg_s(0x3, PMCNTENSET_EL0); 630 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 631 isb(); 632 633 for (i = 0; i < 100; i++) 634 write_sysreg(0x3, pmswinc_el0); 635 636 isb(); 637 report(read_regn_el0(pmevcntr, 0) == cntr0, "counter #0 after + 100 SW_INCR"); 638 report(read_regn_el0(pmevcntr, 1) == 100, "counter #1 after + 100 SW_INCR"); 639 report_info("counter values after 100 SW_INCR #0=0x%lx #1=0x%lx", 640 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1)); 641 report(read_sysreg(pmovsclr_el0) == 0x1, 642 "overflow on counter #0 after 100 SW_INCR"); 643 } 644 645 static void enable_chain_counter(int even) 646 { 647 write_sysreg_s(BIT(even + 1), PMCNTENSET_EL0); /* Enable the high counter first */ 648 isb(); 649 write_sysreg_s(BIT(even), PMCNTENSET_EL0); /* Enable the low counter */ 650 isb(); 651 } 652 653 static void disable_chain_counter(int even) 654 { 655 write_sysreg_s(BIT(even), PMCNTENCLR_EL0); /* Disable the low counter first*/ 656 isb(); 657 write_sysreg_s(BIT(even + 1), PMCNTENCLR_EL0); /* Disable the high counter */ 658 isb(); 659 } 660 661 static void test_chained_counters(bool unused) 662 { 663 uint32_t events[] = {CPU_CYCLES, CHAIN}; 664 uint64_t all_set = pmevcntr_mask(); 665 666 if (!satisfy_prerequisites(events, ARRAY_SIZE(events))) 667 return; 668 669 pmu_reset(); 670 671 write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 672 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 673 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 674 enable_chain_counter(0); 675 676 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E); 677 678 report(read_regn_el0(pmevcntr, 1) == 1, "CHAIN counter #1 incremented"); 679 report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #1"); 680 681 /* test 64b overflow */ 682 683 pmu_reset(); 684 685 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 686 write_regn_el0(pmevcntr, 1, 0x1); 687 enable_chain_counter(0); 688 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E); 689 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0)); 690 report(read_regn_el0(pmevcntr, 1) == 2, "CHAIN counter #1 set to 2"); 691 report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #2"); 692 693 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 694 write_regn_el0(pmevcntr, 1, all_set); 695 696 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E); 697 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0)); 698 report(read_regn_el0(pmevcntr, 1) == 0, "CHAIN counter #1 wrapped"); 699 report(read_sysreg(pmovsclr_el0) == 0x3, "overflow on even and odd counters"); 700 } 701 702 static void test_chained_sw_incr(bool unused) 703 { 704 uint32_t events[] = {SW_INCR, CHAIN}; 705 uint64_t cntr0 = (PRE_OVERFLOW_32 + 100) & pmevcntr_mask(); 706 uint64_t cntr1 = (ALL_SET_32 + 1) & pmevcntr_mask(); 707 int i; 708 709 if (!satisfy_prerequisites(events, ARRAY_SIZE(events))) 710 return; 711 712 pmu_reset(); 713 714 write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 715 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 716 enable_chain_counter(0); 717 718 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 719 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E); 720 isb(); 721 722 for (i = 0; i < 100; i++) 723 write_sysreg(0x1, pmswinc_el0); 724 725 isb(); 726 report((read_sysreg(pmovsclr_el0) == 0x1) && 727 (read_regn_el0(pmevcntr, 1) == 1), 728 "overflow and chain counter incremented after 100 SW_INCR/CHAIN"); 729 report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0), 730 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1)); 731 732 /* 64b SW_INCR and overflow on CHAIN counter*/ 733 pmu_reset(); 734 735 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 736 write_regn_el0(pmevcntr, 1, ALL_SET_32); 737 enable_chain_counter(0); 738 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E); 739 isb(); 740 741 for (i = 0; i < 100; i++) 742 write_sysreg(0x1, pmswinc_el0); 743 744 isb(); 745 report((read_sysreg(pmovsclr_el0) == 0x3) && 746 (read_regn_el0(pmevcntr, 0) == cntr0) && 747 (read_regn_el0(pmevcntr, 1) == cntr1), 748 "expected overflows and values after 100 SW_INCR/CHAIN"); 749 report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0), 750 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1)); 751 } 752 #define PRINT_REGS(__s) \ 753 report_info("%s #1=0x%lx #0=0x%lx overflow=0x%lx", __s, \ 754 read_regn_el0(pmevcntr, 1), \ 755 read_regn_el0(pmevcntr, 0), \ 756 read_sysreg(pmovsclr_el0)) 757 758 /* 759 * This test checks that a mem access loop featuring COUNT accesses 760 * does not overflow with an init value of PRE_OVERFLOW2. It also 761 * records the min/max access count to see how much the counting 762 * is (un)reliable 763 */ 764 static void test_mem_access_reliability(bool overflow_at_64bits) 765 { 766 uint32_t events[] = {MEM_ACCESS}; 767 void *addr = malloc(PAGE_SIZE); 768 uint64_t cntr_val, num_events, max = 0, min = pmevcntr_mask(); 769 uint64_t pre_overflow2 = PRE_OVERFLOW2(overflow_at_64bits); 770 uint64_t all_set = ALL_SET(overflow_at_64bits); 771 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 772 bool overflow = false; 773 774 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 775 !check_overflow_prerequisites(overflow_at_64bits)) 776 return; 777 778 pmu_reset(); 779 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 780 for (int i = 0; i < 100; i++) { 781 pmu_reset(); 782 write_regn_el0(pmevcntr, 0, pre_overflow2); 783 write_sysreg_s(0x1, PMCNTENSET_EL0); 784 isb(); 785 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 786 cntr_val = read_regn_el0(pmevcntr, 0); 787 if (cntr_val >= pre_overflow2) { 788 num_events = cntr_val - pre_overflow2; 789 } else { 790 /* unexpected counter overflow */ 791 num_events = cntr_val + all_set - pre_overflow2; 792 overflow = true; 793 report_info("iter=%d num_events=%ld min=%ld max=%ld overflow!!!", 794 i, num_events, min, max); 795 } 796 /* record extreme value */ 797 max = MAX(num_events, max); 798 min = MIN(num_events, min); 799 } 800 report_info("overflow=%d min=%ld max=%ld expected=%d acceptable margin=%d", 801 overflow, min, max, COUNT, MARGIN); 802 report(!overflow, "mem_access count is reliable"); 803 } 804 805 static void test_chain_promotion(bool unused) 806 { 807 uint32_t events[] = {MEM_ACCESS, CHAIN}; 808 void *addr = malloc(PAGE_SIZE); 809 810 if (!satisfy_prerequisites(events, ARRAY_SIZE(events))) 811 return; 812 813 /* Only enable CHAIN counter */ 814 report_prefix_push("subtest1"); 815 pmu_reset(); 816 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 817 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 818 write_sysreg_s(0x2, PMCNTENSET_EL0); 819 isb(); 820 821 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 822 PRINT_REGS("post"); 823 report(!read_regn_el0(pmevcntr, 0), 824 "chain counter not counting if even counter is disabled"); 825 report_prefix_pop(); 826 827 /* Only enable even counter */ 828 report_prefix_push("subtest2"); 829 pmu_reset(); 830 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 831 write_sysreg_s(0x1, PMCNTENSET_EL0); 832 isb(); 833 834 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 835 PRINT_REGS("post"); 836 report(!read_regn_el0(pmevcntr, 1) && (read_sysreg(pmovsclr_el0) == 0x1), 837 "odd counter did not increment on overflow if disabled"); 838 report_prefix_pop(); 839 840 /* 1st COUNT with CHAIN enabled, next COUNT with CHAIN disabled */ 841 report_prefix_push("subtest3"); 842 pmu_reset(); 843 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 844 enable_chain_counter(0); 845 PRINT_REGS("init"); 846 847 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 848 PRINT_REGS("After 1st loop"); 849 850 /* disable the CHAIN event */ 851 disable_chain_counter(0); 852 write_sysreg_s(0x1, PMCNTENSET_EL0); /* Enable the low counter */ 853 isb(); 854 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 855 PRINT_REGS("After 2nd loop"); 856 report(read_sysreg(pmovsclr_el0) == 0x1, 857 "should have triggered an overflow on #0"); 858 report(!read_regn_el0(pmevcntr, 1), 859 "CHAIN counter #1 shouldn't have incremented"); 860 report_prefix_pop(); 861 862 /* 1st COUNT with CHAIN disabled, next COUNT with CHAIN enabled */ 863 864 report_prefix_push("subtest4"); 865 pmu_reset(); 866 write_sysreg_s(0x1, PMCNTENSET_EL0); 867 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 868 isb(); 869 PRINT_REGS("init"); 870 871 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 872 PRINT_REGS("After 1st loop"); 873 874 /* Disable the low counter first and enable the chain counter */ 875 write_sysreg_s(0x1, PMCNTENCLR_EL0); 876 isb(); 877 enable_chain_counter(0); 878 879 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 880 881 PRINT_REGS("After 2nd loop"); 882 883 report((read_regn_el0(pmevcntr, 1) == 1) && 884 (read_sysreg(pmovsclr_el0) == 0x1), 885 "CHAIN counter enabled: CHAIN counter was incremented and overflow"); 886 report_prefix_pop(); 887 888 /* start as MEM_ACCESS/CPU_CYCLES and move to CHAIN/MEM_ACCESS */ 889 report_prefix_push("subtest5"); 890 pmu_reset(); 891 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 892 write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 893 write_sysreg_s(0x3, PMCNTENSET_EL0); 894 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 895 isb(); 896 PRINT_REGS("init"); 897 898 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 899 PRINT_REGS("After 1st loop"); 900 901 /* 0 becomes CHAINED */ 902 write_sysreg_s(0x3, PMCNTENCLR_EL0); 903 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 904 write_regn_el0(pmevcntr, 1, 0x0); 905 enable_chain_counter(0); 906 907 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 908 PRINT_REGS("After 2nd loop"); 909 910 report((read_regn_el0(pmevcntr, 1) == 1) && 911 (read_sysreg(pmovsclr_el0) == 0x1), 912 "32b->64b: CHAIN counter incremented and overflow"); 913 report_prefix_pop(); 914 915 /* start as CHAIN/MEM_ACCESS and move to MEM_ACCESS/CPU_CYCLES */ 916 report_prefix_push("subtest6"); 917 pmu_reset(); 918 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 919 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 920 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 921 enable_chain_counter(0); 922 PRINT_REGS("init"); 923 924 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 925 PRINT_REGS("After 1st loop"); 926 927 disable_chain_counter(0); 928 write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 929 write_sysreg_s(0x3, PMCNTENSET_EL0); 930 931 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 932 PRINT_REGS("After 2nd loop"); 933 report(read_sysreg(pmovsclr_el0) == 1, 934 "overflow is expected on counter 0"); 935 report_prefix_pop(); 936 } 937 938 static bool expect_interrupts(uint32_t bitmap) 939 { 940 int i; 941 942 if (pmu_stats.bitmap ^ bitmap || pmu_stats.unexpected) 943 return false; 944 945 for (i = 0; i < 32; i++) { 946 if (test_and_clear_bit(i, &pmu_stats.bitmap)) 947 if (pmu_stats.interrupts[i] != 1) 948 return false; 949 } 950 return true; 951 } 952 953 static void test_overflow_interrupt(bool overflow_at_64bits) 954 { 955 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 956 uint64_t all_set = pmevcntr_mask(); 957 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 958 uint32_t events[] = {MEM_ACCESS, SW_INCR}; 959 void *addr = malloc(PAGE_SIZE); 960 int i; 961 962 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 963 !check_overflow_prerequisites(overflow_at_64bits)) 964 return; 965 966 gic_enable_defaults(); 967 install_irq_handler(EL1H_IRQ, irq_handler); 968 local_irq_enable(); 969 gic_enable_irq(23); 970 971 pmu_reset(); 972 973 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 974 write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 975 write_sysreg_s(0x3, PMCNTENSET_EL0); 976 write_regn_el0(pmevcntr, 0, pre_overflow); 977 write_regn_el0(pmevcntr, 1, pre_overflow); 978 isb(); 979 980 /* interrupts are disabled (PMINTENSET_EL1 == 0) */ 981 982 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 983 report(expect_interrupts(0), "no overflow interrupt after preset"); 984 985 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 986 isb(); 987 988 for (i = 0; i < 100; i++) 989 write_sysreg(0x2, pmswinc_el0); 990 991 isb(); 992 set_pmcr(pmu.pmcr_ro); 993 isb(); 994 report(expect_interrupts(0), "no overflow interrupt after counting"); 995 996 /* enable interrupts (PMINTENSET_EL1 <= ALL_SET_32) */ 997 998 pmu_reset_stats(); 999 1000 write_regn_el0(pmevcntr, 0, pre_overflow); 1001 write_regn_el0(pmevcntr, 1, pre_overflow); 1002 write_sysreg(ALL_SET_32, pmovsclr_el0); 1003 write_sysreg(ALL_SET_32, pmintenset_el1); 1004 isb(); 1005 1006 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1007 1008 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1009 isb(); 1010 1011 for (i = 0; i < 100; i++) 1012 write_sysreg(0x3, pmswinc_el0); 1013 1014 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro); 1015 report_info("overflow=0x%lx", read_sysreg(pmovsclr_el0)); 1016 report(expect_interrupts(0x3), 1017 "overflow interrupts expected on #0 and #1"); 1018 1019 /* 1020 * promote to 64-b: 1021 * 1022 * This only applies to the !overflow_at_64bits case, as 1023 * overflow_at_64bits doesn't implement CHAIN events. The 1024 * overflow_at_64bits case just checks that chained counters are 1025 * not incremented when PMCR.LP == 1. 1026 */ 1027 1028 pmu_reset_stats(); 1029 1030 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 1031 write_regn_el0(pmevcntr, 0, pre_overflow); 1032 isb(); 1033 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1034 report(expect_interrupts(0x1), "expect overflow interrupt"); 1035 1036 /* overflow on odd counter */ 1037 pmu_reset_stats(); 1038 write_regn_el0(pmevcntr, 0, pre_overflow); 1039 write_regn_el0(pmevcntr, 1, all_set); 1040 isb(); 1041 mem_access_loop(addr, COUNT_INT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1042 if (overflow_at_64bits) { 1043 report(expect_interrupts(0x1), 1044 "expect overflow interrupt on even counter"); 1045 report(read_regn_el0(pmevcntr, 1) == all_set, 1046 "Odd counter did not change"); 1047 } else { 1048 report(expect_interrupts(0x3), 1049 "expect overflow interrupt on even and odd counter"); 1050 report(read_regn_el0(pmevcntr, 1) != all_set, 1051 "Odd counter wrapped"); 1052 } 1053 } 1054 #endif 1055 1056 /* 1057 * Ensure that the cycle counter progresses between back-to-back reads. 1058 */ 1059 static bool check_cycles_increase(void) 1060 { 1061 bool success = true; 1062 1063 /* init before event access, this test only cares about cycle count */ 1064 pmu_reset(); 1065 set_pmcntenset(1 << PMU_CYCLE_IDX); 1066 set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */ 1067 1068 set_pmcr(get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E); 1069 isb(); 1070 1071 for (int i = 0; i < NR_SAMPLES; i++) { 1072 uint64_t a, b; 1073 1074 a = get_pmccntr(); 1075 b = get_pmccntr(); 1076 1077 if (a >= b) { 1078 printf("Read %"PRId64" then %"PRId64".\n", a, b); 1079 success = false; 1080 break; 1081 } 1082 } 1083 1084 set_pmcr(get_pmcr() & ~PMU_PMCR_E); 1085 isb(); 1086 1087 return success; 1088 } 1089 1090 /* 1091 * Execute a known number of guest instructions. Only even instruction counts 1092 * greater than or equal to 4 are supported by the in-line assembly code. The 1093 * control register (PMCR_EL0) is initialized with the provided value (allowing 1094 * for example for the cycle counter or event counters to be reset). At the end 1095 * of the exact instruction loop, zero is written to PMCR_EL0 to disable 1096 * counting, allowing the cycle counter or event counters to be read at the 1097 * leisure of the calling code. 1098 */ 1099 static void measure_instrs(int num, uint32_t pmcr) 1100 { 1101 int loop = (num - 2) / 2; 1102 1103 assert(num >= 4 && ((num - 2) % 2 == 0)); 1104 precise_instrs_loop(loop, pmcr); 1105 } 1106 1107 /* 1108 * Measure cycle counts for various known instruction counts. Ensure that the 1109 * cycle counter progresses (similar to check_cycles_increase() but with more 1110 * instructions and using reset and stop controls). If supplied a positive, 1111 * nonzero CPI parameter, it also strictly checks that every measurement matches 1112 * it. Strict CPI checking is used to test -icount mode. 1113 */ 1114 static bool check_cpi(int cpi) 1115 { 1116 uint32_t pmcr = get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E; 1117 1118 /* init before event access, this test only cares about cycle count */ 1119 pmu_reset(); 1120 set_pmcntenset(1 << PMU_CYCLE_IDX); 1121 set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */ 1122 1123 if (cpi > 0) 1124 printf("Checking for CPI=%d.\n", cpi); 1125 printf("instrs : cycles0 cycles1 ...\n"); 1126 1127 for (unsigned int i = 4; i < 300; i += 32) { 1128 uint64_t avg, sum = 0; 1129 1130 printf("%4d:", i); 1131 for (int j = 0; j < NR_SAMPLES; j++) { 1132 uint64_t cycles; 1133 1134 set_pmccntr(0); 1135 measure_instrs(i, pmcr); 1136 cycles = get_pmccntr(); 1137 printf(" %4"PRId64"", cycles); 1138 1139 if (!cycles) { 1140 printf("\ncycles not incrementing!\n"); 1141 return false; 1142 } else if (cpi > 0 && cycles != i * cpi) { 1143 printf("\nunexpected cycle count received!\n"); 1144 return false; 1145 } else if ((cycles >> 32) != 0) { 1146 /* The cycles taken by the loop above should 1147 * fit in 32 bits easily. We check the upper 1148 * 32 bits of the cycle counter to make sure 1149 * there is no supprise. */ 1150 printf("\ncycle count bigger than 32bit!\n"); 1151 return false; 1152 } 1153 1154 sum += cycles; 1155 } 1156 avg = sum / NR_SAMPLES; 1157 printf(" avg=%-4"PRId64" %s=%-3"PRId64"\n", avg, 1158 (avg >= i) ? "cpi" : "ipc", 1159 (avg >= i) ? avg / i : i / avg); 1160 } 1161 1162 return true; 1163 } 1164 1165 static void pmccntr64_test(void) 1166 { 1167 #ifdef __arm__ 1168 if (pmu.version == ID_DFR0_PMU_V3) { 1169 if (ERRATA(9e3f7a296940)) { 1170 write_sysreg(0xdead, PMCCNTR64); 1171 report(read_sysreg(PMCCNTR64) == 0xdead, "pmccntr64"); 1172 } else 1173 report_skip("Skipping unsafe pmccntr64 test. Set ERRATA_9e3f7a296940=y to enable."); 1174 } 1175 #endif 1176 } 1177 1178 /* Return FALSE if no PMU found, otherwise return TRUE */ 1179 static bool pmu_probe(void) 1180 { 1181 uint32_t pmcr; 1182 uint8_t implementer; 1183 1184 pmu.version = get_pmu_version(); 1185 if (pmu.version == ID_DFR0_PMU_NOTIMPL || pmu.version == ID_DFR0_PMU_IMPDEF) 1186 return false; 1187 1188 report_info("PMU version: 0x%x", pmu.version); 1189 1190 pmcr = get_pmcr(); 1191 implementer = (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK; 1192 report_info("PMU implementer/ID code: %#"PRIx32"(\"%c\")/%#"PRIx32, 1193 (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK, 1194 implementer ? implementer : ' ', 1195 (pmcr >> PMU_PMCR_ID_SHIFT) & PMU_PMCR_ID_MASK); 1196 1197 /* store read-only and RES0 fields of the PMCR bottom-half*/ 1198 pmu.pmcr_ro = pmcr & 0xFFFFFF00; 1199 pmu.nb_implemented_counters = 1200 (pmcr >> PMU_PMCR_N_SHIFT) & PMU_PMCR_N_MASK; 1201 report_info("Implements %d event counters", 1202 pmu.nb_implemented_counters); 1203 1204 return true; 1205 } 1206 1207 static void run_test(const char *name, const char *prefix, 1208 void (*test)(bool), void *arg) 1209 { 1210 report_prefix_push(name); 1211 report_prefix_push(prefix); 1212 1213 test(arg); 1214 1215 report_prefix_pop(); 1216 report_prefix_pop(); 1217 } 1218 1219 static void run_event_test(char *name, void (*test)(bool), 1220 bool overflow_at_64bits) 1221 { 1222 const char *prefix = overflow_at_64bits ? "64-bit overflows" 1223 : "32-bit overflows"; 1224 1225 run_test(name, prefix, test, (void *)overflow_at_64bits); 1226 } 1227 1228 int main(int argc, char *argv[]) 1229 { 1230 int cpi = 0; 1231 1232 if (!pmu_probe()) { 1233 printf("No PMU found, test skipped...\n"); 1234 return report_summary(); 1235 } 1236 1237 if (argc < 2) 1238 report_abort("no test specified"); 1239 1240 report_prefix_push("pmu"); 1241 1242 if (strcmp(argv[1], "cycle-counter") == 0) { 1243 report_prefix_push(argv[1]); 1244 if (argc > 2) 1245 cpi = atol(argv[2]); 1246 report(check_cycles_increase(), 1247 "Monotonically increasing cycle count"); 1248 report(check_cpi(cpi), "Cycle/instruction ratio"); 1249 pmccntr64_test(); 1250 report_prefix_pop(); 1251 } else if (strcmp(argv[1], "pmu-event-introspection") == 0) { 1252 report_prefix_push(argv[1]); 1253 test_event_introspection(); 1254 report_prefix_pop(); 1255 } else if (strcmp(argv[1], "pmu-event-counter-config") == 0) { 1256 report_prefix_push(argv[1]); 1257 test_event_counter_config(); 1258 report_prefix_pop(); 1259 } else if (strcmp(argv[1], "pmu-basic-event-count") == 0) { 1260 run_event_test(argv[1], test_basic_event_count, false); 1261 run_event_test(argv[1], test_basic_event_count, true); 1262 } else if (strcmp(argv[1], "pmu-mem-access-reliability") == 0) { 1263 run_event_test(argv[1], test_mem_access_reliability, false); 1264 run_event_test(argv[1], test_mem_access_reliability, true); 1265 } else if (strcmp(argv[1], "pmu-mem-access") == 0) { 1266 run_event_test(argv[1], test_mem_access, false); 1267 run_event_test(argv[1], test_mem_access, true); 1268 } else if (strcmp(argv[1], "pmu-sw-incr") == 0) { 1269 run_event_test(argv[1], test_sw_incr, false); 1270 run_event_test(argv[1], test_sw_incr, true); 1271 } else if (strcmp(argv[1], "pmu-chained-counters") == 0) { 1272 run_event_test(argv[1], test_chained_counters, false); 1273 } else if (strcmp(argv[1], "pmu-chained-sw-incr") == 0) { 1274 run_event_test(argv[1], test_chained_sw_incr, false); 1275 } else if (strcmp(argv[1], "pmu-chain-promotion") == 0) { 1276 run_event_test(argv[1], test_chain_promotion, false); 1277 } else if (strcmp(argv[1], "pmu-overflow-interrupt") == 0) { 1278 run_event_test(argv[1], test_overflow_interrupt, false); 1279 run_event_test(argv[1], test_overflow_interrupt, true); 1280 } else { 1281 report_abort("Unknown sub-test '%s'", argv[1]); 1282 } 1283 1284 return report_summary(); 1285 } 1286