1 /* 2 * Test the ARM Performance Monitors Unit (PMU). 3 * 4 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. 5 * Copyright (C) 2016, Red Hat Inc, Wei Huang <wei@redhat.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU Lesser General Public License version 2.1 and 9 * only version 2.1 as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License 14 * for more details. 15 */ 16 #include "libcflat.h" 17 #include "errata.h" 18 #include "asm/barrier.h" 19 #include "asm/sysreg.h" 20 #include "asm/processor.h" 21 #include <bitops.h> 22 #include <asm/gic.h> 23 24 #define PMU_PMCR_E (1 << 0) 25 #define PMU_PMCR_P (1 << 1) 26 #define PMU_PMCR_C (1 << 2) 27 #define PMU_PMCR_D (1 << 3) 28 #define PMU_PMCR_X (1 << 4) 29 #define PMU_PMCR_DP (1 << 5) 30 #define PMU_PMCR_LC (1 << 6) 31 #define PMU_PMCR_LP (1 << 7) 32 #define PMU_PMCR_N_SHIFT 11 33 #define PMU_PMCR_N_MASK 0x1f 34 #define PMU_PMCR_ID_SHIFT 16 35 #define PMU_PMCR_ID_MASK 0xff 36 #define PMU_PMCR_IMP_SHIFT 24 37 #define PMU_PMCR_IMP_MASK 0xff 38 39 #define PMU_CYCLE_IDX 31 40 41 #define NR_SAMPLES 10 42 43 /* Some PMU events */ 44 #define SW_INCR 0x0 45 #define INST_RETIRED 0x8 46 #define CPU_CYCLES 0x11 47 #define MEM_ACCESS 0x13 48 #define INST_PREC 0x1B 49 #define STALL_FRONTEND 0x23 50 #define STALL_BACKEND 0x24 51 #define CHAIN 0x1E 52 53 #define COMMON_EVENTS_LOW 0x0 54 #define COMMON_EVENTS_HIGH 0x3F 55 #define EXT_COMMON_EVENTS_LOW 0x4000 56 #define EXT_COMMON_EVENTS_HIGH 0x403F 57 58 #define ALL_SET_32 0x00000000FFFFFFFFULL 59 #define ALL_SET_64 0xFFFFFFFFFFFFFFFFULL 60 61 #define ALL_SET(__overflow_at_64bits) \ 62 (__overflow_at_64bits ? ALL_SET_64 : ALL_SET_32) 63 64 #define ALL_CLEAR 0x0000000000000000ULL 65 #define PRE_OVERFLOW_32 0x00000000FFFFFFF0ULL 66 #define PRE_OVERFLOW_64 0xFFFFFFFFFFFFFFF0ULL 67 #define COUNT 250 68 #define MARGIN 100 69 /* 70 * PRE_OVERFLOW2 is set so that 1st @COUNT iterations do not 71 * produce 32b overflow and 2nd @COUNT iterations do. To accommodate 72 * for some observed variability we take into account a given @MARGIN 73 */ 74 #define PRE_OVERFLOW2_32 (ALL_SET_32 - COUNT - MARGIN) 75 #define PRE_OVERFLOW2_64 (ALL_SET_64 - COUNT - MARGIN) 76 77 #define PRE_OVERFLOW2(__overflow_at_64bits) \ 78 (__overflow_at_64bits ? PRE_OVERFLOW2_64 : PRE_OVERFLOW2_32) 79 80 #define PRE_OVERFLOW(__overflow_at_64bits) \ 81 (__overflow_at_64bits ? PRE_OVERFLOW_64 : PRE_OVERFLOW_32) 82 83 #define PMU_PPI 23 84 85 struct pmu { 86 unsigned int version; 87 unsigned int nb_implemented_counters; 88 uint32_t pmcr_ro; 89 }; 90 91 struct pmu_stats { 92 unsigned long bitmap; 93 uint32_t interrupts[32]; 94 bool unexpected; 95 }; 96 97 static struct pmu pmu; 98 99 #if defined(__arm__) 100 #define ID_DFR0_PERFMON_SHIFT 24 101 #define ID_DFR0_PERFMON_MASK 0xf 102 103 #define ID_DFR0_PMU_NOTIMPL 0b0000 104 #define ID_DFR0_PMU_V1 0b0001 105 #define ID_DFR0_PMU_V2 0b0010 106 #define ID_DFR0_PMU_V3 0b0011 107 #define ID_DFR0_PMU_V3_8_1 0b0100 108 #define ID_DFR0_PMU_V3_8_4 0b0101 109 #define ID_DFR0_PMU_V3_8_5 0b0110 110 #define ID_DFR0_PMU_IMPDEF 0b1111 111 112 #define PMCR __ACCESS_CP15(c9, 0, c12, 0) 113 #define ID_DFR0 __ACCESS_CP15(c0, 0, c1, 2) 114 #define PMSELR __ACCESS_CP15(c9, 0, c12, 5) 115 #define PMXEVTYPER __ACCESS_CP15(c9, 0, c13, 1) 116 #define PMCNTENSET __ACCESS_CP15(c9, 0, c12, 1) 117 #define PMCNTENCLR __ACCESS_CP15(c9, 0, c12, 2) 118 #define PMOVSR __ACCESS_CP15(c9, 0, c12, 3) 119 #define PMCCNTR32 __ACCESS_CP15(c9, 0, c13, 0) 120 #define PMINTENCLR __ACCESS_CP15(c9, 0, c14, 2) 121 #define PMCCNTR64 __ACCESS_CP15_64(0, c9) 122 123 static inline uint32_t get_id_dfr0(void) { return read_sysreg(ID_DFR0); } 124 static inline uint32_t get_pmcr(void) { return read_sysreg(PMCR); } 125 static inline void set_pmcr(uint32_t v) { write_sysreg(v, PMCR); } 126 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, PMCNTENSET); } 127 128 static inline uint8_t get_pmu_version(void) 129 { 130 return (get_id_dfr0() >> ID_DFR0_PERFMON_SHIFT) & ID_DFR0_PERFMON_MASK; 131 } 132 133 static inline uint64_t get_pmccntr(void) 134 { 135 return read_sysreg(PMCCNTR32); 136 } 137 138 static inline void set_pmccntr(uint64_t value) 139 { 140 write_sysreg(value & 0xffffffff, PMCCNTR32); 141 } 142 143 /* PMCCFILTR is an obsolete name for PMXEVTYPER31 in ARMv7 */ 144 static inline void set_pmccfiltr(uint32_t value) 145 { 146 write_sysreg(PMU_CYCLE_IDX, PMSELR); 147 write_sysreg(value, PMXEVTYPER); 148 isb(); 149 } 150 151 /* 152 * Extra instructions inserted by the compiler would be difficult to compensate 153 * for, so hand assemble everything between, and including, the PMCR accesses 154 * to start and stop counting. isb instructions were inserted to make sure 155 * pmccntr read after this function returns the exact instructions executed in 156 * the controlled block. Total instrs = isb + mcr + 2*loop = 2 + 2*loop. 157 */ 158 static inline void precise_instrs_loop(int loop, uint32_t pmcr) 159 { 160 asm volatile( 161 " mcr p15, 0, %[pmcr], c9, c12, 0\n" 162 " isb\n" 163 "1: subs %[loop], %[loop], #1\n" 164 " bgt 1b\n" 165 " mcr p15, 0, %[z], c9, c12, 0\n" 166 " isb\n" 167 : [loop] "+r" (loop) 168 : [pmcr] "r" (pmcr), [z] "r" (0) 169 : "cc"); 170 } 171 172 static void pmu_reset(void) 173 { 174 /* reset all counters, counting disabled at PMCR level*/ 175 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P); 176 /* Disable all counters */ 177 write_sysreg(ALL_SET_32, PMCNTENCLR); 178 /* clear overflow reg */ 179 write_sysreg(ALL_SET_32, PMOVSR); 180 /* disable overflow interrupts on all counters */ 181 write_sysreg(ALL_SET_32, PMINTENCLR); 182 isb(); 183 } 184 185 /* event counter tests only implemented for aarch64 */ 186 static void test_event_introspection(void) {} 187 static void test_event_counter_config(void) {} 188 static void test_basic_event_count(bool overflow_at_64bits) {} 189 static void test_mem_access_reliability(bool overflow_at_64bits) {} 190 static void test_mem_access(bool overflow_at_64bits) {} 191 static void test_sw_incr(bool overflow_at_64bits) {} 192 static void test_chained_counters(bool unused) {} 193 static void test_chained_sw_incr(bool unused) {} 194 static void test_chain_promotion(bool unused) {} 195 static void test_overflow_interrupt(bool overflow_at_64bits) {} 196 197 #elif defined(__aarch64__) 198 #define ID_AA64DFR0_PERFMON_SHIFT 8 199 #define ID_AA64DFR0_PERFMON_MASK 0xf 200 201 #define ID_DFR0_PMU_NOTIMPL 0b0000 202 #define ID_DFR0_PMU_V3 0b0001 203 #define ID_DFR0_PMU_V3_8_1 0b0100 204 #define ID_DFR0_PMU_V3_8_4 0b0101 205 #define ID_DFR0_PMU_V3_8_5 0b0110 206 #define ID_DFR0_PMU_IMPDEF 0b1111 207 208 static inline uint32_t get_id_aa64dfr0(void) { return read_sysreg(id_aa64dfr0_el1); } 209 static inline uint32_t get_pmcr(void) { return read_sysreg(pmcr_el0); } 210 static inline void set_pmcr(uint32_t v) { write_sysreg(v, pmcr_el0); } 211 static inline uint64_t get_pmccntr(void) { return read_sysreg(pmccntr_el0); } 212 static inline void set_pmccntr(uint64_t v) { write_sysreg(v, pmccntr_el0); } 213 static inline void set_pmcntenset(uint32_t v) { write_sysreg(v, pmcntenset_el0); } 214 static inline void set_pmccfiltr(uint32_t v) { write_sysreg(v, pmccfiltr_el0); } 215 216 static inline uint8_t get_pmu_version(void) 217 { 218 uint8_t ver = (get_id_aa64dfr0() >> ID_AA64DFR0_PERFMON_SHIFT) & ID_AA64DFR0_PERFMON_MASK; 219 return ver; 220 } 221 222 /* 223 * Extra instructions inserted by the compiler would be difficult to compensate 224 * for, so hand assemble everything between, and including, the PMCR accesses 225 * to start and stop counting. isb instructions are inserted to make sure 226 * pmccntr read after this function returns the exact instructions executed 227 * in the controlled block. Total instrs = isb + msr + 2*loop = 2 + 2*loop. 228 */ 229 static inline void precise_instrs_loop(int loop, uint32_t pmcr) 230 { 231 uint64_t pmcr64 = pmcr; 232 asm volatile( 233 " msr pmcr_el0, %[pmcr]\n" 234 " isb\n" 235 "1: subs %w[loop], %w[loop], #1\n" 236 " b.gt 1b\n" 237 " msr pmcr_el0, xzr\n" 238 " isb\n" 239 : [loop] "+r" (loop) 240 : [pmcr] "r" (pmcr64) 241 : "cc"); 242 } 243 244 #define PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7) 245 #define PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1) 246 #define PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2) 247 248 #define PMEVTYPER_EXCLUDE_EL1 BIT(31) 249 #define PMEVTYPER_EXCLUDE_EL0 BIT(30) 250 251 static bool is_event_supported(uint32_t n, bool warn) 252 { 253 uint64_t pmceid0 = read_sysreg(pmceid0_el0); 254 uint64_t pmceid1 = read_sysreg_s(PMCEID1_EL0); 255 bool supported; 256 uint64_t reg; 257 258 /* 259 * The low 32-bits of PMCEID0/1 respectively describe 260 * event support for events 0-31/32-63. Their High 261 * 32-bits describe support for extended events 262 * starting at 0x4000, using the same split. 263 */ 264 assert((n >= COMMON_EVENTS_LOW && n <= COMMON_EVENTS_HIGH) || 265 (n >= EXT_COMMON_EVENTS_LOW && n <= EXT_COMMON_EVENTS_HIGH)); 266 267 if (n <= COMMON_EVENTS_HIGH) 268 reg = lower_32_bits(pmceid0) | ((u64)lower_32_bits(pmceid1) << 32); 269 else 270 reg = upper_32_bits(pmceid0) | ((u64)upper_32_bits(pmceid1) << 32); 271 272 supported = reg & (1UL << (n & 0x3F)); 273 274 if (!supported && warn) 275 report_info("event 0x%x is not supported", n); 276 return supported; 277 } 278 279 static void test_event_introspection(void) 280 { 281 bool required_events; 282 283 if (!pmu.nb_implemented_counters) { 284 report_skip("No event counter, skip ..."); 285 return; 286 } 287 288 /* PMUv3 requires an implementation includes some common events */ 289 required_events = is_event_supported(SW_INCR, true) && 290 is_event_supported(CPU_CYCLES, true) && 291 (is_event_supported(INST_RETIRED, true) || 292 is_event_supported(INST_PREC, true)); 293 294 if (pmu.version >= ID_DFR0_PMU_V3_8_1) { 295 required_events = required_events && 296 is_event_supported(STALL_FRONTEND, true) && 297 is_event_supported(STALL_BACKEND, true); 298 } 299 300 report(required_events, "Check required events are implemented"); 301 } 302 303 /* 304 * Extra instructions inserted by the compiler would be difficult to compensate 305 * for, so hand assemble everything between, and including, the PMCR accesses 306 * to start and stop counting. isb instructions are inserted to make sure 307 * pmccntr read after this function returns the exact instructions executed 308 * in the controlled block. Loads @loop times the data at @address into x9. 309 */ 310 static void mem_access_loop(void *addr, long loop, uint32_t pmcr) 311 { 312 uint64_t pmcr64 = pmcr; 313 asm volatile( 314 " dsb ish\n" 315 " msr pmcr_el0, %[pmcr]\n" 316 " isb\n" 317 " dsb ish\n" 318 " mov x10, %[loop]\n" 319 "1: sub x10, x10, #1\n" 320 " ldr x9, [%[addr]]\n" 321 " cmp x10, #0x0\n" 322 " b.gt 1b\n" 323 " dsb ish\n" 324 " msr pmcr_el0, xzr\n" 325 " isb\n" 326 : 327 : [addr] "r" (addr), [pmcr] "r" (pmcr64), [loop] "r" (loop) 328 : "x9", "x10", "cc"); 329 } 330 331 static struct pmu_stats pmu_stats; 332 333 static void irq_handler(struct pt_regs *regs) 334 { 335 uint32_t irqstat, irqnr; 336 337 irqstat = gic_read_iar(); 338 irqnr = gic_iar_irqnr(irqstat); 339 340 if (irqnr == PMU_PPI) { 341 unsigned long overflows = read_sysreg(pmovsclr_el0); 342 int i; 343 344 for (i = 0; i < 32; i++) { 345 if (test_and_clear_bit(i, &overflows)) { 346 pmu_stats.interrupts[i]++; 347 pmu_stats.bitmap |= 1 << i; 348 } 349 } 350 write_sysreg(ALL_SET_32, pmovsclr_el0); 351 isb(); 352 } else { 353 pmu_stats.unexpected = true; 354 } 355 gic_write_eoir(irqstat); 356 } 357 358 static void pmu_reset_stats(void) 359 { 360 int i; 361 362 for (i = 0; i < 32; i++) 363 pmu_stats.interrupts[i] = 0; 364 365 pmu_stats.bitmap = 0; 366 pmu_stats.unexpected = false; 367 } 368 369 static void pmu_reset(void) 370 { 371 /* reset all counters, counting disabled at PMCR level*/ 372 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P); 373 /* Disable all counters */ 374 write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0); 375 /* clear overflow reg */ 376 write_sysreg(ALL_SET_32, pmovsclr_el0); 377 /* disable overflow interrupts on all counters */ 378 write_sysreg(ALL_SET_32, pmintenclr_el1); 379 pmu_reset_stats(); 380 isb(); 381 } 382 383 static void test_event_counter_config(void) 384 { 385 int i; 386 387 if (!pmu.nb_implemented_counters) { 388 report_skip("No event counter, skip ..."); 389 return; 390 } 391 392 pmu_reset(); 393 394 /* 395 * Test setting through PMESELR/PMXEVTYPER and PMEVTYPERn read, 396 * select counter 0 397 */ 398 write_sysreg(1, PMSELR_EL0); 399 /* program this counter to count unsupported event */ 400 write_sysreg(0xEA, PMXEVTYPER_EL0); 401 write_sysreg(0xdeadbeef, PMXEVCNTR_EL0); 402 report((read_regn_el0(pmevtyper, 1) & 0xFFF) == 0xEA, 403 "PMESELR/PMXEVTYPER/PMEVTYPERn"); 404 report((read_regn_el0(pmevcntr, 1) == 0xdeadbeef), 405 "PMESELR/PMXEVCNTR/PMEVCNTRn"); 406 407 /* try to configure an unsupported event within the range [0x0, 0x3F] */ 408 for (i = 0; i <= 0x3F; i++) { 409 if (!is_event_supported(i, false)) 410 break; 411 } 412 if (i > 0x3F) { 413 report_skip("pmevtyper: all events within [0x0, 0x3F] are supported"); 414 return; 415 } 416 417 /* select counter 0 */ 418 write_sysreg(0, PMSELR_EL0); 419 /* program this counter to count unsupported event */ 420 write_sysreg(i, PMXEVCNTR_EL0); 421 /* read the counter value */ 422 read_sysreg(PMXEVCNTR_EL0); 423 report(read_sysreg(PMXEVCNTR_EL0) == i, 424 "read of a counter programmed with unsupported event"); 425 } 426 427 static bool satisfy_prerequisites(uint32_t *events, unsigned int nb_events) 428 { 429 int i; 430 431 if (pmu.nb_implemented_counters < nb_events) { 432 report_skip("Skip test as number of counters is too small (%d)", 433 pmu.nb_implemented_counters); 434 return false; 435 } 436 437 for (i = 0; i < nb_events; i++) { 438 if (!is_event_supported(events[i], false)) { 439 report_skip("Skip test as event 0x%x is not supported", 440 events[i]); 441 return false; 442 } 443 } 444 return true; 445 } 446 447 static uint64_t pmevcntr_mask(void) 448 { 449 /* 450 * Bits [63:0] are always incremented for 64-bit counters, 451 * even if the PMU is configured to generate an overflow at 452 * bits [31:0] 453 * 454 * For more details see the AArch64.IncrementEventCounter() 455 * pseudo-code in the ARM ARM DDI 0487I.a, section J1.1.1. 456 */ 457 if (pmu.version >= ID_DFR0_PMU_V3_8_5) 458 return ~0; 459 460 return (uint32_t)~0; 461 } 462 463 static bool check_overflow_prerequisites(bool overflow_at_64bits) 464 { 465 if (overflow_at_64bits && pmu.version < ID_DFR0_PMU_V3_8_5) { 466 report_skip("Skip test as 64 overflows need FEAT_PMUv3p5"); 467 return false; 468 } 469 470 return true; 471 } 472 473 static void test_basic_event_count(bool overflow_at_64bits) 474 { 475 uint32_t implemented_counter_mask, non_implemented_counter_mask; 476 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 477 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 478 uint32_t events[] = {CPU_CYCLES, INST_RETIRED}; 479 uint32_t counter_mask; 480 481 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 482 !check_overflow_prerequisites(overflow_at_64bits)) 483 return; 484 485 implemented_counter_mask = BIT(pmu.nb_implemented_counters) - 1; 486 non_implemented_counter_mask = ~(BIT(31) | implemented_counter_mask); 487 counter_mask = implemented_counter_mask | non_implemented_counter_mask; 488 489 write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 490 write_regn_el0(pmevtyper, 1, INST_RETIRED | PMEVTYPER_EXCLUDE_EL0); 491 492 /* disable all counters */ 493 write_sysreg_s(ALL_SET_32, PMCNTENCLR_EL0); 494 report(!read_sysreg_s(PMCNTENCLR_EL0) && !read_sysreg_s(PMCNTENSET_EL0), 495 "pmcntenclr: disable all counters"); 496 497 /* 498 * clear cycle and all event counters and allow counter enablement 499 * through PMCNTENSET. LC is RES1. 500 */ 501 set_pmcr(pmu.pmcr_ro | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_P | pmcr_lp); 502 isb(); 503 report(get_pmcr() == (pmu.pmcr_ro | PMU_PMCR_LC | pmcr_lp), "pmcr: reset counters"); 504 505 /* Preset counter #0 to pre overflow value to trigger an overflow */ 506 write_regn_el0(pmevcntr, 0, pre_overflow); 507 report(read_regn_el0(pmevcntr, 0) == pre_overflow, 508 "counter #0 preset to pre-overflow value"); 509 report(!read_regn_el0(pmevcntr, 1), "counter #1 is 0"); 510 511 /* 512 * Enable all implemented counters and also attempt to enable 513 * not supported counters. Counting still is disabled by !PMCR.E 514 */ 515 write_sysreg_s(counter_mask, PMCNTENSET_EL0); 516 517 /* check only those implemented are enabled */ 518 report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) && 519 (read_sysreg_s(PMCNTENSET_EL0) == implemented_counter_mask), 520 "pmcntenset: enabled implemented_counters"); 521 522 /* Disable all counters but counters #0 and #1 */ 523 write_sysreg_s(~0x3, PMCNTENCLR_EL0); 524 report((read_sysreg_s(PMCNTENSET_EL0) == read_sysreg_s(PMCNTENCLR_EL0)) && 525 (read_sysreg_s(PMCNTENSET_EL0) == 0x3), 526 "pmcntenset: just enabled #0 and #1"); 527 528 /* clear overflow register */ 529 write_sysreg(ALL_SET_32, pmovsclr_el0); 530 report(!read_sysreg(pmovsclr_el0), "check overflow reg is 0"); 531 532 /* disable overflow interrupts on all counters*/ 533 write_sysreg(ALL_SET_32, pmintenclr_el1); 534 report(!read_sysreg(pmintenclr_el1), 535 "pmintenclr_el1=0, all interrupts disabled"); 536 537 /* enable overflow interrupts on all event counters */ 538 write_sysreg(implemented_counter_mask | non_implemented_counter_mask, 539 pmintenset_el1); 540 report(read_sysreg(pmintenset_el1) == implemented_counter_mask, 541 "overflow interrupts enabled on all implemented counters"); 542 543 /* Set PMCR.E, execute asm code and unset PMCR.E */ 544 precise_instrs_loop(20, pmu.pmcr_ro | PMU_PMCR_E); 545 546 report_info("counter #0 is 0x%lx (CPU_CYCLES)", 547 read_regn_el0(pmevcntr, 0)); 548 report_info("counter #1 is 0x%lx (INST_RETIRED)", 549 read_regn_el0(pmevcntr, 1)); 550 551 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0)); 552 report(read_sysreg(pmovsclr_el0) & 0x1, 553 "check overflow happened on #0 only"); 554 } 555 556 static void test_mem_access(bool overflow_at_64bits) 557 { 558 void *addr = malloc(PAGE_SIZE); 559 uint32_t events[] = {MEM_ACCESS, MEM_ACCESS}; 560 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 561 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 562 563 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 564 !check_overflow_prerequisites(overflow_at_64bits)) 565 return; 566 567 pmu_reset(); 568 569 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 570 write_regn_el0(pmevtyper, 1, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 571 write_sysreg_s(0x3, PMCNTENSET_EL0); 572 isb(); 573 mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 574 report_info("counter #0 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 0)); 575 report_info("counter #1 is 0x%lx (MEM_ACCESS)", read_regn_el0(pmevcntr, 1)); 576 /* We may measure more than 20 mem access depending on the core */ 577 report((read_regn_el0(pmevcntr, 0) == read_regn_el0(pmevcntr, 1)) && 578 (read_regn_el0(pmevcntr, 0) >= 20) && !read_sysreg(pmovsclr_el0), 579 "Ran 20 mem accesses"); 580 581 pmu_reset(); 582 583 write_regn_el0(pmevcntr, 0, pre_overflow); 584 write_regn_el0(pmevcntr, 1, pre_overflow); 585 write_sysreg_s(0x3, PMCNTENSET_EL0); 586 isb(); 587 mem_access_loop(addr, 20, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 588 report(read_sysreg(pmovsclr_el0) == 0x3, 589 "Ran 20 mem accesses with expected overflows on both counters"); 590 report_info("cnt#0=0x%lx cnt#1=0x%lx overflow=0x%lx", 591 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1), 592 read_sysreg(pmovsclr_el0)); 593 } 594 595 static void test_sw_incr(bool overflow_at_64bits) 596 { 597 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 598 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 599 uint32_t events[] = {SW_INCR, SW_INCR}; 600 uint64_t cntr0 = (pre_overflow + 100) & pmevcntr_mask(); 601 int i; 602 603 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 604 !check_overflow_prerequisites(overflow_at_64bits)) 605 return; 606 607 pmu_reset(); 608 609 write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 610 write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 611 /* enable counters #0 and #1 */ 612 write_sysreg_s(0x3, PMCNTENSET_EL0); 613 614 write_regn_el0(pmevcntr, 0, pre_overflow); 615 isb(); 616 617 for (i = 0; i < 100; i++) 618 write_sysreg(0x1, pmswinc_el0); 619 620 isb(); 621 report_info("SW_INCR counter #0 has value 0x%lx", read_regn_el0(pmevcntr, 0)); 622 report(read_regn_el0(pmevcntr, 0) == pre_overflow, 623 "PWSYNC does not increment if PMCR.E is unset"); 624 625 pmu_reset(); 626 627 write_regn_el0(pmevcntr, 0, pre_overflow); 628 write_sysreg_s(0x3, PMCNTENSET_EL0); 629 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 630 isb(); 631 632 for (i = 0; i < 100; i++) 633 write_sysreg(0x3, pmswinc_el0); 634 635 isb(); 636 report(read_regn_el0(pmevcntr, 0) == cntr0, "counter #0 after + 100 SW_INCR"); 637 report(read_regn_el0(pmevcntr, 1) == 100, "counter #1 after + 100 SW_INCR"); 638 report_info("counter values after 100 SW_INCR #0=0x%lx #1=0x%lx", 639 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1)); 640 report(read_sysreg(pmovsclr_el0) == 0x1, 641 "overflow on counter #0 after 100 SW_INCR"); 642 } 643 644 static void enable_chain_counter(int even) 645 { 646 write_sysreg_s(BIT(even + 1), PMCNTENSET_EL0); /* Enable the high counter first */ 647 isb(); 648 write_sysreg_s(BIT(even), PMCNTENSET_EL0); /* Enable the low counter */ 649 isb(); 650 } 651 652 static void disable_chain_counter(int even) 653 { 654 write_sysreg_s(BIT(even), PMCNTENCLR_EL0); /* Disable the low counter first*/ 655 isb(); 656 write_sysreg_s(BIT(even + 1), PMCNTENCLR_EL0); /* Disable the high counter */ 657 isb(); 658 } 659 660 static void test_chained_counters(bool unused) 661 { 662 uint32_t events[] = {CPU_CYCLES, CHAIN}; 663 uint64_t all_set = pmevcntr_mask(); 664 665 if (!satisfy_prerequisites(events, ARRAY_SIZE(events))) 666 return; 667 668 pmu_reset(); 669 670 write_regn_el0(pmevtyper, 0, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 671 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 672 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 673 enable_chain_counter(0); 674 675 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E); 676 677 report(read_regn_el0(pmevcntr, 1) == 1, "CHAIN counter #1 incremented"); 678 report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #1"); 679 680 /* test 64b overflow */ 681 682 pmu_reset(); 683 684 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 685 write_regn_el0(pmevcntr, 1, 0x1); 686 enable_chain_counter(0); 687 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E); 688 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0)); 689 report(read_regn_el0(pmevcntr, 1) == 2, "CHAIN counter #1 set to 2"); 690 report(read_sysreg(pmovsclr_el0) == 0x1, "overflow recorded for chained incr #2"); 691 692 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 693 write_regn_el0(pmevcntr, 1, all_set); 694 695 precise_instrs_loop(22, pmu.pmcr_ro | PMU_PMCR_E); 696 report_info("overflow reg = 0x%lx", read_sysreg(pmovsclr_el0)); 697 report(read_regn_el0(pmevcntr, 1) == 0, "CHAIN counter #1 wrapped"); 698 report(read_sysreg(pmovsclr_el0) == 0x3, "overflow on even and odd counters"); 699 } 700 701 static void test_chained_sw_incr(bool unused) 702 { 703 uint32_t events[] = {SW_INCR, CHAIN}; 704 uint64_t cntr0 = (PRE_OVERFLOW_32 + 100) & pmevcntr_mask(); 705 uint64_t cntr1 = (ALL_SET_32 + 1) & pmevcntr_mask(); 706 int i; 707 708 if (!satisfy_prerequisites(events, ARRAY_SIZE(events))) 709 return; 710 711 pmu_reset(); 712 713 write_regn_el0(pmevtyper, 0, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 714 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 715 enable_chain_counter(0); 716 717 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 718 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E); 719 isb(); 720 721 for (i = 0; i < 100; i++) 722 write_sysreg(0x1, pmswinc_el0); 723 724 isb(); 725 report((read_sysreg(pmovsclr_el0) == 0x1) && 726 (read_regn_el0(pmevcntr, 1) == 1), 727 "overflow and chain counter incremented after 100 SW_INCR/CHAIN"); 728 report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0), 729 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1)); 730 731 /* 64b SW_INCR and overflow on CHAIN counter*/ 732 pmu_reset(); 733 734 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 735 write_regn_el0(pmevcntr, 1, ALL_SET_32); 736 enable_chain_counter(0); 737 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E); 738 isb(); 739 740 for (i = 0; i < 100; i++) 741 write_sysreg(0x1, pmswinc_el0); 742 743 isb(); 744 report((read_sysreg(pmovsclr_el0) == 0x3) && 745 (read_regn_el0(pmevcntr, 0) == cntr0) && 746 (read_regn_el0(pmevcntr, 1) == cntr1), 747 "expected overflows and values after 100 SW_INCR/CHAIN"); 748 report_info("overflow=0x%lx, #0=0x%lx #1=0x%lx", read_sysreg(pmovsclr_el0), 749 read_regn_el0(pmevcntr, 0), read_regn_el0(pmevcntr, 1)); 750 } 751 #define PRINT_REGS(__s) \ 752 report_info("%s #1=0x%lx #0=0x%lx overflow=0x%lx", __s, \ 753 read_regn_el0(pmevcntr, 1), \ 754 read_regn_el0(pmevcntr, 0), \ 755 read_sysreg(pmovsclr_el0)) 756 757 /* 758 * This test checks that a mem access loop featuring COUNT accesses 759 * does not overflow with an init value of PRE_OVERFLOW2. It also 760 * records the min/max access count to see how much the counting 761 * is (un)reliable 762 */ 763 static void test_mem_access_reliability(bool overflow_at_64bits) 764 { 765 uint32_t events[] = {MEM_ACCESS}; 766 void *addr = malloc(PAGE_SIZE); 767 uint64_t cntr_val, num_events, max = 0, min = pmevcntr_mask(); 768 uint64_t pre_overflow2 = PRE_OVERFLOW2(overflow_at_64bits); 769 uint64_t all_set = ALL_SET(overflow_at_64bits); 770 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 771 bool overflow = false; 772 773 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 774 !check_overflow_prerequisites(overflow_at_64bits)) 775 return; 776 777 pmu_reset(); 778 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 779 for (int i = 0; i < 100; i++) { 780 pmu_reset(); 781 write_regn_el0(pmevcntr, 0, pre_overflow2); 782 write_sysreg_s(0x1, PMCNTENSET_EL0); 783 isb(); 784 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 785 cntr_val = read_regn_el0(pmevcntr, 0); 786 if (cntr_val >= pre_overflow2) { 787 num_events = cntr_val - pre_overflow2; 788 } else { 789 /* unexpected counter overflow */ 790 num_events = cntr_val + all_set - pre_overflow2; 791 overflow = true; 792 report_info("iter=%d num_events=%ld min=%ld max=%ld overflow!!!", 793 i, num_events, min, max); 794 } 795 /* record extreme value */ 796 max = MAX(num_events, max); 797 min = MIN(num_events, min); 798 } 799 report_info("overflow=%d min=%ld max=%ld expected=%d acceptable margin=%d", 800 overflow, min, max, COUNT, MARGIN); 801 report(!overflow, "mem_access count is reliable"); 802 } 803 804 static void test_chain_promotion(bool unused) 805 { 806 uint32_t events[] = {MEM_ACCESS, CHAIN}; 807 void *addr = malloc(PAGE_SIZE); 808 809 if (!satisfy_prerequisites(events, ARRAY_SIZE(events))) 810 return; 811 812 /* Only enable CHAIN counter */ 813 report_prefix_push("subtest1"); 814 pmu_reset(); 815 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 816 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 817 write_sysreg_s(0x2, PMCNTENSET_EL0); 818 isb(); 819 820 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 821 PRINT_REGS("post"); 822 report(!read_regn_el0(pmevcntr, 0), 823 "chain counter not counting if even counter is disabled"); 824 report_prefix_pop(); 825 826 /* Only enable even counter */ 827 report_prefix_push("subtest2"); 828 pmu_reset(); 829 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW_32); 830 write_sysreg_s(0x1, PMCNTENSET_EL0); 831 isb(); 832 833 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 834 PRINT_REGS("post"); 835 report(!read_regn_el0(pmevcntr, 1) && (read_sysreg(pmovsclr_el0) == 0x1), 836 "odd counter did not increment on overflow if disabled"); 837 report_prefix_pop(); 838 839 /* 1st COUNT with CHAIN enabled, next COUNT with CHAIN disabled */ 840 report_prefix_push("subtest3"); 841 pmu_reset(); 842 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 843 enable_chain_counter(0); 844 PRINT_REGS("init"); 845 846 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 847 PRINT_REGS("After 1st loop"); 848 849 /* disable the CHAIN event */ 850 disable_chain_counter(0); 851 write_sysreg_s(0x1, PMCNTENSET_EL0); /* Enable the low counter */ 852 isb(); 853 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 854 PRINT_REGS("After 2nd loop"); 855 report(read_sysreg(pmovsclr_el0) == 0x1, 856 "should have triggered an overflow on #0"); 857 report(!read_regn_el0(pmevcntr, 1), 858 "CHAIN counter #1 shouldn't have incremented"); 859 report_prefix_pop(); 860 861 /* 1st COUNT with CHAIN disabled, next COUNT with CHAIN enabled */ 862 863 report_prefix_push("subtest4"); 864 pmu_reset(); 865 write_sysreg_s(0x1, PMCNTENSET_EL0); 866 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 867 isb(); 868 PRINT_REGS("init"); 869 870 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 871 PRINT_REGS("After 1st loop"); 872 873 /* Disable the low counter first and enable the chain counter */ 874 write_sysreg_s(0x1, PMCNTENCLR_EL0); 875 isb(); 876 enable_chain_counter(0); 877 878 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 879 880 PRINT_REGS("After 2nd loop"); 881 882 report((read_regn_el0(pmevcntr, 1) == 1) && 883 (read_sysreg(pmovsclr_el0) == 0x1), 884 "CHAIN counter enabled: CHAIN counter was incremented and overflow"); 885 report_prefix_pop(); 886 887 /* start as MEM_ACCESS/CPU_CYCLES and move to CHAIN/MEM_ACCESS */ 888 report_prefix_push("subtest5"); 889 pmu_reset(); 890 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 891 write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 892 write_sysreg_s(0x3, PMCNTENSET_EL0); 893 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 894 isb(); 895 PRINT_REGS("init"); 896 897 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 898 PRINT_REGS("After 1st loop"); 899 900 /* 0 becomes CHAINED */ 901 write_sysreg_s(0x3, PMCNTENCLR_EL0); 902 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 903 write_regn_el0(pmevcntr, 1, 0x0); 904 enable_chain_counter(0); 905 906 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 907 PRINT_REGS("After 2nd loop"); 908 909 report((read_regn_el0(pmevcntr, 1) == 1) && 910 (read_sysreg(pmovsclr_el0) == 0x1), 911 "32b->64b: CHAIN counter incremented and overflow"); 912 report_prefix_pop(); 913 914 /* start as CHAIN/MEM_ACCESS and move to MEM_ACCESS/CPU_CYCLES */ 915 report_prefix_push("subtest6"); 916 pmu_reset(); 917 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 918 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 919 write_regn_el0(pmevcntr, 0, PRE_OVERFLOW2_32); 920 enable_chain_counter(0); 921 PRINT_REGS("init"); 922 923 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 924 PRINT_REGS("After 1st loop"); 925 926 disable_chain_counter(0); 927 write_regn_el0(pmevtyper, 1, CPU_CYCLES | PMEVTYPER_EXCLUDE_EL0); 928 write_sysreg_s(0x3, PMCNTENSET_EL0); 929 930 mem_access_loop(addr, COUNT, pmu.pmcr_ro | PMU_PMCR_E); 931 PRINT_REGS("After 2nd loop"); 932 report(read_sysreg(pmovsclr_el0) == 1, 933 "overflow is expected on counter 0"); 934 report_prefix_pop(); 935 } 936 937 static bool expect_interrupts(uint32_t bitmap) 938 { 939 int i; 940 941 if (pmu_stats.bitmap ^ bitmap || pmu_stats.unexpected) 942 return false; 943 944 for (i = 0; i < 32; i++) { 945 if (test_and_clear_bit(i, &pmu_stats.bitmap)) 946 if (pmu_stats.interrupts[i] != 1) 947 return false; 948 } 949 return true; 950 } 951 952 static void test_overflow_interrupt(bool overflow_at_64bits) 953 { 954 uint64_t pre_overflow = PRE_OVERFLOW(overflow_at_64bits); 955 uint64_t all_set = pmevcntr_mask(); 956 uint64_t pmcr_lp = overflow_at_64bits ? PMU_PMCR_LP : 0; 957 uint32_t events[] = {MEM_ACCESS, SW_INCR}; 958 void *addr = malloc(PAGE_SIZE); 959 int i; 960 961 if (!satisfy_prerequisites(events, ARRAY_SIZE(events)) || 962 !check_overflow_prerequisites(overflow_at_64bits)) 963 return; 964 965 gic_enable_defaults(); 966 install_irq_handler(EL1H_IRQ, irq_handler); 967 local_irq_enable(); 968 gic_enable_irq(23); 969 970 pmu_reset(); 971 972 write_regn_el0(pmevtyper, 0, MEM_ACCESS | PMEVTYPER_EXCLUDE_EL0); 973 write_regn_el0(pmevtyper, 1, SW_INCR | PMEVTYPER_EXCLUDE_EL0); 974 write_sysreg_s(0x3, PMCNTENSET_EL0); 975 write_regn_el0(pmevcntr, 0, pre_overflow); 976 write_regn_el0(pmevcntr, 1, pre_overflow); 977 isb(); 978 979 /* interrupts are disabled (PMINTENSET_EL1 == 0) */ 980 981 mem_access_loop(addr, 200, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 982 report(expect_interrupts(0), "no overflow interrupt after preset"); 983 984 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 985 isb(); 986 987 for (i = 0; i < 100; i++) 988 write_sysreg(0x2, pmswinc_el0); 989 990 isb(); 991 set_pmcr(pmu.pmcr_ro); 992 isb(); 993 report(expect_interrupts(0), "no overflow interrupt after counting"); 994 995 /* enable interrupts (PMINTENSET_EL1 <= ALL_SET_32) */ 996 997 pmu_reset_stats(); 998 999 write_regn_el0(pmevcntr, 0, pre_overflow); 1000 write_regn_el0(pmevcntr, 1, pre_overflow); 1001 write_sysreg(ALL_SET_32, pmovsclr_el0); 1002 write_sysreg(ALL_SET_32, pmintenset_el1); 1003 isb(); 1004 1005 mem_access_loop(addr, 200, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1006 1007 set_pmcr(pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1008 isb(); 1009 1010 for (i = 0; i < 100; i++) 1011 write_sysreg(0x3, pmswinc_el0); 1012 1013 mem_access_loop(addr, 200, pmu.pmcr_ro); 1014 report_info("overflow=0x%lx", read_sysreg(pmovsclr_el0)); 1015 report(expect_interrupts(0x3), 1016 "overflow interrupts expected on #0 and #1"); 1017 1018 /* 1019 * promote to 64-b: 1020 * 1021 * This only applies to the !overflow_at_64bits case, as 1022 * overflow_at_64bits doesn't implement CHAIN events. The 1023 * overflow_at_64bits case just checks that chained counters are 1024 * not incremented when PMCR.LP == 1. 1025 */ 1026 1027 pmu_reset_stats(); 1028 1029 write_regn_el0(pmevtyper, 1, CHAIN | PMEVTYPER_EXCLUDE_EL0); 1030 write_regn_el0(pmevcntr, 0, pre_overflow); 1031 isb(); 1032 mem_access_loop(addr, 200, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1033 report(expect_interrupts(0x1), "expect overflow interrupt"); 1034 1035 /* overflow on odd counter */ 1036 pmu_reset_stats(); 1037 write_regn_el0(pmevcntr, 0, pre_overflow); 1038 write_regn_el0(pmevcntr, 1, all_set); 1039 isb(); 1040 mem_access_loop(addr, 400, pmu.pmcr_ro | PMU_PMCR_E | pmcr_lp); 1041 if (overflow_at_64bits) { 1042 report(expect_interrupts(0x1), 1043 "expect overflow interrupt on even counter"); 1044 report(read_regn_el0(pmevcntr, 1) == all_set, 1045 "Odd counter did not change"); 1046 } else { 1047 report(expect_interrupts(0x3), 1048 "expect overflow interrupt on even and odd counter"); 1049 report(read_regn_el0(pmevcntr, 1) != all_set, 1050 "Odd counter wrapped"); 1051 } 1052 } 1053 #endif 1054 1055 /* 1056 * Ensure that the cycle counter progresses between back-to-back reads. 1057 */ 1058 static bool check_cycles_increase(void) 1059 { 1060 bool success = true; 1061 1062 /* init before event access, this test only cares about cycle count */ 1063 pmu_reset(); 1064 set_pmcntenset(1 << PMU_CYCLE_IDX); 1065 set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */ 1066 1067 set_pmcr(get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E); 1068 isb(); 1069 1070 for (int i = 0; i < NR_SAMPLES; i++) { 1071 uint64_t a, b; 1072 1073 a = get_pmccntr(); 1074 b = get_pmccntr(); 1075 1076 if (a >= b) { 1077 printf("Read %"PRId64" then %"PRId64".\n", a, b); 1078 success = false; 1079 break; 1080 } 1081 } 1082 1083 set_pmcr(get_pmcr() & ~PMU_PMCR_E); 1084 isb(); 1085 1086 return success; 1087 } 1088 1089 /* 1090 * Execute a known number of guest instructions. Only even instruction counts 1091 * greater than or equal to 4 are supported by the in-line assembly code. The 1092 * control register (PMCR_EL0) is initialized with the provided value (allowing 1093 * for example for the cycle counter or event counters to be reset). At the end 1094 * of the exact instruction loop, zero is written to PMCR_EL0 to disable 1095 * counting, allowing the cycle counter or event counters to be read at the 1096 * leisure of the calling code. 1097 */ 1098 static void measure_instrs(int num, uint32_t pmcr) 1099 { 1100 int loop = (num - 2) / 2; 1101 1102 assert(num >= 4 && ((num - 2) % 2 == 0)); 1103 precise_instrs_loop(loop, pmcr); 1104 } 1105 1106 /* 1107 * Measure cycle counts for various known instruction counts. Ensure that the 1108 * cycle counter progresses (similar to check_cycles_increase() but with more 1109 * instructions and using reset and stop controls). If supplied a positive, 1110 * nonzero CPI parameter, it also strictly checks that every measurement matches 1111 * it. Strict CPI checking is used to test -icount mode. 1112 */ 1113 static bool check_cpi(int cpi) 1114 { 1115 uint32_t pmcr = get_pmcr() | PMU_PMCR_LC | PMU_PMCR_C | PMU_PMCR_E; 1116 1117 /* init before event access, this test only cares about cycle count */ 1118 pmu_reset(); 1119 set_pmcntenset(1 << PMU_CYCLE_IDX); 1120 set_pmccfiltr(0); /* count cycles in EL0, EL1, but not EL2 */ 1121 1122 if (cpi > 0) 1123 printf("Checking for CPI=%d.\n", cpi); 1124 printf("instrs : cycles0 cycles1 ...\n"); 1125 1126 for (unsigned int i = 4; i < 300; i += 32) { 1127 uint64_t avg, sum = 0; 1128 1129 printf("%4d:", i); 1130 for (int j = 0; j < NR_SAMPLES; j++) { 1131 uint64_t cycles; 1132 1133 set_pmccntr(0); 1134 measure_instrs(i, pmcr); 1135 cycles = get_pmccntr(); 1136 printf(" %4"PRId64"", cycles); 1137 1138 if (!cycles) { 1139 printf("\ncycles not incrementing!\n"); 1140 return false; 1141 } else if (cpi > 0 && cycles != i * cpi) { 1142 printf("\nunexpected cycle count received!\n"); 1143 return false; 1144 } else if ((cycles >> 32) != 0) { 1145 /* The cycles taken by the loop above should 1146 * fit in 32 bits easily. We check the upper 1147 * 32 bits of the cycle counter to make sure 1148 * there is no supprise. */ 1149 printf("\ncycle count bigger than 32bit!\n"); 1150 return false; 1151 } 1152 1153 sum += cycles; 1154 } 1155 avg = sum / NR_SAMPLES; 1156 printf(" avg=%-4"PRId64" %s=%-3"PRId64"\n", avg, 1157 (avg >= i) ? "cpi" : "ipc", 1158 (avg >= i) ? avg / i : i / avg); 1159 } 1160 1161 return true; 1162 } 1163 1164 static void pmccntr64_test(void) 1165 { 1166 #ifdef __arm__ 1167 if (pmu.version == ID_DFR0_PMU_V3) { 1168 if (ERRATA(9e3f7a296940)) { 1169 write_sysreg(0xdead, PMCCNTR64); 1170 report(read_sysreg(PMCCNTR64) == 0xdead, "pmccntr64"); 1171 } else 1172 report_skip("Skipping unsafe pmccntr64 test. Set ERRATA_9e3f7a296940=y to enable."); 1173 } 1174 #endif 1175 } 1176 1177 /* Return FALSE if no PMU found, otherwise return TRUE */ 1178 static bool pmu_probe(void) 1179 { 1180 uint32_t pmcr; 1181 uint8_t implementer; 1182 1183 pmu.version = get_pmu_version(); 1184 if (pmu.version == ID_DFR0_PMU_NOTIMPL || pmu.version == ID_DFR0_PMU_IMPDEF) 1185 return false; 1186 1187 report_info("PMU version: 0x%x", pmu.version); 1188 1189 pmcr = get_pmcr(); 1190 implementer = (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK; 1191 report_info("PMU implementer/ID code: %#"PRIx32"(\"%c\")/%#"PRIx32, 1192 (pmcr >> PMU_PMCR_IMP_SHIFT) & PMU_PMCR_IMP_MASK, 1193 implementer ? implementer : ' ', 1194 (pmcr >> PMU_PMCR_ID_SHIFT) & PMU_PMCR_ID_MASK); 1195 1196 /* store read-only and RES0 fields of the PMCR bottom-half*/ 1197 pmu.pmcr_ro = pmcr & 0xFFFFFF00; 1198 pmu.nb_implemented_counters = 1199 (pmcr >> PMU_PMCR_N_SHIFT) & PMU_PMCR_N_MASK; 1200 report_info("Implements %d event counters", 1201 pmu.nb_implemented_counters); 1202 1203 return true; 1204 } 1205 1206 static void run_test(const char *name, const char *prefix, 1207 void (*test)(bool), void *arg) 1208 { 1209 report_prefix_push(name); 1210 report_prefix_push(prefix); 1211 1212 test(arg); 1213 1214 report_prefix_pop(); 1215 report_prefix_pop(); 1216 } 1217 1218 static void run_event_test(char *name, void (*test)(bool), 1219 bool overflow_at_64bits) 1220 { 1221 const char *prefix = overflow_at_64bits ? "64-bit overflows" 1222 : "32-bit overflows"; 1223 1224 run_test(name, prefix, test, (void *)overflow_at_64bits); 1225 } 1226 1227 int main(int argc, char *argv[]) 1228 { 1229 int cpi = 0; 1230 1231 if (!pmu_probe()) { 1232 printf("No PMU found, test skipped...\n"); 1233 return report_summary(); 1234 } 1235 1236 if (argc < 2) 1237 report_abort("no test specified"); 1238 1239 report_prefix_push("pmu"); 1240 1241 if (strcmp(argv[1], "cycle-counter") == 0) { 1242 report_prefix_push(argv[1]); 1243 if (argc > 2) 1244 cpi = atol(argv[2]); 1245 report(check_cycles_increase(), 1246 "Monotonically increasing cycle count"); 1247 report(check_cpi(cpi), "Cycle/instruction ratio"); 1248 pmccntr64_test(); 1249 report_prefix_pop(); 1250 } else if (strcmp(argv[1], "pmu-event-introspection") == 0) { 1251 report_prefix_push(argv[1]); 1252 test_event_introspection(); 1253 report_prefix_pop(); 1254 } else if (strcmp(argv[1], "pmu-event-counter-config") == 0) { 1255 report_prefix_push(argv[1]); 1256 test_event_counter_config(); 1257 report_prefix_pop(); 1258 } else if (strcmp(argv[1], "pmu-basic-event-count") == 0) { 1259 run_event_test(argv[1], test_basic_event_count, false); 1260 run_event_test(argv[1], test_basic_event_count, true); 1261 } else if (strcmp(argv[1], "pmu-mem-access-reliability") == 0) { 1262 run_event_test(argv[1], test_mem_access_reliability, false); 1263 run_event_test(argv[1], test_mem_access_reliability, true); 1264 } else if (strcmp(argv[1], "pmu-mem-access") == 0) { 1265 run_event_test(argv[1], test_mem_access, false); 1266 run_event_test(argv[1], test_mem_access, true); 1267 } else if (strcmp(argv[1], "pmu-sw-incr") == 0) { 1268 run_event_test(argv[1], test_sw_incr, false); 1269 run_event_test(argv[1], test_sw_incr, true); 1270 } else if (strcmp(argv[1], "pmu-chained-counters") == 0) { 1271 run_event_test(argv[1], test_chained_counters, false); 1272 } else if (strcmp(argv[1], "pmu-chained-sw-incr") == 0) { 1273 run_event_test(argv[1], test_chained_sw_incr, false); 1274 } else if (strcmp(argv[1], "pmu-chain-promotion") == 0) { 1275 run_event_test(argv[1], test_chain_promotion, false); 1276 } else if (strcmp(argv[1], "pmu-overflow-interrupt") == 0) { 1277 run_event_test(argv[1], test_overflow_interrupt, false); 1278 run_event_test(argv[1], test_overflow_interrupt, true); 1279 } else { 1280 report_abort("Unknown sub-test '%s'", argv[1]); 1281 } 1282 1283 return report_summary(); 1284 } 1285