1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CPU PMU driver for the Apple M1 and derivatives 4 * 5 * Copyright (C) 2021 Google LLC 6 * 7 * Author: Marc Zyngier <maz@kernel.org> 8 * 9 * Most of the information used in this driver was provided by the 10 * Asahi Linux project. The rest was experimentally discovered. 11 */ 12 13 #include <linux/of.h> 14 #include <linux/perf/arm_pmu.h> 15 #include <linux/perf/arm_pmuv3.h> 16 #include <linux/platform_device.h> 17 18 #include <asm/apple_m1_pmu.h> 19 #include <asm/irq_regs.h> 20 #include <asm/perf_event.h> 21 22 #define M1_PMU_NR_COUNTERS 10 23 24 #define M1_PMU_CFG_EVENT GENMASK(7, 0) 25 26 #define ANY_BUT_0_1 GENMASK(9, 2) 27 #define ONLY_2_TO_7 GENMASK(7, 2) 28 #define ONLY_2_4_6 (BIT(2) | BIT(4) | BIT(6)) 29 #define ONLY_5_6_7 (BIT(5) | BIT(6) | BIT(7)) 30 31 /* 32 * Description of the events we actually know about, as well as those with 33 * a specific counter affinity. Yes, this is a grand total of two known 34 * counters, and the rest is anybody's guess. 35 * 36 * Not all counters can count all events. Counters #0 and #1 are wired to 37 * count cycles and instructions respectively, and some events have 38 * bizarre mappings (every other counter, or even *one* counter). These 39 * restrictions equally apply to both P and E cores. 40 * 41 * It is worth noting that the PMUs attached to P and E cores are likely 42 * to be different because the underlying uarches are different. At the 43 * moment, we don't really need to distinguish between the two because we 44 * know next to nothing about the events themselves, and we already have 45 * per cpu-type PMU abstractions. 46 * 47 * If we eventually find out that the events are different across 48 * implementations, we'll have to introduce per cpu-type tables. 49 */ 50 enum m1_pmu_events { 51 M1_PMU_PERFCTR_RETIRE_UOP = 0x1, 52 M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE = 0x2, 53 M1_PMU_PERFCTR_L1I_TLB_FILL = 0x4, 54 M1_PMU_PERFCTR_L1D_TLB_FILL = 0x5, 55 M1_PMU_PERFCTR_MMU_TABLE_WALK_INSTRUCTION = 0x7, 56 M1_PMU_PERFCTR_MMU_TABLE_WALK_DATA = 0x8, 57 M1_PMU_PERFCTR_L2_TLB_MISS_INSTRUCTION = 0xa, 58 M1_PMU_PERFCTR_L2_TLB_MISS_DATA = 0xb, 59 M1_PMU_PERFCTR_MMU_VIRTUAL_MEMORY_FAULT_NONSPEC = 0xd, 60 M1_PMU_PERFCTR_SCHEDULE_UOP = 0x52, 61 M1_PMU_PERFCTR_INTERRUPT_PENDING = 0x6c, 62 M1_PMU_PERFCTR_MAP_STALL_DISPATCH = 0x70, 63 M1_PMU_PERFCTR_MAP_REWIND = 0x75, 64 M1_PMU_PERFCTR_MAP_STALL = 0x76, 65 M1_PMU_PERFCTR_MAP_INT_UOP = 0x7c, 66 M1_PMU_PERFCTR_MAP_LDST_UOP = 0x7d, 67 M1_PMU_PERFCTR_MAP_SIMD_UOP = 0x7e, 68 M1_PMU_PERFCTR_FLUSH_RESTART_OTHER_NONSPEC = 0x84, 69 M1_PMU_PERFCTR_INST_ALL = 0x8c, 70 M1_PMU_PERFCTR_INST_BRANCH = 0x8d, 71 M1_PMU_PERFCTR_INST_BRANCH_CALL = 0x8e, 72 M1_PMU_PERFCTR_INST_BRANCH_RET = 0x8f, 73 M1_PMU_PERFCTR_INST_BRANCH_TAKEN = 0x90, 74 M1_PMU_PERFCTR_INST_BRANCH_INDIR = 0x93, 75 M1_PMU_PERFCTR_INST_BRANCH_COND = 0x94, 76 M1_PMU_PERFCTR_INST_INT_LD = 0x95, 77 M1_PMU_PERFCTR_INST_INT_ST = 0x96, 78 M1_PMU_PERFCTR_INST_INT_ALU = 0x97, 79 M1_PMU_PERFCTR_INST_SIMD_LD = 0x98, 80 M1_PMU_PERFCTR_INST_SIMD_ST = 0x99, 81 M1_PMU_PERFCTR_INST_SIMD_ALU = 0x9a, 82 M1_PMU_PERFCTR_INST_LDST = 0x9b, 83 M1_PMU_PERFCTR_INST_BARRIER = 0x9c, 84 M1_PMU_PERFCTR_UNKNOWN_9f = 0x9f, 85 M1_PMU_PERFCTR_L1D_TLB_ACCESS = 0xa0, 86 M1_PMU_PERFCTR_L1D_TLB_MISS = 0xa1, 87 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST = 0xa2, 88 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD = 0xa3, 89 M1_PMU_PERFCTR_LD_UNIT_UOP = 0xa6, 90 M1_PMU_PERFCTR_ST_UNIT_UOP = 0xa7, 91 M1_PMU_PERFCTR_L1D_CACHE_WRITEBACK = 0xa8, 92 M1_PMU_PERFCTR_LDST_X64_UOP = 0xb1, 93 M1_PMU_PERFCTR_LDST_XPG_UOP = 0xb2, 94 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_SUCC = 0xb3, 95 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_FAIL = 0xb4, 96 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC = 0xbf, 97 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC = 0xc0, 98 M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC = 0xc1, 99 M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC = 0xc4, 100 M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC = 0xc5, 101 M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC = 0xc6, 102 M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC = 0xc8, 103 M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC = 0xca, 104 M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC = 0xcb, 105 M1_PMU_PERFCTR_L1I_TLB_MISS_DEMAND = 0xd4, 106 M1_PMU_PERFCTR_MAP_DISPATCH_BUBBLE = 0xd6, 107 M1_PMU_PERFCTR_L1I_CACHE_MISS_DEMAND = 0xdb, 108 M1_PMU_PERFCTR_FETCH_RESTART = 0xde, 109 M1_PMU_PERFCTR_ST_NT_UOP = 0xe5, 110 M1_PMU_PERFCTR_LD_NT_UOP = 0xe6, 111 M1_PMU_PERFCTR_UNKNOWN_f5 = 0xf5, 112 M1_PMU_PERFCTR_UNKNOWN_f6 = 0xf6, 113 M1_PMU_PERFCTR_UNKNOWN_f7 = 0xf7, 114 M1_PMU_PERFCTR_UNKNOWN_f8 = 0xf8, 115 M1_PMU_PERFCTR_UNKNOWN_fd = 0xfd, 116 M1_PMU_PERFCTR_LAST = M1_PMU_CFG_EVENT, 117 118 /* 119 * From this point onwards, these are not actual HW events, 120 * but attributes that get stored in hw->config_base. 121 */ 122 M1_PMU_CFG_COUNT_USER = BIT(8), 123 M1_PMU_CFG_COUNT_KERNEL = BIT(9), 124 M1_PMU_CFG_COUNT_HOST = BIT(10), 125 M1_PMU_CFG_COUNT_GUEST = BIT(11), 126 }; 127 128 /* 129 * Per-event affinity table. Most events can be installed on counter 130 * 2-9, but there are a number of exceptions. Note that this table 131 * has been created experimentally, and I wouldn't be surprised if more 132 * counters had strange affinities. 133 */ 134 static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = { 135 [0 ... M1_PMU_PERFCTR_LAST] = ANY_BUT_0_1, 136 [M1_PMU_PERFCTR_RETIRE_UOP] = BIT(7), 137 [M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE] = ANY_BUT_0_1 | BIT(0), 138 [M1_PMU_PERFCTR_INST_ALL] = BIT(7) | BIT(1), 139 [M1_PMU_PERFCTR_INST_BRANCH] = ONLY_5_6_7, 140 [M1_PMU_PERFCTR_INST_BRANCH_CALL] = ONLY_5_6_7, 141 [M1_PMU_PERFCTR_INST_BRANCH_RET] = ONLY_5_6_7, 142 [M1_PMU_PERFCTR_INST_BRANCH_TAKEN] = ONLY_5_6_7, 143 [M1_PMU_PERFCTR_INST_BRANCH_INDIR] = ONLY_5_6_7, 144 [M1_PMU_PERFCTR_INST_BRANCH_COND] = ONLY_5_6_7, 145 [M1_PMU_PERFCTR_INST_INT_LD] = ONLY_5_6_7, 146 [M1_PMU_PERFCTR_INST_INT_ST] = BIT(7), 147 [M1_PMU_PERFCTR_INST_INT_ALU] = BIT(7), 148 [M1_PMU_PERFCTR_INST_SIMD_LD] = ONLY_5_6_7, 149 [M1_PMU_PERFCTR_INST_SIMD_ST] = ONLY_5_6_7, 150 [M1_PMU_PERFCTR_INST_SIMD_ALU] = BIT(7), 151 [M1_PMU_PERFCTR_INST_LDST] = BIT(7), 152 [M1_PMU_PERFCTR_INST_BARRIER] = ONLY_5_6_7, 153 [M1_PMU_PERFCTR_UNKNOWN_9f] = BIT(7), 154 [M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC] = ONLY_5_6_7, 155 [M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC] = ONLY_5_6_7, 156 [M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC] = ONLY_5_6_7, 157 [M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC] = ONLY_5_6_7, 158 [M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC] = ONLY_5_6_7, 159 [M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, 160 [M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, 161 [M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, 162 [M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC] = ONLY_5_6_7, 163 [M1_PMU_PERFCTR_UNKNOWN_f5] = ONLY_2_4_6, 164 [M1_PMU_PERFCTR_UNKNOWN_f6] = ONLY_2_4_6, 165 [M1_PMU_PERFCTR_UNKNOWN_f7] = ONLY_2_4_6, 166 [M1_PMU_PERFCTR_UNKNOWN_f8] = ONLY_2_TO_7, 167 [M1_PMU_PERFCTR_UNKNOWN_fd] = ONLY_2_4_6, 168 }; 169 170 static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = { 171 PERF_MAP_ALL_UNSUPPORTED, 172 [PERF_COUNT_HW_CPU_CYCLES] = M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE, 173 [PERF_COUNT_HW_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_ALL, 174 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_BRANCH, 175 [PERF_COUNT_HW_BRANCH_MISSES] = M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC, 176 }; 177 178 #define M1_PMUV3_EVENT_MAP(pmuv3_event, m1_event) \ 179 [ARMV8_PMUV3_PERFCTR_##pmuv3_event] = M1_PMU_PERFCTR_##m1_event 180 181 static const u16 m1_pmu_pmceid_map[ARMV8_PMUV3_MAX_COMMON_EVENTS] = { 182 [0 ... ARMV8_PMUV3_MAX_COMMON_EVENTS - 1] = HW_OP_UNSUPPORTED, 183 M1_PMUV3_EVENT_MAP(INST_RETIRED, INST_ALL), 184 M1_PMUV3_EVENT_MAP(CPU_CYCLES, CORE_ACTIVE_CYCLE), 185 M1_PMUV3_EVENT_MAP(BR_RETIRED, INST_BRANCH), 186 M1_PMUV3_EVENT_MAP(BR_MIS_PRED_RETIRED, BRANCH_MISPRED_NONSPEC), 187 }; 188 189 /* sysfs definitions */ 190 static ssize_t m1_pmu_events_sysfs_show(struct device *dev, 191 struct device_attribute *attr, 192 char *page) 193 { 194 struct perf_pmu_events_attr *pmu_attr; 195 196 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 197 198 return sprintf(page, "event=0x%04llx\n", pmu_attr->id); 199 } 200 201 #define M1_PMU_EVENT_ATTR(name, config) \ 202 PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config) 203 204 static struct attribute *m1_pmu_event_attrs[] = { 205 M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE), 206 M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INST_ALL), 207 NULL, 208 }; 209 210 static const struct attribute_group m1_pmu_events_attr_group = { 211 .name = "events", 212 .attrs = m1_pmu_event_attrs, 213 }; 214 215 PMU_FORMAT_ATTR(event, "config:0-7"); 216 217 static struct attribute *m1_pmu_format_attrs[] = { 218 &format_attr_event.attr, 219 NULL, 220 }; 221 222 static const struct attribute_group m1_pmu_format_attr_group = { 223 .name = "format", 224 .attrs = m1_pmu_format_attrs, 225 }; 226 227 /* Low level accessors. No synchronisation. */ 228 #define PMU_READ_COUNTER(_idx) \ 229 case _idx: return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1) 230 231 #define PMU_WRITE_COUNTER(_val, _idx) \ 232 case _idx: \ 233 write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1); \ 234 return 235 236 static u64 m1_pmu_read_hw_counter(unsigned int index) 237 { 238 switch (index) { 239 PMU_READ_COUNTER(0); 240 PMU_READ_COUNTER(1); 241 PMU_READ_COUNTER(2); 242 PMU_READ_COUNTER(3); 243 PMU_READ_COUNTER(4); 244 PMU_READ_COUNTER(5); 245 PMU_READ_COUNTER(6); 246 PMU_READ_COUNTER(7); 247 PMU_READ_COUNTER(8); 248 PMU_READ_COUNTER(9); 249 } 250 251 BUG(); 252 } 253 254 static void m1_pmu_write_hw_counter(u64 val, unsigned int index) 255 { 256 switch (index) { 257 PMU_WRITE_COUNTER(val, 0); 258 PMU_WRITE_COUNTER(val, 1); 259 PMU_WRITE_COUNTER(val, 2); 260 PMU_WRITE_COUNTER(val, 3); 261 PMU_WRITE_COUNTER(val, 4); 262 PMU_WRITE_COUNTER(val, 5); 263 PMU_WRITE_COUNTER(val, 6); 264 PMU_WRITE_COUNTER(val, 7); 265 PMU_WRITE_COUNTER(val, 8); 266 PMU_WRITE_COUNTER(val, 9); 267 } 268 269 BUG(); 270 } 271 272 #define get_bit_offset(index, mask) (__ffs(mask) + (index)) 273 274 static void __m1_pmu_enable_counter(unsigned int index, bool en) 275 { 276 u64 val, bit; 277 278 switch (index) { 279 case 0 ... 7: 280 bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7)); 281 break; 282 case 8 ... 9: 283 bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9)); 284 break; 285 default: 286 BUG(); 287 } 288 289 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 290 291 if (en) 292 val |= bit; 293 else 294 val &= ~bit; 295 296 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); 297 } 298 299 static void m1_pmu_enable_counter(unsigned int index) 300 { 301 __m1_pmu_enable_counter(index, true); 302 } 303 304 static void m1_pmu_disable_counter(unsigned int index) 305 { 306 __m1_pmu_enable_counter(index, false); 307 } 308 309 static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en) 310 { 311 u64 val, bit; 312 313 switch (index) { 314 case 0 ... 7: 315 bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7)); 316 break; 317 case 8 ... 9: 318 bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9)); 319 break; 320 default: 321 BUG(); 322 } 323 324 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 325 326 if (en) 327 val |= bit; 328 else 329 val &= ~bit; 330 331 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); 332 } 333 334 static void m1_pmu_enable_counter_interrupt(unsigned int index) 335 { 336 __m1_pmu_enable_counter_interrupt(index, true); 337 } 338 339 static void m1_pmu_disable_counter_interrupt(unsigned int index) 340 { 341 __m1_pmu_enable_counter_interrupt(index, false); 342 } 343 344 static void __m1_pmu_configure_event_filter(unsigned int index, bool user, 345 bool kernel, bool host) 346 { 347 u64 clear, set, user_bit, kernel_bit; 348 349 switch (index) { 350 case 0 ... 7: 351 user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7)); 352 kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7)); 353 break; 354 case 8 ... 9: 355 user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9)); 356 kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9)); 357 break; 358 default: 359 BUG(); 360 } 361 362 clear = set = 0; 363 if (user) 364 set |= user_bit; 365 else 366 clear |= user_bit; 367 368 if (kernel) 369 set |= kernel_bit; 370 else 371 clear |= kernel_bit; 372 373 if (host) 374 sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL1, clear, set); 375 else if (is_kernel_in_hyp_mode()) 376 sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL12, clear, set); 377 } 378 379 static void __m1_pmu_configure_eventsel(unsigned int index, u8 event) 380 { 381 u64 clear = 0, set = 0; 382 int shift; 383 384 /* 385 * Counters 0 and 1 have fixed events. For anything else, 386 * place the event at the expected location in the relevant 387 * register (PMESR0 holds the event configuration for counters 388 * 2-5, resp. PMESR1 for counters 6-9). 389 */ 390 switch (index) { 391 case 0 ... 1: 392 break; 393 case 2 ... 5: 394 shift = (index - 2) * 8; 395 clear |= (u64)0xff << shift; 396 set |= (u64)event << shift; 397 sysreg_clear_set_s(SYS_IMP_APL_PMESR0_EL1, clear, set); 398 break; 399 case 6 ... 9: 400 shift = (index - 6) * 8; 401 clear |= (u64)0xff << shift; 402 set |= (u64)event << shift; 403 sysreg_clear_set_s(SYS_IMP_APL_PMESR1_EL1, clear, set); 404 break; 405 } 406 } 407 408 static void m1_pmu_configure_counter(unsigned int index, unsigned long config_base) 409 { 410 bool kernel = config_base & M1_PMU_CFG_COUNT_KERNEL; 411 bool guest = config_base & M1_PMU_CFG_COUNT_GUEST; 412 bool host = config_base & M1_PMU_CFG_COUNT_HOST; 413 bool user = config_base & M1_PMU_CFG_COUNT_USER; 414 u8 evt = config_base & M1_PMU_CFG_EVENT; 415 416 __m1_pmu_configure_event_filter(index, user && host, kernel && host, true); 417 __m1_pmu_configure_event_filter(index, user && guest, kernel && guest, false); 418 __m1_pmu_configure_eventsel(index, evt); 419 } 420 421 /* arm_pmu backend */ 422 static void m1_pmu_enable_event(struct perf_event *event) 423 { 424 bool user, kernel; 425 u8 evt; 426 427 evt = event->hw.config_base & M1_PMU_CFG_EVENT; 428 user = event->hw.config_base & M1_PMU_CFG_COUNT_USER; 429 kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL; 430 431 m1_pmu_configure_counter(event->hw.idx, event->hw.config_base); 432 m1_pmu_enable_counter(event->hw.idx); 433 m1_pmu_enable_counter_interrupt(event->hw.idx); 434 isb(); 435 } 436 437 static void m1_pmu_disable_event(struct perf_event *event) 438 { 439 m1_pmu_disable_counter_interrupt(event->hw.idx); 440 m1_pmu_disable_counter(event->hw.idx); 441 isb(); 442 } 443 444 static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu) 445 { 446 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); 447 struct pt_regs *regs; 448 u64 overflow, state; 449 int idx; 450 451 overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1); 452 if (!overflow) { 453 /* Spurious interrupt? */ 454 state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 455 state &= ~PMCR0_IACT; 456 write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1); 457 isb(); 458 return IRQ_NONE; 459 } 460 461 cpu_pmu->stop(cpu_pmu); 462 463 regs = get_irq_regs(); 464 465 for_each_set_bit(idx, cpu_pmu->cntr_mask, M1_PMU_NR_COUNTERS) { 466 struct perf_event *event = cpuc->events[idx]; 467 struct perf_sample_data data; 468 469 if (!event) 470 continue; 471 472 armpmu_event_update(event); 473 perf_sample_data_init(&data, 0, event->hw.last_period); 474 if (!armpmu_event_set_period(event)) 475 continue; 476 477 perf_event_overflow(event, &data, regs); 478 } 479 480 cpu_pmu->start(cpu_pmu); 481 482 return IRQ_HANDLED; 483 } 484 485 static u64 m1_pmu_read_counter(struct perf_event *event) 486 { 487 return m1_pmu_read_hw_counter(event->hw.idx); 488 } 489 490 static void m1_pmu_write_counter(struct perf_event *event, u64 value) 491 { 492 m1_pmu_write_hw_counter(value, event->hw.idx); 493 isb(); 494 } 495 496 static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc, 497 struct perf_event *event) 498 { 499 unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT; 500 unsigned long affinity = m1_pmu_event_affinity[evtype]; 501 int idx; 502 503 /* 504 * Place the event on the first free counter that can count 505 * this event. 506 * 507 * We could do a better job if we had a view of all the events 508 * counting on the PMU at any given time, and by placing the 509 * most constraining events first. 510 */ 511 for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) { 512 if (!test_and_set_bit(idx, cpuc->used_mask)) 513 return idx; 514 } 515 516 return -EAGAIN; 517 } 518 519 static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc, 520 struct perf_event *event) 521 { 522 clear_bit(event->hw.idx, cpuc->used_mask); 523 } 524 525 static void __m1_pmu_set_mode(u8 mode) 526 { 527 u64 val; 528 529 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 530 val &= ~(PMCR0_IMODE | PMCR0_IACT); 531 val |= FIELD_PREP(PMCR0_IMODE, mode); 532 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); 533 isb(); 534 } 535 536 static void m1_pmu_start(struct arm_pmu *cpu_pmu) 537 { 538 __m1_pmu_set_mode(PMCR0_IMODE_FIQ); 539 } 540 541 static void m1_pmu_stop(struct arm_pmu *cpu_pmu) 542 { 543 __m1_pmu_set_mode(PMCR0_IMODE_OFF); 544 } 545 546 static int m1_pmu_map_event(struct perf_event *event) 547 { 548 /* 549 * Although the counters are 48bit wide, bit 47 is what 550 * triggers the overflow interrupt. Advertise the counters 551 * being 47bit wide to mimick the behaviour of the ARM PMU. 552 */ 553 event->hw.flags |= ARMPMU_EVT_47BIT; 554 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); 555 } 556 557 static int m2_pmu_map_event(struct perf_event *event) 558 { 559 /* 560 * Same deal as the above, except that M2 has 64bit counters. 561 * Which, as far as we're concerned, actually means 63 bits. 562 * Yes, this is getting awkward. 563 */ 564 event->hw.flags |= ARMPMU_EVT_63BIT; 565 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); 566 } 567 568 static int m1_pmu_map_pmuv3_event(unsigned int eventsel) 569 { 570 u16 m1_event = HW_OP_UNSUPPORTED; 571 572 if (eventsel < ARMV8_PMUV3_MAX_COMMON_EVENTS) 573 m1_event = m1_pmu_pmceid_map[eventsel]; 574 575 return m1_event == HW_OP_UNSUPPORTED ? -EOPNOTSUPP : m1_event; 576 } 577 578 static void m1_pmu_init_pmceid(struct arm_pmu *pmu) 579 { 580 unsigned int event; 581 582 for (event = 0; event < ARMV8_PMUV3_MAX_COMMON_EVENTS; event++) { 583 if (m1_pmu_map_pmuv3_event(event) >= 0) 584 set_bit(event, pmu->pmceid_bitmap); 585 } 586 } 587 588 static void m1_pmu_reset(void *info) 589 { 590 int i; 591 592 __m1_pmu_set_mode(PMCR0_IMODE_OFF); 593 594 for (i = 0; i < M1_PMU_NR_COUNTERS; i++) { 595 m1_pmu_disable_counter(i); 596 m1_pmu_disable_counter_interrupt(i); 597 m1_pmu_write_hw_counter(0, i); 598 } 599 600 isb(); 601 } 602 603 static int m1_pmu_set_event_filter(struct hw_perf_event *event, 604 struct perf_event_attr *attr) 605 { 606 unsigned long config_base = 0; 607 608 if (!attr->exclude_guest && !is_kernel_in_hyp_mode()) { 609 pr_debug("ARM performance counters do not support mode exclusion\n"); 610 return -EOPNOTSUPP; 611 } 612 if (!attr->exclude_kernel) 613 config_base |= M1_PMU_CFG_COUNT_KERNEL; 614 if (!attr->exclude_user) 615 config_base |= M1_PMU_CFG_COUNT_USER; 616 if (!attr->exclude_host) 617 config_base |= M1_PMU_CFG_COUNT_HOST; 618 if (!attr->exclude_guest) 619 config_base |= M1_PMU_CFG_COUNT_GUEST; 620 621 event->config_base = config_base; 622 623 return 0; 624 } 625 626 static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags) 627 { 628 cpu_pmu->handle_irq = m1_pmu_handle_irq; 629 cpu_pmu->enable = m1_pmu_enable_event; 630 cpu_pmu->disable = m1_pmu_disable_event; 631 cpu_pmu->read_counter = m1_pmu_read_counter; 632 cpu_pmu->write_counter = m1_pmu_write_counter; 633 cpu_pmu->get_event_idx = m1_pmu_get_event_idx; 634 cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx; 635 cpu_pmu->start = m1_pmu_start; 636 cpu_pmu->stop = m1_pmu_stop; 637 638 if (flags & ARMPMU_EVT_47BIT) 639 cpu_pmu->map_event = m1_pmu_map_event; 640 else if (flags & ARMPMU_EVT_63BIT) 641 cpu_pmu->map_event = m2_pmu_map_event; 642 else 643 return WARN_ON(-EINVAL); 644 645 cpu_pmu->reset = m1_pmu_reset; 646 cpu_pmu->set_event_filter = m1_pmu_set_event_filter; 647 648 cpu_pmu->map_pmuv3_event = m1_pmu_map_pmuv3_event; 649 m1_pmu_init_pmceid(cpu_pmu); 650 651 bitmap_set(cpu_pmu->cntr_mask, 0, M1_PMU_NR_COUNTERS); 652 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group; 653 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group; 654 return 0; 655 } 656 657 /* Device driver gunk */ 658 static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu) 659 { 660 cpu_pmu->name = "apple_icestorm_pmu"; 661 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); 662 } 663 664 static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu) 665 { 666 cpu_pmu->name = "apple_firestorm_pmu"; 667 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); 668 } 669 670 static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) 671 { 672 cpu_pmu->name = "apple_avalanche_pmu"; 673 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); 674 } 675 676 static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) 677 { 678 cpu_pmu->name = "apple_blizzard_pmu"; 679 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); 680 } 681 682 static const struct of_device_id m1_pmu_of_device_ids[] = { 683 { .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, }, 684 { .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, }, 685 { .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, }, 686 { .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, }, 687 { }, 688 }; 689 MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids); 690 691 static int m1_pmu_device_probe(struct platform_device *pdev) 692 { 693 return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL); 694 } 695 696 static struct platform_driver m1_pmu_driver = { 697 .driver = { 698 .name = "apple-m1-cpu-pmu", 699 .of_match_table = m1_pmu_of_device_ids, 700 .suppress_bind_attrs = true, 701 }, 702 .probe = m1_pmu_device_probe, 703 }; 704 705 module_platform_driver(m1_pmu_driver); 706