1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * CPU PMU driver for the Apple M1 and derivatives
4 *
5 * Copyright (C) 2021 Google LLC
6 *
7 * Author: Marc Zyngier <maz@kernel.org>
8 *
9 * Most of the information used in this driver was provided by the
10 * Asahi Linux project. The rest was experimentally discovered.
11 */
12
13 #include <linux/of.h>
14 #include <linux/perf/arm_pmu.h>
15 #include <linux/perf/arm_pmuv3.h>
16 #include <linux/platform_device.h>
17
18 #include <asm/apple_m1_pmu.h>
19 #include <asm/irq_regs.h>
20 #include <asm/perf_event.h>
21
22 #define M1_PMU_NR_COUNTERS 10
23
24 #define M1_PMU_CFG_EVENT GENMASK(7, 0)
25
26 #define ANY_BUT_0_1 GENMASK(9, 2)
27 #define ONLY_2_TO_7 GENMASK(7, 2)
28 #define ONLY_2_4_6 (BIT(2) | BIT(4) | BIT(6))
29 #define ONLY_5_6_7 (BIT(5) | BIT(6) | BIT(7))
30
31 /*
32 * Description of the events we actually know about, as well as those with
33 * a specific counter affinity. Yes, this is a grand total of two known
34 * counters, and the rest is anybody's guess.
35 *
36 * Not all counters can count all events. Counters #0 and #1 are wired to
37 * count cycles and instructions respectively, and some events have
38 * bizarre mappings (every other counter, or even *one* counter). These
39 * restrictions equally apply to both P and E cores.
40 *
41 * It is worth noting that the PMUs attached to P and E cores are likely
42 * to be different because the underlying uarches are different. At the
43 * moment, we don't really need to distinguish between the two because we
44 * know next to nothing about the events themselves, and we already have
45 * per cpu-type PMU abstractions.
46 *
47 * If we eventually find out that the events are different across
48 * implementations, we'll have to introduce per cpu-type tables.
49 */
50 enum m1_pmu_events {
51 M1_PMU_PERFCTR_RETIRE_UOP = 0x1,
52 M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE = 0x2,
53 M1_PMU_PERFCTR_L1I_TLB_FILL = 0x4,
54 M1_PMU_PERFCTR_L1D_TLB_FILL = 0x5,
55 M1_PMU_PERFCTR_MMU_TABLE_WALK_INSTRUCTION = 0x7,
56 M1_PMU_PERFCTR_MMU_TABLE_WALK_DATA = 0x8,
57 M1_PMU_PERFCTR_L2_TLB_MISS_INSTRUCTION = 0xa,
58 M1_PMU_PERFCTR_L2_TLB_MISS_DATA = 0xb,
59 M1_PMU_PERFCTR_MMU_VIRTUAL_MEMORY_FAULT_NONSPEC = 0xd,
60 M1_PMU_PERFCTR_SCHEDULE_UOP = 0x52,
61 M1_PMU_PERFCTR_INTERRUPT_PENDING = 0x6c,
62 M1_PMU_PERFCTR_MAP_STALL_DISPATCH = 0x70,
63 M1_PMU_PERFCTR_MAP_REWIND = 0x75,
64 M1_PMU_PERFCTR_MAP_STALL = 0x76,
65 M1_PMU_PERFCTR_MAP_INT_UOP = 0x7c,
66 M1_PMU_PERFCTR_MAP_LDST_UOP = 0x7d,
67 M1_PMU_PERFCTR_MAP_SIMD_UOP = 0x7e,
68 M1_PMU_PERFCTR_FLUSH_RESTART_OTHER_NONSPEC = 0x84,
69 M1_PMU_PERFCTR_INST_ALL = 0x8c,
70 M1_PMU_PERFCTR_INST_BRANCH = 0x8d,
71 M1_PMU_PERFCTR_INST_BRANCH_CALL = 0x8e,
72 M1_PMU_PERFCTR_INST_BRANCH_RET = 0x8f,
73 M1_PMU_PERFCTR_INST_BRANCH_TAKEN = 0x90,
74 M1_PMU_PERFCTR_INST_BRANCH_INDIR = 0x93,
75 M1_PMU_PERFCTR_INST_BRANCH_COND = 0x94,
76 M1_PMU_PERFCTR_INST_INT_LD = 0x95,
77 M1_PMU_PERFCTR_INST_INT_ST = 0x96,
78 M1_PMU_PERFCTR_INST_INT_ALU = 0x97,
79 M1_PMU_PERFCTR_INST_SIMD_LD = 0x98,
80 M1_PMU_PERFCTR_INST_SIMD_ST = 0x99,
81 M1_PMU_PERFCTR_INST_SIMD_ALU = 0x9a,
82 M1_PMU_PERFCTR_INST_LDST = 0x9b,
83 M1_PMU_PERFCTR_INST_BARRIER = 0x9c,
84 M1_PMU_PERFCTR_UNKNOWN_9f = 0x9f,
85 M1_PMU_PERFCTR_L1D_TLB_ACCESS = 0xa0,
86 M1_PMU_PERFCTR_L1D_TLB_MISS = 0xa1,
87 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST = 0xa2,
88 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD = 0xa3,
89 M1_PMU_PERFCTR_LD_UNIT_UOP = 0xa6,
90 M1_PMU_PERFCTR_ST_UNIT_UOP = 0xa7,
91 M1_PMU_PERFCTR_L1D_CACHE_WRITEBACK = 0xa8,
92 M1_PMU_PERFCTR_LDST_X64_UOP = 0xb1,
93 M1_PMU_PERFCTR_LDST_XPG_UOP = 0xb2,
94 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_SUCC = 0xb3,
95 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_FAIL = 0xb4,
96 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC = 0xbf,
97 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC = 0xc0,
98 M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC = 0xc1,
99 M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC = 0xc4,
100 M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC = 0xc5,
101 M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC = 0xc6,
102 M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC = 0xc8,
103 M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC = 0xca,
104 M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC = 0xcb,
105 M1_PMU_PERFCTR_L1I_TLB_MISS_DEMAND = 0xd4,
106 M1_PMU_PERFCTR_MAP_DISPATCH_BUBBLE = 0xd6,
107 M1_PMU_PERFCTR_L1I_CACHE_MISS_DEMAND = 0xdb,
108 M1_PMU_PERFCTR_FETCH_RESTART = 0xde,
109 M1_PMU_PERFCTR_ST_NT_UOP = 0xe5,
110 M1_PMU_PERFCTR_LD_NT_UOP = 0xe6,
111 M1_PMU_PERFCTR_UNKNOWN_f5 = 0xf5,
112 M1_PMU_PERFCTR_UNKNOWN_f6 = 0xf6,
113 M1_PMU_PERFCTR_UNKNOWN_f7 = 0xf7,
114 M1_PMU_PERFCTR_UNKNOWN_f8 = 0xf8,
115 M1_PMU_PERFCTR_UNKNOWN_fd = 0xfd,
116 M1_PMU_PERFCTR_LAST = M1_PMU_CFG_EVENT,
117
118 /*
119 * From this point onwards, these are not actual HW events,
120 * but attributes that get stored in hw->config_base.
121 */
122 M1_PMU_CFG_COUNT_USER = BIT(8),
123 M1_PMU_CFG_COUNT_KERNEL = BIT(9),
124 M1_PMU_CFG_COUNT_HOST = BIT(10),
125 M1_PMU_CFG_COUNT_GUEST = BIT(11),
126 };
127
128 /*
129 * Per-event affinity table. Most events can be installed on counter
130 * 2-9, but there are a number of exceptions. Note that this table
131 * has been created experimentally, and I wouldn't be surprised if more
132 * counters had strange affinities.
133 */
134 static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = {
135 [0 ... M1_PMU_PERFCTR_LAST] = ANY_BUT_0_1,
136 [M1_PMU_PERFCTR_RETIRE_UOP] = BIT(7),
137 [M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE] = ANY_BUT_0_1 | BIT(0),
138 [M1_PMU_PERFCTR_INST_ALL] = BIT(7) | BIT(1),
139 [M1_PMU_PERFCTR_INST_BRANCH] = ONLY_5_6_7,
140 [M1_PMU_PERFCTR_INST_BRANCH_CALL] = ONLY_5_6_7,
141 [M1_PMU_PERFCTR_INST_BRANCH_RET] = ONLY_5_6_7,
142 [M1_PMU_PERFCTR_INST_BRANCH_TAKEN] = ONLY_5_6_7,
143 [M1_PMU_PERFCTR_INST_BRANCH_INDIR] = ONLY_5_6_7,
144 [M1_PMU_PERFCTR_INST_BRANCH_COND] = ONLY_5_6_7,
145 [M1_PMU_PERFCTR_INST_INT_LD] = ONLY_5_6_7,
146 [M1_PMU_PERFCTR_INST_INT_ST] = BIT(7),
147 [M1_PMU_PERFCTR_INST_INT_ALU] = BIT(7),
148 [M1_PMU_PERFCTR_INST_SIMD_LD] = ONLY_5_6_7,
149 [M1_PMU_PERFCTR_INST_SIMD_ST] = ONLY_5_6_7,
150 [M1_PMU_PERFCTR_INST_SIMD_ALU] = BIT(7),
151 [M1_PMU_PERFCTR_INST_LDST] = BIT(7),
152 [M1_PMU_PERFCTR_INST_BARRIER] = ONLY_5_6_7,
153 [M1_PMU_PERFCTR_UNKNOWN_9f] = BIT(7),
154 [M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC] = ONLY_5_6_7,
155 [M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC] = ONLY_5_6_7,
156 [M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC] = ONLY_5_6_7,
157 [M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC] = ONLY_5_6_7,
158 [M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC] = ONLY_5_6_7,
159 [M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7,
160 [M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7,
161 [M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7,
162 [M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC] = ONLY_5_6_7,
163 [M1_PMU_PERFCTR_UNKNOWN_f5] = ONLY_2_4_6,
164 [M1_PMU_PERFCTR_UNKNOWN_f6] = ONLY_2_4_6,
165 [M1_PMU_PERFCTR_UNKNOWN_f7] = ONLY_2_4_6,
166 [M1_PMU_PERFCTR_UNKNOWN_f8] = ONLY_2_TO_7,
167 [M1_PMU_PERFCTR_UNKNOWN_fd] = ONLY_2_4_6,
168 };
169
170 static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = {
171 PERF_MAP_ALL_UNSUPPORTED,
172 [PERF_COUNT_HW_CPU_CYCLES] = M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE,
173 [PERF_COUNT_HW_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_ALL,
174 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_BRANCH,
175 [PERF_COUNT_HW_BRANCH_MISSES] = M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC,
176 };
177
178 #define M1_PMUV3_EVENT_MAP(pmuv3_event, m1_event) \
179 [ARMV8_PMUV3_PERFCTR_##pmuv3_event] = M1_PMU_PERFCTR_##m1_event
180
181 static const u16 m1_pmu_pmceid_map[ARMV8_PMUV3_MAX_COMMON_EVENTS] = {
182 [0 ... ARMV8_PMUV3_MAX_COMMON_EVENTS - 1] = HW_OP_UNSUPPORTED,
183 M1_PMUV3_EVENT_MAP(INST_RETIRED, INST_ALL),
184 M1_PMUV3_EVENT_MAP(CPU_CYCLES, CORE_ACTIVE_CYCLE),
185 M1_PMUV3_EVENT_MAP(BR_RETIRED, INST_BRANCH),
186 M1_PMUV3_EVENT_MAP(BR_MIS_PRED_RETIRED, BRANCH_MISPRED_NONSPEC),
187 };
188
189 /* sysfs definitions */
m1_pmu_events_sysfs_show(struct device * dev,struct device_attribute * attr,char * page)190 static ssize_t m1_pmu_events_sysfs_show(struct device *dev,
191 struct device_attribute *attr,
192 char *page)
193 {
194 struct perf_pmu_events_attr *pmu_attr;
195
196 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
197
198 return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
199 }
200
201 #define M1_PMU_EVENT_ATTR(name, config) \
202 PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config)
203
204 static struct attribute *m1_pmu_event_attrs[] = {
205 M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE),
206 M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INST_ALL),
207 NULL,
208 };
209
210 static const struct attribute_group m1_pmu_events_attr_group = {
211 .name = "events",
212 .attrs = m1_pmu_event_attrs,
213 };
214
215 PMU_FORMAT_ATTR(event, "config:0-7");
216
217 static struct attribute *m1_pmu_format_attrs[] = {
218 &format_attr_event.attr,
219 NULL,
220 };
221
222 static const struct attribute_group m1_pmu_format_attr_group = {
223 .name = "format",
224 .attrs = m1_pmu_format_attrs,
225 };
226
227 /* Low level accessors. No synchronisation. */
228 #define PMU_READ_COUNTER(_idx) \
229 case _idx: return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1)
230
231 #define PMU_WRITE_COUNTER(_val, _idx) \
232 case _idx: \
233 write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1); \
234 return
235
m1_pmu_read_hw_counter(unsigned int index)236 static u64 m1_pmu_read_hw_counter(unsigned int index)
237 {
238 switch (index) {
239 PMU_READ_COUNTER(0);
240 PMU_READ_COUNTER(1);
241 PMU_READ_COUNTER(2);
242 PMU_READ_COUNTER(3);
243 PMU_READ_COUNTER(4);
244 PMU_READ_COUNTER(5);
245 PMU_READ_COUNTER(6);
246 PMU_READ_COUNTER(7);
247 PMU_READ_COUNTER(8);
248 PMU_READ_COUNTER(9);
249 }
250
251 BUG();
252 }
253
m1_pmu_write_hw_counter(u64 val,unsigned int index)254 static void m1_pmu_write_hw_counter(u64 val, unsigned int index)
255 {
256 switch (index) {
257 PMU_WRITE_COUNTER(val, 0);
258 PMU_WRITE_COUNTER(val, 1);
259 PMU_WRITE_COUNTER(val, 2);
260 PMU_WRITE_COUNTER(val, 3);
261 PMU_WRITE_COUNTER(val, 4);
262 PMU_WRITE_COUNTER(val, 5);
263 PMU_WRITE_COUNTER(val, 6);
264 PMU_WRITE_COUNTER(val, 7);
265 PMU_WRITE_COUNTER(val, 8);
266 PMU_WRITE_COUNTER(val, 9);
267 }
268
269 BUG();
270 }
271
272 #define get_bit_offset(index, mask) (__ffs(mask) + (index))
273
__m1_pmu_enable_counter(unsigned int index,bool en)274 static void __m1_pmu_enable_counter(unsigned int index, bool en)
275 {
276 u64 val, bit;
277
278 switch (index) {
279 case 0 ... 7:
280 bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7));
281 break;
282 case 8 ... 9:
283 bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9));
284 break;
285 default:
286 BUG();
287 }
288
289 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
290
291 if (en)
292 val |= bit;
293 else
294 val &= ~bit;
295
296 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
297 }
298
m1_pmu_enable_counter(unsigned int index)299 static void m1_pmu_enable_counter(unsigned int index)
300 {
301 __m1_pmu_enable_counter(index, true);
302 }
303
m1_pmu_disable_counter(unsigned int index)304 static void m1_pmu_disable_counter(unsigned int index)
305 {
306 __m1_pmu_enable_counter(index, false);
307 }
308
__m1_pmu_enable_counter_interrupt(unsigned int index,bool en)309 static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en)
310 {
311 u64 val, bit;
312
313 switch (index) {
314 case 0 ... 7:
315 bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7));
316 break;
317 case 8 ... 9:
318 bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9));
319 break;
320 default:
321 BUG();
322 }
323
324 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
325
326 if (en)
327 val |= bit;
328 else
329 val &= ~bit;
330
331 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
332 }
333
m1_pmu_enable_counter_interrupt(unsigned int index)334 static void m1_pmu_enable_counter_interrupt(unsigned int index)
335 {
336 __m1_pmu_enable_counter_interrupt(index, true);
337 }
338
m1_pmu_disable_counter_interrupt(unsigned int index)339 static void m1_pmu_disable_counter_interrupt(unsigned int index)
340 {
341 __m1_pmu_enable_counter_interrupt(index, false);
342 }
343
__m1_pmu_configure_event_filter(unsigned int index,bool user,bool kernel,bool host)344 static void __m1_pmu_configure_event_filter(unsigned int index, bool user,
345 bool kernel, bool host)
346 {
347 u64 clear, set, user_bit, kernel_bit;
348
349 switch (index) {
350 case 0 ... 7:
351 user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7));
352 kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7));
353 break;
354 case 8 ... 9:
355 user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9));
356 kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9));
357 break;
358 default:
359 BUG();
360 }
361
362 clear = set = 0;
363 if (user)
364 set |= user_bit;
365 else
366 clear |= user_bit;
367
368 if (kernel)
369 set |= kernel_bit;
370 else
371 clear |= kernel_bit;
372
373 if (host)
374 sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL1, clear, set);
375 else if (is_kernel_in_hyp_mode())
376 sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL12, clear, set);
377 }
378
__m1_pmu_configure_eventsel(unsigned int index,u8 event)379 static void __m1_pmu_configure_eventsel(unsigned int index, u8 event)
380 {
381 u64 clear = 0, set = 0;
382 int shift;
383
384 /*
385 * Counters 0 and 1 have fixed events. For anything else,
386 * place the event at the expected location in the relevant
387 * register (PMESR0 holds the event configuration for counters
388 * 2-5, resp. PMESR1 for counters 6-9).
389 */
390 switch (index) {
391 case 0 ... 1:
392 break;
393 case 2 ... 5:
394 shift = (index - 2) * 8;
395 clear |= (u64)0xff << shift;
396 set |= (u64)event << shift;
397 sysreg_clear_set_s(SYS_IMP_APL_PMESR0_EL1, clear, set);
398 break;
399 case 6 ... 9:
400 shift = (index - 6) * 8;
401 clear |= (u64)0xff << shift;
402 set |= (u64)event << shift;
403 sysreg_clear_set_s(SYS_IMP_APL_PMESR1_EL1, clear, set);
404 break;
405 }
406 }
407
m1_pmu_configure_counter(unsigned int index,unsigned long config_base)408 static void m1_pmu_configure_counter(unsigned int index, unsigned long config_base)
409 {
410 bool kernel = config_base & M1_PMU_CFG_COUNT_KERNEL;
411 bool guest = config_base & M1_PMU_CFG_COUNT_GUEST;
412 bool host = config_base & M1_PMU_CFG_COUNT_HOST;
413 bool user = config_base & M1_PMU_CFG_COUNT_USER;
414 u8 evt = config_base & M1_PMU_CFG_EVENT;
415
416 __m1_pmu_configure_event_filter(index, user && host, kernel && host, true);
417 __m1_pmu_configure_event_filter(index, user && guest, kernel && guest, false);
418 __m1_pmu_configure_eventsel(index, evt);
419 }
420
421 /* arm_pmu backend */
m1_pmu_enable_event(struct perf_event * event)422 static void m1_pmu_enable_event(struct perf_event *event)
423 {
424 bool user, kernel;
425 u8 evt;
426
427 evt = event->hw.config_base & M1_PMU_CFG_EVENT;
428 user = event->hw.config_base & M1_PMU_CFG_COUNT_USER;
429 kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL;
430
431 m1_pmu_configure_counter(event->hw.idx, event->hw.config_base);
432 m1_pmu_enable_counter(event->hw.idx);
433 m1_pmu_enable_counter_interrupt(event->hw.idx);
434 isb();
435 }
436
m1_pmu_disable_event(struct perf_event * event)437 static void m1_pmu_disable_event(struct perf_event *event)
438 {
439 m1_pmu_disable_counter_interrupt(event->hw.idx);
440 m1_pmu_disable_counter(event->hw.idx);
441 isb();
442 }
443
m1_pmu_handle_irq(struct arm_pmu * cpu_pmu)444 static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu)
445 {
446 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
447 struct pt_regs *regs;
448 u64 overflow, state;
449 int idx;
450
451 overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1);
452 if (!overflow) {
453 /* Spurious interrupt? */
454 state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
455 state &= ~PMCR0_IACT;
456 write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1);
457 isb();
458 return IRQ_NONE;
459 }
460
461 cpu_pmu->stop(cpu_pmu);
462
463 regs = get_irq_regs();
464
465 for_each_set_bit(idx, cpu_pmu->cntr_mask, M1_PMU_NR_COUNTERS) {
466 struct perf_event *event = cpuc->events[idx];
467 struct perf_sample_data data;
468
469 if (!event)
470 continue;
471
472 armpmu_event_update(event);
473 perf_sample_data_init(&data, 0, event->hw.last_period);
474 if (!armpmu_event_set_period(event))
475 continue;
476
477 if (perf_event_overflow(event, &data, regs))
478 m1_pmu_disable_event(event);
479 }
480
481 cpu_pmu->start(cpu_pmu);
482
483 return IRQ_HANDLED;
484 }
485
m1_pmu_read_counter(struct perf_event * event)486 static u64 m1_pmu_read_counter(struct perf_event *event)
487 {
488 return m1_pmu_read_hw_counter(event->hw.idx);
489 }
490
m1_pmu_write_counter(struct perf_event * event,u64 value)491 static void m1_pmu_write_counter(struct perf_event *event, u64 value)
492 {
493 m1_pmu_write_hw_counter(value, event->hw.idx);
494 isb();
495 }
496
m1_pmu_get_event_idx(struct pmu_hw_events * cpuc,struct perf_event * event)497 static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc,
498 struct perf_event *event)
499 {
500 unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT;
501 unsigned long affinity = m1_pmu_event_affinity[evtype];
502 int idx;
503
504 /*
505 * Place the event on the first free counter that can count
506 * this event.
507 *
508 * We could do a better job if we had a view of all the events
509 * counting on the PMU at any given time, and by placing the
510 * most constraining events first.
511 */
512 for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) {
513 if (!test_and_set_bit(idx, cpuc->used_mask))
514 return idx;
515 }
516
517 return -EAGAIN;
518 }
519
m1_pmu_clear_event_idx(struct pmu_hw_events * cpuc,struct perf_event * event)520 static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
521 struct perf_event *event)
522 {
523 clear_bit(event->hw.idx, cpuc->used_mask);
524 }
525
__m1_pmu_set_mode(u8 mode)526 static void __m1_pmu_set_mode(u8 mode)
527 {
528 u64 val;
529
530 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
531 val &= ~(PMCR0_IMODE | PMCR0_IACT);
532 val |= FIELD_PREP(PMCR0_IMODE, mode);
533 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
534 isb();
535 }
536
m1_pmu_start(struct arm_pmu * cpu_pmu)537 static void m1_pmu_start(struct arm_pmu *cpu_pmu)
538 {
539 __m1_pmu_set_mode(PMCR0_IMODE_FIQ);
540 }
541
m1_pmu_stop(struct arm_pmu * cpu_pmu)542 static void m1_pmu_stop(struct arm_pmu *cpu_pmu)
543 {
544 __m1_pmu_set_mode(PMCR0_IMODE_OFF);
545 }
546
m1_pmu_map_event(struct perf_event * event)547 static int m1_pmu_map_event(struct perf_event *event)
548 {
549 /*
550 * Although the counters are 48bit wide, bit 47 is what
551 * triggers the overflow interrupt. Advertise the counters
552 * being 47bit wide to mimick the behaviour of the ARM PMU.
553 */
554 event->hw.flags |= ARMPMU_EVT_47BIT;
555 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT);
556 }
557
m2_pmu_map_event(struct perf_event * event)558 static int m2_pmu_map_event(struct perf_event *event)
559 {
560 /*
561 * Same deal as the above, except that M2 has 64bit counters.
562 * Which, as far as we're concerned, actually means 63 bits.
563 * Yes, this is getting awkward.
564 */
565 event->hw.flags |= ARMPMU_EVT_63BIT;
566 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT);
567 }
568
m1_pmu_map_pmuv3_event(unsigned int eventsel)569 static int m1_pmu_map_pmuv3_event(unsigned int eventsel)
570 {
571 u16 m1_event = HW_OP_UNSUPPORTED;
572
573 if (eventsel < ARMV8_PMUV3_MAX_COMMON_EVENTS)
574 m1_event = m1_pmu_pmceid_map[eventsel];
575
576 return m1_event == HW_OP_UNSUPPORTED ? -EOPNOTSUPP : m1_event;
577 }
578
m1_pmu_init_pmceid(struct arm_pmu * pmu)579 static void m1_pmu_init_pmceid(struct arm_pmu *pmu)
580 {
581 unsigned int event;
582
583 for (event = 0; event < ARMV8_PMUV3_MAX_COMMON_EVENTS; event++) {
584 if (m1_pmu_map_pmuv3_event(event) >= 0)
585 set_bit(event, pmu->pmceid_bitmap);
586 }
587 }
588
m1_pmu_reset(void * info)589 static void m1_pmu_reset(void *info)
590 {
591 int i;
592
593 __m1_pmu_set_mode(PMCR0_IMODE_OFF);
594
595 for (i = 0; i < M1_PMU_NR_COUNTERS; i++) {
596 m1_pmu_disable_counter(i);
597 m1_pmu_disable_counter_interrupt(i);
598 m1_pmu_write_hw_counter(0, i);
599 }
600
601 isb();
602 }
603
m1_pmu_set_event_filter(struct hw_perf_event * event,struct perf_event_attr * attr)604 static int m1_pmu_set_event_filter(struct hw_perf_event *event,
605 struct perf_event_attr *attr)
606 {
607 unsigned long config_base = 0;
608
609 if (!attr->exclude_guest && !is_kernel_in_hyp_mode()) {
610 pr_debug("ARM performance counters do not support mode exclusion\n");
611 return -EOPNOTSUPP;
612 }
613 if (!attr->exclude_kernel)
614 config_base |= M1_PMU_CFG_COUNT_KERNEL;
615 if (!attr->exclude_user)
616 config_base |= M1_PMU_CFG_COUNT_USER;
617 if (!attr->exclude_host)
618 config_base |= M1_PMU_CFG_COUNT_HOST;
619 if (!attr->exclude_guest)
620 config_base |= M1_PMU_CFG_COUNT_GUEST;
621
622 event->config_base = config_base;
623
624 return 0;
625 }
626
m1_pmu_init(struct arm_pmu * cpu_pmu,u32 flags)627 static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags)
628 {
629 cpu_pmu->handle_irq = m1_pmu_handle_irq;
630 cpu_pmu->enable = m1_pmu_enable_event;
631 cpu_pmu->disable = m1_pmu_disable_event;
632 cpu_pmu->read_counter = m1_pmu_read_counter;
633 cpu_pmu->write_counter = m1_pmu_write_counter;
634 cpu_pmu->get_event_idx = m1_pmu_get_event_idx;
635 cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx;
636 cpu_pmu->start = m1_pmu_start;
637 cpu_pmu->stop = m1_pmu_stop;
638
639 if (flags & ARMPMU_EVT_47BIT)
640 cpu_pmu->map_event = m1_pmu_map_event;
641 else if (flags & ARMPMU_EVT_63BIT)
642 cpu_pmu->map_event = m2_pmu_map_event;
643 else
644 return WARN_ON(-EINVAL);
645
646 cpu_pmu->reset = m1_pmu_reset;
647 cpu_pmu->set_event_filter = m1_pmu_set_event_filter;
648
649 cpu_pmu->map_pmuv3_event = m1_pmu_map_pmuv3_event;
650 m1_pmu_init_pmceid(cpu_pmu);
651
652 bitmap_set(cpu_pmu->cntr_mask, 0, M1_PMU_NR_COUNTERS);
653 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group;
654 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group;
655 return 0;
656 }
657
658 /* Device driver gunk */
m1_pmu_ice_init(struct arm_pmu * cpu_pmu)659 static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu)
660 {
661 cpu_pmu->name = "apple_icestorm_pmu";
662 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT);
663 }
664
m1_pmu_fire_init(struct arm_pmu * cpu_pmu)665 static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu)
666 {
667 cpu_pmu->name = "apple_firestorm_pmu";
668 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT);
669 }
670
m2_pmu_avalanche_init(struct arm_pmu * cpu_pmu)671 static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu)
672 {
673 cpu_pmu->name = "apple_avalanche_pmu";
674 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT);
675 }
676
m2_pmu_blizzard_init(struct arm_pmu * cpu_pmu)677 static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu)
678 {
679 cpu_pmu->name = "apple_blizzard_pmu";
680 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT);
681 }
682
683 static const struct of_device_id m1_pmu_of_device_ids[] = {
684 { .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, },
685 { .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, },
686 { .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, },
687 { .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, },
688 { },
689 };
690 MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids);
691
m1_pmu_device_probe(struct platform_device * pdev)692 static int m1_pmu_device_probe(struct platform_device *pdev)
693 {
694 return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL);
695 }
696
697 static struct platform_driver m1_pmu_driver = {
698 .driver = {
699 .name = "apple-m1-cpu-pmu",
700 .of_match_table = m1_pmu_of_device_ids,
701 .suppress_bind_attrs = true,
702 },
703 .probe = m1_pmu_device_probe,
704 };
705
706 module_platform_driver(m1_pmu_driver);
707