1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/types.h>
3 #include <linux/interrupt.h>
4
5 #include <asm/msr.h>
6 #include <asm/xen/hypercall.h>
7 #include <xen/xen.h>
8 #include <xen/page.h>
9 #include <xen/interface/xen.h>
10 #include <xen/interface/vcpu.h>
11 #include <xen/interface/xenpmu.h>
12
13 #include "xen-ops.h"
14
15 /* x86_pmu.handle_irq definition */
16 #include "../events/perf_event.h"
17
18 #define XENPMU_IRQ_PROCESSING 1
19 struct xenpmu {
20 /* Shared page between hypervisor and domain */
21 struct xen_pmu_data *xenpmu_data;
22
23 uint8_t flags;
24 };
25 static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
26 #define get_xenpmu_data() (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
27 #define get_xenpmu_flags() (this_cpu_ptr(&xenpmu_shared)->flags)
28
29 /* Macro for computing address of a PMU MSR bank */
30 #define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
31 (uintptr_t)ctxt->field))
32
33 /* AMD PMU */
34 #define F15H_NUM_COUNTERS 6
35 #define F10H_NUM_COUNTERS 4
36
37 static __read_mostly uint32_t amd_counters_base;
38 static __read_mostly uint32_t amd_ctrls_base;
39 static __read_mostly int amd_msr_step;
40 static __read_mostly int k7_counters_mirrored;
41 static __read_mostly int amd_num_counters;
42
43 /* Intel PMU */
44 #define MSR_TYPE_COUNTER 0
45 #define MSR_TYPE_CTRL 1
46 #define MSR_TYPE_GLOBAL 2
47 #define MSR_TYPE_ARCH_COUNTER 3
48 #define MSR_TYPE_ARCH_CTRL 4
49
50 /* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
51 #define PMU_GENERAL_NR_SHIFT 8
52 #define PMU_GENERAL_NR_BITS 8
53 #define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \
54 << PMU_GENERAL_NR_SHIFT)
55
56 /* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
57 #define PMU_FIXED_NR_SHIFT 0
58 #define PMU_FIXED_NR_BITS 5
59 #define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \
60 << PMU_FIXED_NR_SHIFT)
61
62 /* Alias registers (0x4c1) for full-width writes to PMCs */
63 #define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
64
65 #define INTEL_PMC_TYPE_SHIFT 30
66
67 static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
68
69
xen_pmu_arch_init(void)70 static void xen_pmu_arch_init(void)
71 {
72 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
73
74 switch (boot_cpu_data.x86) {
75 case 0x15:
76 amd_num_counters = F15H_NUM_COUNTERS;
77 amd_counters_base = MSR_F15H_PERF_CTR;
78 amd_ctrls_base = MSR_F15H_PERF_CTL;
79 amd_msr_step = 2;
80 k7_counters_mirrored = 1;
81 break;
82 case 0x10:
83 case 0x12:
84 case 0x14:
85 case 0x16:
86 default:
87 amd_num_counters = F10H_NUM_COUNTERS;
88 amd_counters_base = MSR_K7_PERFCTR0;
89 amd_ctrls_base = MSR_K7_EVNTSEL0;
90 amd_msr_step = 1;
91 k7_counters_mirrored = 0;
92 break;
93 }
94 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
95 amd_num_counters = F10H_NUM_COUNTERS;
96 amd_counters_base = MSR_K7_PERFCTR0;
97 amd_ctrls_base = MSR_K7_EVNTSEL0;
98 amd_msr_step = 1;
99 k7_counters_mirrored = 0;
100 } else {
101 uint32_t eax, ebx, ecx, edx;
102
103 cpuid(0xa, &eax, &ebx, &ecx, &edx);
104
105 intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
106 PMU_GENERAL_NR_SHIFT;
107 intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
108 PMU_FIXED_NR_SHIFT;
109 }
110 }
111
get_fam15h_addr(u32 addr)112 static inline uint32_t get_fam15h_addr(u32 addr)
113 {
114 switch (addr) {
115 case MSR_K7_PERFCTR0:
116 case MSR_K7_PERFCTR1:
117 case MSR_K7_PERFCTR2:
118 case MSR_K7_PERFCTR3:
119 return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
120 case MSR_K7_EVNTSEL0:
121 case MSR_K7_EVNTSEL1:
122 case MSR_K7_EVNTSEL2:
123 case MSR_K7_EVNTSEL3:
124 return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
125 default:
126 break;
127 }
128
129 return addr;
130 }
131
is_amd_pmu_msr(u32 msr)132 static bool is_amd_pmu_msr(u32 msr)
133 {
134 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
135 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
136 return false;
137
138 if ((msr >= MSR_F15H_PERF_CTL &&
139 msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
140 (msr >= MSR_K7_EVNTSEL0 &&
141 msr < MSR_K7_PERFCTR0 + amd_num_counters))
142 return true;
143
144 return false;
145 }
146
is_intel_pmu_msr(u32 msr_index,int * type,int * index)147 static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index)
148 {
149 u32 msr_index_pmc;
150
151 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
152 boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR &&
153 boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
154 return false;
155
156 switch (msr_index) {
157 case MSR_CORE_PERF_FIXED_CTR_CTRL:
158 case MSR_IA32_DS_AREA:
159 case MSR_IA32_PEBS_ENABLE:
160 *type = MSR_TYPE_CTRL;
161 return true;
162
163 case MSR_CORE_PERF_GLOBAL_CTRL:
164 case MSR_CORE_PERF_GLOBAL_STATUS:
165 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
166 *type = MSR_TYPE_GLOBAL;
167 return true;
168
169 default:
170
171 if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
172 (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
173 intel_num_fixed_counters)) {
174 *index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
175 *type = MSR_TYPE_COUNTER;
176 return true;
177 }
178
179 if ((msr_index >= MSR_P6_EVNTSEL0) &&
180 (msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) {
181 *index = msr_index - MSR_P6_EVNTSEL0;
182 *type = MSR_TYPE_ARCH_CTRL;
183 return true;
184 }
185
186 msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
187 if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
188 (msr_index_pmc < MSR_IA32_PERFCTR0 +
189 intel_num_arch_counters)) {
190 *type = MSR_TYPE_ARCH_COUNTER;
191 *index = msr_index_pmc - MSR_IA32_PERFCTR0;
192 return true;
193 }
194 return false;
195 }
196 }
197
xen_intel_pmu_emulate(u32 msr,u64 * val,int type,int index,bool is_read)198 static bool xen_intel_pmu_emulate(u32 msr, u64 *val, int type, int index, bool is_read)
199 {
200 uint64_t *reg = NULL;
201 struct xen_pmu_intel_ctxt *ctxt;
202 uint64_t *fix_counters;
203 struct xen_pmu_cntr_pair *arch_cntr_pair;
204 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
205 uint8_t xenpmu_flags = get_xenpmu_flags();
206
207
208 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
209 return false;
210
211 ctxt = &xenpmu_data->pmu.c.intel;
212
213 switch (msr) {
214 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
215 reg = &ctxt->global_ovf_ctrl;
216 break;
217 case MSR_CORE_PERF_GLOBAL_STATUS:
218 reg = &ctxt->global_status;
219 break;
220 case MSR_CORE_PERF_GLOBAL_CTRL:
221 reg = &ctxt->global_ctrl;
222 break;
223 case MSR_CORE_PERF_FIXED_CTR_CTRL:
224 reg = &ctxt->fixed_ctrl;
225 break;
226 default:
227 switch (type) {
228 case MSR_TYPE_COUNTER:
229 fix_counters = field_offset(ctxt, fixed_counters);
230 reg = &fix_counters[index];
231 break;
232 case MSR_TYPE_ARCH_COUNTER:
233 arch_cntr_pair = field_offset(ctxt, arch_counters);
234 reg = &arch_cntr_pair[index].counter;
235 break;
236 case MSR_TYPE_ARCH_CTRL:
237 arch_cntr_pair = field_offset(ctxt, arch_counters);
238 reg = &arch_cntr_pair[index].control;
239 break;
240 default:
241 return false;
242 }
243 }
244
245 if (reg) {
246 if (is_read)
247 *val = *reg;
248 else {
249 *reg = *val;
250
251 if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
252 ctxt->global_status &= (~(*val));
253 }
254 return true;
255 }
256
257 return false;
258 }
259
xen_amd_pmu_emulate(u32 msr,u64 * val,bool is_read)260 static bool xen_amd_pmu_emulate(u32 msr, u64 *val, bool is_read)
261 {
262 uint64_t *reg = NULL;
263 int i, off = 0;
264 struct xen_pmu_amd_ctxt *ctxt;
265 uint64_t *counter_regs, *ctrl_regs;
266 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
267 uint8_t xenpmu_flags = get_xenpmu_flags();
268
269 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
270 return false;
271
272 if (k7_counters_mirrored &&
273 ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
274 msr = get_fam15h_addr(msr);
275
276 ctxt = &xenpmu_data->pmu.c.amd;
277 for (i = 0; i < amd_num_counters; i++) {
278 if (msr == amd_ctrls_base + off) {
279 ctrl_regs = field_offset(ctxt, ctrls);
280 reg = &ctrl_regs[i];
281 break;
282 } else if (msr == amd_counters_base + off) {
283 counter_regs = field_offset(ctxt, counters);
284 reg = &counter_regs[i];
285 break;
286 }
287 off += amd_msr_step;
288 }
289
290 if (reg) {
291 if (is_read)
292 *val = *reg;
293 else
294 *reg = *val;
295
296 return true;
297 }
298 return false;
299 }
300
pmu_msr_chk_emulated(u32 msr,u64 * val,bool is_read)301 bool pmu_msr_chk_emulated(u32 msr, u64 *val, bool is_read)
302 {
303 int type, index = 0;
304
305 if (is_amd_pmu_msr(msr))
306 return xen_amd_pmu_emulate(msr, val, is_read);
307
308 if (is_intel_pmu_msr(msr, &type, &index))
309 return xen_intel_pmu_emulate(msr, val, type, index, is_read);
310
311 return false;
312 }
313
xen_amd_read_pmc(int counter)314 static u64 xen_amd_read_pmc(int counter)
315 {
316 struct xen_pmu_amd_ctxt *ctxt;
317 uint64_t *counter_regs;
318 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
319 uint8_t xenpmu_flags = get_xenpmu_flags();
320
321 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
322 u32 msr;
323 u64 val;
324
325 msr = amd_counters_base + (counter * amd_msr_step);
326 native_read_msr_safe(msr, &val);
327 return val;
328 }
329
330 ctxt = &xenpmu_data->pmu.c.amd;
331 counter_regs = field_offset(ctxt, counters);
332 return counter_regs[counter];
333 }
334
xen_intel_read_pmc(int counter)335 static u64 xen_intel_read_pmc(int counter)
336 {
337 struct xen_pmu_intel_ctxt *ctxt;
338 uint64_t *fixed_counters;
339 struct xen_pmu_cntr_pair *arch_cntr_pair;
340 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
341 uint8_t xenpmu_flags = get_xenpmu_flags();
342
343 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
344 u32 msr;
345 u64 val;
346
347 if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
348 msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
349 else
350 msr = MSR_IA32_PERFCTR0 + counter;
351
352 native_read_msr_safe(msr, &val);
353 return val;
354 }
355
356 ctxt = &xenpmu_data->pmu.c.intel;
357 if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
358 fixed_counters = field_offset(ctxt, fixed_counters);
359 return fixed_counters[counter & 0xffff];
360 }
361
362 arch_cntr_pair = field_offset(ctxt, arch_counters);
363 return arch_cntr_pair[counter].counter;
364 }
365
xen_read_pmc(int counter)366 u64 xen_read_pmc(int counter)
367 {
368 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
369 return xen_amd_read_pmc(counter);
370 else
371 return xen_intel_read_pmc(counter);
372 }
373
pmu_apic_update(uint32_t val)374 int pmu_apic_update(uint32_t val)
375 {
376 int ret;
377 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
378
379 if (!xenpmu_data) {
380 pr_warn_once("%s: pmudata not initialized\n", __func__);
381 return -EINVAL;
382 }
383
384 xenpmu_data->pmu.l.lapic_lvtpc = val;
385
386 if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
387 return 0;
388
389 ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
390
391 return ret;
392 }
393
394 /* perf callbacks */
xen_guest_state(void)395 static unsigned int xen_guest_state(void)
396 {
397 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
398 unsigned int state = 0;
399
400 if (!xenpmu_data) {
401 pr_warn_once("%s: pmudata not initialized\n", __func__);
402 return state;
403 }
404
405 if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
406 return state;
407
408 state |= PERF_GUEST_ACTIVE;
409
410 if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) {
411 if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER)
412 state |= PERF_GUEST_USER;
413 } else if (xenpmu_data->pmu.r.regs.cpl & 3) {
414 state |= PERF_GUEST_USER;
415 }
416
417 return state;
418 }
419
xen_get_guest_ip(void)420 static unsigned long xen_get_guest_ip(void)
421 {
422 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
423
424 if (!xenpmu_data) {
425 pr_warn_once("%s: pmudata not initialized\n", __func__);
426 return 0;
427 }
428
429 return xenpmu_data->pmu.r.regs.ip;
430 }
431
432 static struct perf_guest_info_callbacks xen_guest_cbs = {
433 .state = xen_guest_state,
434 .get_ip = xen_get_guest_ip,
435 };
436
437 /* Convert registers from Xen's format to Linux' */
xen_convert_regs(const struct xen_pmu_regs * xen_regs,struct pt_regs * regs,uint64_t pmu_flags)438 static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
439 struct pt_regs *regs, uint64_t pmu_flags)
440 {
441 regs->ip = xen_regs->ip;
442 regs->cs = xen_regs->cs;
443 regs->sp = xen_regs->sp;
444
445 if (pmu_flags & PMU_SAMPLE_PV) {
446 if (pmu_flags & PMU_SAMPLE_USER)
447 regs->cs |= 3;
448 else
449 regs->cs &= ~3;
450 } else {
451 if (xen_regs->cpl)
452 regs->cs |= 3;
453 else
454 regs->cs &= ~3;
455 }
456 }
457
xen_pmu_irq_handler(int irq,void * dev_id)458 irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
459 {
460 int err, ret = IRQ_NONE;
461 struct pt_regs regs = {0};
462 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
463 uint8_t xenpmu_flags = get_xenpmu_flags();
464
465 if (!xenpmu_data) {
466 pr_warn_once("%s: pmudata not initialized\n", __func__);
467 return ret;
468 }
469
470 this_cpu_ptr(&xenpmu_shared)->flags =
471 xenpmu_flags | XENPMU_IRQ_PROCESSING;
472 xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s,
473 xenpmu_data->pmu.pmu_flags);
474 if (x86_pmu.handle_irq(®s))
475 ret = IRQ_HANDLED;
476
477 /* Write out cached context to HW */
478 err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
479 this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
480 if (err) {
481 pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
482 return IRQ_NONE;
483 }
484
485 return ret;
486 }
487
488 bool is_xen_pmu;
489
xen_pmu_init(int cpu)490 void xen_pmu_init(int cpu)
491 {
492 int err;
493 struct xen_pmu_params xp;
494 unsigned long pfn;
495 struct xen_pmu_data *xenpmu_data;
496
497 BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
498
499 if (xen_hvm_domain() || (cpu != 0 && !is_xen_pmu))
500 return;
501
502 xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
503 if (!xenpmu_data) {
504 pr_err("VPMU init: No memory\n");
505 return;
506 }
507 pfn = virt_to_pfn(xenpmu_data);
508
509 xp.val = pfn_to_mfn(pfn);
510 xp.vcpu = cpu;
511 xp.version.maj = XENPMU_VER_MAJ;
512 xp.version.min = XENPMU_VER_MIN;
513 err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
514 if (err)
515 goto fail;
516
517 per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
518 per_cpu(xenpmu_shared, cpu).flags = 0;
519
520 if (!is_xen_pmu) {
521 is_xen_pmu = true;
522 perf_register_guest_info_callbacks(&xen_guest_cbs);
523 xen_pmu_arch_init();
524 }
525
526 return;
527
528 fail:
529 if (err == -EOPNOTSUPP || err == -ENOSYS)
530 pr_info_once("VPMU disabled by hypervisor.\n");
531 else
532 pr_info_once("Could not initialize VPMU for cpu %d, error %d\n",
533 cpu, err);
534 free_pages((unsigned long)xenpmu_data, 0);
535 }
536
xen_pmu_finish(int cpu)537 void xen_pmu_finish(int cpu)
538 {
539 struct xen_pmu_params xp;
540
541 if (xen_hvm_domain())
542 return;
543
544 xp.vcpu = cpu;
545 xp.version.maj = XENPMU_VER_MAJ;
546 xp.version.min = XENPMU_VER_MIN;
547
548 (void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
549
550 free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
551 per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
552 }
553