xref: /kvm-unit-tests/x86/pmu.c (revision 699264f5ef8129c60e9db7c281e572016ad41a45)
1a9f8b16fSGleb Natapov 
2a9f8b16fSGleb Natapov #include "x86/msr.h"
3a9f8b16fSGleb Natapov #include "x86/processor.h"
49f17508dSLike Xu #include "x86/pmu.h"
5a9f8b16fSGleb Natapov #include "x86/apic-defs.h"
6a9f8b16fSGleb Natapov #include "x86/apic.h"
7a9f8b16fSGleb Natapov #include "x86/desc.h"
8a9f8b16fSGleb Natapov #include "x86/isr.h"
995a94088SNicholas Piggin #include "vmalloc.h"
10dcda215bSPaolo Bonzini #include "alloc.h"
11a9f8b16fSGleb Natapov 
12a9f8b16fSGleb Natapov #include "libcflat.h"
13a9f8b16fSGleb Natapov #include <stdint.h>
14a9f8b16fSGleb Natapov 
15a9f8b16fSGleb Natapov #define N 1000000
16a9f8b16fSGleb Natapov 
178dbfe326SDapeng Mi #define IBPB_JMP_INSNS		9
188dbfe326SDapeng Mi #define IBPB_JMP_BRANCHES	2
198dbfe326SDapeng Mi 
208dbfe326SDapeng Mi #if defined(__i386__) || defined(_M_IX86) /* i386 */
218dbfe326SDapeng Mi #define IBPB_JMP_ASM(_wrmsr)				\
228dbfe326SDapeng Mi 	"mov $1, %%eax; xor %%edx, %%edx;\n\t"		\
238dbfe326SDapeng Mi 	"mov $73, %%ecx;\n\t"				\
2450f8e27eSDapeng Mi 	_wrmsr "\n\t"					\
258dbfe326SDapeng Mi 	"call 1f\n\t"					\
268dbfe326SDapeng Mi 	"1: pop %%eax\n\t"				\
278dbfe326SDapeng Mi 	"add $(2f-1b), %%eax\n\t"			\
288dbfe326SDapeng Mi 	"jmp *%%eax;\n\t"                               \
298dbfe326SDapeng Mi 	"nop;\n\t"					\
308dbfe326SDapeng Mi 	"2: nop;\n\t"
318dbfe326SDapeng Mi #else /* x86_64 */
328dbfe326SDapeng Mi #define IBPB_JMP_ASM(_wrmsr)				\
338dbfe326SDapeng Mi 	"mov $1, %%eax; xor %%edx, %%edx;\n\t"		\
348dbfe326SDapeng Mi 	"mov $73, %%ecx;\n\t"				\
358dbfe326SDapeng Mi 	_wrmsr "\n\t"					\
368dbfe326SDapeng Mi 	"call 1f\n\t"					\
378dbfe326SDapeng Mi 	"1: pop %%rax\n\t"				\
388dbfe326SDapeng Mi 	"add $(2f-1b), %%rax\n\t"                       \
398dbfe326SDapeng Mi 	"jmp *%%rax;\n\t"                               \
408dbfe326SDapeng Mi 	"nop;\n\t"					\
418dbfe326SDapeng Mi 	"2: nop;\n\t"
428dbfe326SDapeng Mi #endif
438dbfe326SDapeng Mi 
448dbfe326SDapeng Mi /* GLOBAL_CTRL enable + disable + clflush/mfence + IBPB_JMP */
458dbfe326SDapeng Mi #define EXTRA_INSNS  (3 + 3 + 2 + IBPB_JMP_INSNS)
468dbfe326SDapeng Mi #define LOOP_INSNS   (N * 10 + EXTRA_INSNS)
478dbfe326SDapeng Mi #define LOOP_BRANCHES  (N + IBPB_JMP_BRANCHES)
488dbfe326SDapeng Mi #define LOOP_ASM(_wrmsr1, _clflush, _wrmsr2)				\
498dbfe326SDapeng Mi 	_wrmsr1 "\n\t"							\
5050f8e27eSDapeng Mi 	"mov %%ecx, %%edi; mov %%ebx, %%ecx;\n\t"			\
5138b5b426SDapeng Mi 	_clflush "\n\t"                                 		\
5238b5b426SDapeng Mi 	"mfence;\n\t"                                   		\
5350f8e27eSDapeng Mi 	"1: mov (%1), %2; add $64, %1;\n\t"				\
5450f8e27eSDapeng Mi 	"nop; nop; nop; nop; nop; nop; nop;\n\t"			\
5550f8e27eSDapeng Mi 	"loop 1b;\n\t"							\
568dbfe326SDapeng Mi 	IBPB_JMP_ASM(_wrmsr2) 						\
5750f8e27eSDapeng Mi 	"mov %%edi, %%ecx; xor %%eax, %%eax; xor %%edx, %%edx;\n\t"	\
588dbfe326SDapeng Mi 	_wrmsr1 "\n\t"
5950f8e27eSDapeng Mi 
608dbfe326SDapeng Mi #define _loop_asm(_wrmsr1, _clflush, _wrmsr2)			\
6138b5b426SDapeng Mi do {								\
628dbfe326SDapeng Mi 	asm volatile(LOOP_ASM(_wrmsr1, _clflush, _wrmsr2)	\
6338b5b426SDapeng Mi 		     : "=b"(tmp), "=r"(tmp2), "=r"(tmp3)	\
6438b5b426SDapeng Mi 		     : "a"(eax), "d"(edx), "c"(global_ctl),	\
6538b5b426SDapeng Mi 		       "0"(N), "1"(buf)				\
6638b5b426SDapeng Mi 		     : "edi");					\
6738b5b426SDapeng Mi } while (0)
6838b5b426SDapeng Mi 
695dcbe0ddSDapeng Mi /* the number of instructions and branches of the kvm_fep_asm() blob */
705dcbe0ddSDapeng Mi #define KVM_FEP_INSNS		22
715dcbe0ddSDapeng Mi #define KVM_FEP_BRANCHES	5
725dcbe0ddSDapeng Mi 
735dcbe0ddSDapeng Mi /*
745dcbe0ddSDapeng Mi  * KVM_FEP is a magic prefix that forces emulation so
755dcbe0ddSDapeng Mi  * 'KVM_FEP "jne label\n"' just counts as a single instruction.
765dcbe0ddSDapeng Mi  */
775dcbe0ddSDapeng Mi #define kvm_fep_asm(_wrmsr)			\
785dcbe0ddSDapeng Mi do {						\
795dcbe0ddSDapeng Mi 	asm volatile(				\
805dcbe0ddSDapeng Mi 		_wrmsr "\n\t"			\
815dcbe0ddSDapeng Mi 		"mov %%ecx, %%edi;\n\t"		\
825dcbe0ddSDapeng Mi 		"mov $0x0, %%eax;\n\t"		\
835dcbe0ddSDapeng Mi 		"cmp $0x0, %%eax;\n\t"		\
845dcbe0ddSDapeng Mi 		KVM_FEP "jne 1f\n\t"		\
855dcbe0ddSDapeng Mi 		KVM_FEP "jne 1f\n\t"		\
865dcbe0ddSDapeng Mi 		KVM_FEP "jne 1f\n\t"		\
875dcbe0ddSDapeng Mi 		KVM_FEP "jne 1f\n\t"		\
885dcbe0ddSDapeng Mi 		KVM_FEP "jne 1f\n\t"		\
895dcbe0ddSDapeng Mi 		"mov $0xa, %%eax; cpuid;\n\t"	\
905dcbe0ddSDapeng Mi 		"mov $0xa, %%eax; cpuid;\n\t"	\
915dcbe0ddSDapeng Mi 		"mov $0xa, %%eax; cpuid;\n\t"	\
925dcbe0ddSDapeng Mi 		"mov $0xa, %%eax; cpuid;\n\t"	\
935dcbe0ddSDapeng Mi 		"mov $0xa, %%eax; cpuid;\n\t"	\
945dcbe0ddSDapeng Mi 		"1: mov %%edi, %%ecx; \n\t"	\
955dcbe0ddSDapeng Mi 		"xor %%eax, %%eax; \n\t"	\
965dcbe0ddSDapeng Mi 		"xor %%edx, %%edx;\n\t"		\
975dcbe0ddSDapeng Mi 		_wrmsr "\n\t"			\
985dcbe0ddSDapeng Mi 		:				\
995dcbe0ddSDapeng Mi 		: "a"(eax), "d"(edx), "c"(ecx)	\
1005dcbe0ddSDapeng Mi 		: "ebx", "edi");		\
1015dcbe0ddSDapeng Mi } while (0)
1025dcbe0ddSDapeng Mi 
103a9f8b16fSGleb Natapov typedef struct {
104a9f8b16fSGleb Natapov 	uint32_t ctr;
1059720e46cSDapeng Mi 	uint32_t idx;
106006b089dSLike Xu 	uint64_t config;
107a9f8b16fSGleb Natapov 	uint64_t count;
108a9f8b16fSGleb Natapov } pmu_counter_t;
109a9f8b16fSGleb Natapov 
110a9f8b16fSGleb Natapov struct pmu_event {
111797d79a2SThomas Huth 	const char *name;
112a9f8b16fSGleb Natapov 	uint32_t unit_sel;
113a9f8b16fSGleb Natapov 	int min;
114a9f8b16fSGleb Natapov 	int max;
1157c648ce2SLike Xu } intel_gp_events[] = {
116a9f8b16fSGleb Natapov 	{"core cycles", 0x003c, 1*N, 50*N},
117a9f8b16fSGleb Natapov 	{"instructions", 0x00c0, 10*N, 10.2*N},
118290f4213SJim Mattson 	{"ref cycles", 0x013c, 1*N, 30*N},
119290f4213SJim Mattson 	{"llc references", 0x4f2e, 1, 2*N},
120a9f8b16fSGleb Natapov 	{"llc misses", 0x412e, 1, 1*N},
121a9f8b16fSGleb Natapov 	{"branches", 0x00c4, 1*N, 1.1*N},
12228437cdbSDapeng Mi 	{"branch misses", 0x00c5, 1, 0.1*N},
123b883751aSLike Xu }, amd_gp_events[] = {
124b883751aSLike Xu 	{"core cycles", 0x0076, 1*N, 50*N},
125b883751aSLike Xu 	{"instructions", 0x00c0, 10*N, 10.2*N},
126b883751aSLike Xu 	{"branches", 0x00c2, 1*N, 1.1*N},
12728437cdbSDapeng Mi 	{"branch misses", 0x00c3, 1, 0.1*N},
128a9f8b16fSGleb Natapov }, fixed_events[] = {
1295d6a3a54SDapeng Mi 	{"fixed 0", MSR_CORE_PERF_FIXED_CTR0, 10*N, 10.2*N},
1305d6a3a54SDapeng Mi 	{"fixed 1", MSR_CORE_PERF_FIXED_CTR0 + 1, 1*N, 30*N},
1315d6a3a54SDapeng Mi 	{"fixed 2", MSR_CORE_PERF_FIXED_CTR0 + 2, 0.1*N, 30*N}
132a9f8b16fSGleb Natapov };
133a9f8b16fSGleb Natapov 
134f4e97f59SDapeng Mi /*
135f4e97f59SDapeng Mi  * Events index in intel_gp_events[], ensure consistent with
136f4e97f59SDapeng Mi  * intel_gp_events[].
137f4e97f59SDapeng Mi  */
138f4e97f59SDapeng Mi enum {
13985c75578SDapeng Mi 	INTEL_INSTRUCTIONS_IDX  = 1,
14025cc1ea7SDapeng Mi 	INTEL_REF_CYCLES_IDX	= 2,
141e0d0022fSDapeng Mi 	INTEL_LLC_MISSES_IDX	= 4,
142f4e97f59SDapeng Mi 	INTEL_BRANCHES_IDX	= 5,
14328437cdbSDapeng Mi 	INTEL_BRANCH_MISS_IDX	= 6,
144f4e97f59SDapeng Mi };
145f4e97f59SDapeng Mi 
146f4e97f59SDapeng Mi /*
147f4e97f59SDapeng Mi  * Events index in amd_gp_events[], ensure consistent with
148f4e97f59SDapeng Mi  * amd_gp_events[].
149f4e97f59SDapeng Mi  */
150f4e97f59SDapeng Mi enum {
15185c75578SDapeng Mi 	AMD_INSTRUCTIONS_IDX    = 1,
152f4e97f59SDapeng Mi 	AMD_BRANCHES_IDX	= 2,
15328437cdbSDapeng Mi 	AMD_BRANCH_MISS_IDX	= 3,
154f4e97f59SDapeng Mi };
155f4e97f59SDapeng Mi 
156a9f8b16fSGleb Natapov char *buf;
157a9f8b16fSGleb Natapov 
1587c648ce2SLike Xu static struct pmu_event *gp_events;
1597c648ce2SLike Xu static unsigned int gp_events_size;
1609c07c92bSDapeng Mi static unsigned int fixed_counters_num;
1617c648ce2SLike Xu 
has_ibpb(void)1628dbfe326SDapeng Mi static int has_ibpb(void)
1638dbfe326SDapeng Mi {
1648dbfe326SDapeng Mi 	return this_cpu_has(X86_FEATURE_SPEC_CTRL) ||
1658dbfe326SDapeng Mi 	       this_cpu_has(X86_FEATURE_AMD_IBPB);
1668dbfe326SDapeng Mi }
1678dbfe326SDapeng Mi 
__loop(void)16850f8e27eSDapeng Mi static inline void __loop(void)
169a9f8b16fSGleb Natapov {
170a9f8b16fSGleb Natapov 	unsigned long tmp, tmp2, tmp3;
17138b5b426SDapeng Mi 	u32 global_ctl = 0;
17238b5b426SDapeng Mi 	u32 eax = 0;
17338b5b426SDapeng Mi 	u32 edx = 0;
174a9f8b16fSGleb Natapov 
1758dbfe326SDapeng Mi 	if (this_cpu_has(X86_FEATURE_CLFLUSH) && has_ibpb())
1768dbfe326SDapeng Mi 		_loop_asm("nop", "clflush (%1)", "wrmsr");
1778dbfe326SDapeng Mi 	else if (this_cpu_has(X86_FEATURE_CLFLUSH))
1788dbfe326SDapeng Mi 		_loop_asm("nop", "clflush (%1)", "nop");
1798dbfe326SDapeng Mi 	else if (has_ibpb())
1808dbfe326SDapeng Mi 		_loop_asm("nop", "nop", "wrmsr");
18138b5b426SDapeng Mi 	else
1828dbfe326SDapeng Mi 		_loop_asm("nop", "nop", "nop");
18350f8e27eSDapeng Mi }
184a9f8b16fSGleb Natapov 
18550f8e27eSDapeng Mi /*
18650f8e27eSDapeng Mi  * Enable and disable counters in a whole asm blob to ensure
18750f8e27eSDapeng Mi  * no other instructions are counted in the window between
18850f8e27eSDapeng Mi  * counters enabling and really LOOP_ASM code executing.
18950f8e27eSDapeng Mi  * Thus counters can verify instructions and branches events
19050f8e27eSDapeng Mi  * against precise counts instead of a rough valid count range.
19150f8e27eSDapeng Mi  */
__precise_loop(u64 cntrs)19250f8e27eSDapeng Mi static inline void __precise_loop(u64 cntrs)
19350f8e27eSDapeng Mi {
19450f8e27eSDapeng Mi 	unsigned long tmp, tmp2, tmp3;
19538b5b426SDapeng Mi 	u32 global_ctl = pmu.msr_global_ctl;
19650f8e27eSDapeng Mi 	u32 eax = cntrs & (BIT_ULL(32) - 1);
19750f8e27eSDapeng Mi 	u32 edx = cntrs >> 32;
19850f8e27eSDapeng Mi 
1998dbfe326SDapeng Mi 	if (this_cpu_has(X86_FEATURE_CLFLUSH) && has_ibpb())
2008dbfe326SDapeng Mi 		_loop_asm("wrmsr", "clflush (%1)", "wrmsr");
2018dbfe326SDapeng Mi 	else if (this_cpu_has(X86_FEATURE_CLFLUSH))
2028dbfe326SDapeng Mi 		_loop_asm("wrmsr", "clflush (%1)", "nop");
2038dbfe326SDapeng Mi 	else if (has_ibpb())
2048dbfe326SDapeng Mi 		_loop_asm("wrmsr", "nop", "wrmsr");
20538b5b426SDapeng Mi 	else
2068dbfe326SDapeng Mi 		_loop_asm("wrmsr", "nop", "nop");
20750f8e27eSDapeng Mi }
20850f8e27eSDapeng Mi 
loop(u64 cntrs)20950f8e27eSDapeng Mi static inline void loop(u64 cntrs)
21050f8e27eSDapeng Mi {
21150f8e27eSDapeng Mi 	if (!this_cpu_has_perf_global_ctrl())
21250f8e27eSDapeng Mi 		__loop();
21350f8e27eSDapeng Mi 	else
21450f8e27eSDapeng Mi 		__precise_loop(cntrs);
215a9f8b16fSGleb Natapov }
216a9f8b16fSGleb Natapov 
adjust_events_range(struct pmu_event * gp_events,int instruction_idx,int branch_idx,int branch_miss_idx)21789126fa4SDapeng Mi static void adjust_events_range(struct pmu_event *gp_events,
21828437cdbSDapeng Mi 				int instruction_idx, int branch_idx,
21928437cdbSDapeng Mi 				int branch_miss_idx)
22089126fa4SDapeng Mi {
22189126fa4SDapeng Mi 	/*
22289126fa4SDapeng Mi 	 * If HW supports GLOBAL_CTRL MSR, enabling and disabling PMCs are
22389126fa4SDapeng Mi 	 * moved in __precise_loop(). Thus, instructions and branches events
22489126fa4SDapeng Mi 	 * can be verified against a precise count instead of a rough range.
22589126fa4SDapeng Mi 	 *
22689126fa4SDapeng Mi 	 * Skip the precise checks on AMD, as AMD CPUs count VMRUN as a branch
22789126fa4SDapeng Mi 	 * instruction in guest context, which* leads to intermittent failures
22889126fa4SDapeng Mi 	 * as the counts will vary depending on how many asynchronous VM-Exits
22989126fa4SDapeng Mi 	 * occur while running the measured code, e.g. if the host takes IRQs.
23089126fa4SDapeng Mi 	 */
23189126fa4SDapeng Mi 	if (pmu.is_intel && this_cpu_has_perf_global_ctrl()) {
23289126fa4SDapeng Mi 		gp_events[instruction_idx].min = LOOP_INSNS;
23389126fa4SDapeng Mi 		gp_events[instruction_idx].max = LOOP_INSNS;
23489126fa4SDapeng Mi 		gp_events[branch_idx].min = LOOP_BRANCHES;
23589126fa4SDapeng Mi 		gp_events[branch_idx].max = LOOP_BRANCHES;
23689126fa4SDapeng Mi 	}
23728437cdbSDapeng Mi 
23828437cdbSDapeng Mi 	/*
23928437cdbSDapeng Mi 	 * For CPUs without IBPB support, no way to force to trigger a branch
24028437cdbSDapeng Mi 	 * miss and the measured branch misses is possible to be 0.  Thus
24128437cdbSDapeng Mi 	 * overwrite the lower boundary of branch misses event to 0 to avoid
24228437cdbSDapeng Mi 	 * false positive.
24328437cdbSDapeng Mi 	 */
24428437cdbSDapeng Mi 	if (!has_ibpb())
24528437cdbSDapeng Mi 		gp_events[branch_miss_idx].min = 0;
24689126fa4SDapeng Mi }
24789126fa4SDapeng Mi 
248a9f8b16fSGleb Natapov volatile uint64_t irq_received;
249a9f8b16fSGleb Natapov 
cnt_overflow(isr_regs_t * regs)250a9f8b16fSGleb Natapov static void cnt_overflow(isr_regs_t *regs)
251a9f8b16fSGleb Natapov {
252a9f8b16fSGleb Natapov 	irq_received++;
253c595c361SMingwei Zhang 	apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);
254a9f8b16fSGleb Natapov 	apic_write(APIC_EOI, 0);
255a9f8b16fSGleb Natapov }
256a9f8b16fSGleb Natapov 
check_irq(void)257a9f8b16fSGleb Natapov static bool check_irq(void)
258a9f8b16fSGleb Natapov {
259a9f8b16fSGleb Natapov 	int i;
260a9f8b16fSGleb Natapov 	irq_received = 0;
261787f0aebSMaxim Levitsky 	sti();
262a9f8b16fSGleb Natapov 	for (i = 0; i < 100000 && !irq_received; i++)
263a9f8b16fSGleb Natapov 		asm volatile("pause");
264787f0aebSMaxim Levitsky 	cli();
265a9f8b16fSGleb Natapov 	return irq_received;
266a9f8b16fSGleb Natapov }
267a9f8b16fSGleb Natapov 
is_gp(pmu_counter_t * evt)268a9f8b16fSGleb Natapov static bool is_gp(pmu_counter_t *evt)
269a9f8b16fSGleb Natapov {
270b883751aSLike Xu 	if (!pmu.is_intel)
271b883751aSLike Xu 		return true;
272b883751aSLike Xu 
27322f2901aSLike Xu 	return evt->ctr < MSR_CORE_PERF_FIXED_CTR0 ||
27422f2901aSLike Xu 		evt->ctr >= MSR_IA32_PMC0;
275a9f8b16fSGleb Natapov }
276a9f8b16fSGleb Natapov 
event_to_global_idx(pmu_counter_t * cnt)277a9f8b16fSGleb Natapov static int event_to_global_idx(pmu_counter_t *cnt)
278a9f8b16fSGleb Natapov {
279b883751aSLike Xu 	if (pmu.is_intel)
280cda64e80SLike Xu 		return cnt->ctr - (is_gp(cnt) ? pmu.msr_gp_counter_base :
281a9f8b16fSGleb Natapov 			(MSR_CORE_PERF_FIXED_CTR0 - FIXED_CNT_INDEX));
282b883751aSLike Xu 
283b883751aSLike Xu 	if (pmu.msr_gp_counter_base == MSR_F15H_PERF_CTR0)
284b883751aSLike Xu 		return (cnt->ctr - pmu.msr_gp_counter_base) / 2;
285b883751aSLike Xu 	else
286b883751aSLike Xu 		return cnt->ctr - pmu.msr_gp_counter_base;
287a9f8b16fSGleb Natapov }
288a9f8b16fSGleb Natapov 
get_counter_event(pmu_counter_t * cnt)289a9f8b16fSGleb Natapov static struct pmu_event* get_counter_event(pmu_counter_t *cnt)
290a9f8b16fSGleb Natapov {
291a9f8b16fSGleb Natapov 	if (is_gp(cnt)) {
292a9f8b16fSGleb Natapov 		int i;
293a9f8b16fSGleb Natapov 
2947c648ce2SLike Xu 		for (i = 0; i < gp_events_size; i++)
295a9f8b16fSGleb Natapov 			if (gp_events[i].unit_sel == (cnt->config & 0xffff))
296a9f8b16fSGleb Natapov 				return &gp_events[i];
2979c07c92bSDapeng Mi 	} else {
2989c07c92bSDapeng Mi 		unsigned int idx = cnt->ctr - MSR_CORE_PERF_FIXED_CTR0;
2999c07c92bSDapeng Mi 
3009c07c92bSDapeng Mi 		if (idx < ARRAY_SIZE(fixed_events))
3019c07c92bSDapeng Mi 			return &fixed_events[idx];
3029c07c92bSDapeng Mi 	}
303a9f8b16fSGleb Natapov 
304a9f8b16fSGleb Natapov 	return (void*)0;
305a9f8b16fSGleb Natapov }
306a9f8b16fSGleb Natapov 
global_enable(pmu_counter_t * cnt)307a9f8b16fSGleb Natapov static void global_enable(pmu_counter_t *cnt)
308a9f8b16fSGleb Natapov {
30962ba5036SLike Xu 	if (!this_cpu_has_perf_global_ctrl())
31062ba5036SLike Xu 		return;
31162ba5036SLike Xu 
312a9f8b16fSGleb Natapov 	cnt->idx = event_to_global_idx(cnt);
3138a2866d1SLike Xu 	wrmsr(pmu.msr_global_ctl, rdmsr(pmu.msr_global_ctl) | BIT_ULL(cnt->idx));
314a9f8b16fSGleb Natapov }
315a9f8b16fSGleb Natapov 
global_disable(pmu_counter_t * cnt)316a9f8b16fSGleb Natapov static void global_disable(pmu_counter_t *cnt)
317a9f8b16fSGleb Natapov {
31862ba5036SLike Xu 	if (!this_cpu_has_perf_global_ctrl())
31962ba5036SLike Xu 		return;
32062ba5036SLike Xu 
3218a2866d1SLike Xu 	wrmsr(pmu.msr_global_ctl, rdmsr(pmu.msr_global_ctl) & ~BIT_ULL(cnt->idx));
322a9f8b16fSGleb Natapov }
323a9f8b16fSGleb Natapov 
__start_event(pmu_counter_t * evt,uint64_t count)324e9e7577bSLike Xu static void __start_event(pmu_counter_t *evt, uint64_t count)
325a9f8b16fSGleb Natapov {
326e9e7577bSLike Xu     evt->count = count;
327a9f8b16fSGleb Natapov     wrmsr(evt->ctr, evt->count);
328cda64e80SLike Xu     if (is_gp(evt)) {
329cda64e80SLike Xu 	    wrmsr(MSR_GP_EVENT_SELECTx(event_to_global_idx(evt)),
330a9f8b16fSGleb Natapov 		  evt->config | EVNTSEL_EN);
331cda64e80SLike Xu     } else {
332a9f8b16fSGleb Natapov 	    uint32_t ctrl = rdmsr(MSR_CORE_PERF_FIXED_CTR_CTRL);
333a9f8b16fSGleb Natapov 	    int shift = (evt->ctr - MSR_CORE_PERF_FIXED_CTR0) * 4;
334a9f8b16fSGleb Natapov 	    uint32_t usrospmi = 0;
335a9f8b16fSGleb Natapov 
336a9f8b16fSGleb Natapov 	    if (evt->config & EVNTSEL_OS)
337a9f8b16fSGleb Natapov 		    usrospmi |= (1 << 0);
338a9f8b16fSGleb Natapov 	    if (evt->config & EVNTSEL_USR)
339a9f8b16fSGleb Natapov 		    usrospmi |= (1 << 1);
340a9f8b16fSGleb Natapov 	    if (evt->config & EVNTSEL_INT)
341a9f8b16fSGleb Natapov 		    usrospmi |= (1 << 3); // PMI on overflow
342a9f8b16fSGleb Natapov 	    ctrl = (ctrl & ~(0xf << shift)) | (usrospmi << shift);
343a9f8b16fSGleb Natapov 	    wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, ctrl);
344a9f8b16fSGleb Natapov     }
3455a2cb3e6SLike Xu     apic_write(APIC_LVTPC, PMI_VECTOR);
346a9f8b16fSGleb Natapov }
347a9f8b16fSGleb Natapov 
start_event(pmu_counter_t * evt)348e9e7577bSLike Xu static void start_event(pmu_counter_t *evt)
349e9e7577bSLike Xu {
350e9e7577bSLike Xu 	__start_event(evt, 0);
35150f8e27eSDapeng Mi 	global_enable(evt);
352e9e7577bSLike Xu }
353e9e7577bSLike Xu 
__stop_event(pmu_counter_t * evt)35450f8e27eSDapeng Mi static void __stop_event(pmu_counter_t *evt)
355a9f8b16fSGleb Natapov {
356cda64e80SLike Xu 	if (is_gp(evt)) {
357cda64e80SLike Xu 		wrmsr(MSR_GP_EVENT_SELECTx(event_to_global_idx(evt)),
358a9f8b16fSGleb Natapov 		      evt->config & ~EVNTSEL_EN);
359cda64e80SLike Xu 	} else {
360a9f8b16fSGleb Natapov 		uint32_t ctrl = rdmsr(MSR_CORE_PERF_FIXED_CTR_CTRL);
361a9f8b16fSGleb Natapov 		int shift = (evt->ctr - MSR_CORE_PERF_FIXED_CTR0) * 4;
362a9f8b16fSGleb Natapov 		wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, ctrl & ~(0xf << shift));
363a9f8b16fSGleb Natapov 	}
364a9f8b16fSGleb Natapov 	evt->count = rdmsr(evt->ctr);
365a9f8b16fSGleb Natapov }
366a9f8b16fSGleb Natapov 
stop_event(pmu_counter_t * evt)36750f8e27eSDapeng Mi static void stop_event(pmu_counter_t *evt)
36850f8e27eSDapeng Mi {
36950f8e27eSDapeng Mi 	global_disable(evt);
37050f8e27eSDapeng Mi 	__stop_event(evt);
37150f8e27eSDapeng Mi }
37250f8e27eSDapeng Mi 
measure_many(pmu_counter_t * evt,int count)3738554261fSLike Xu static noinline void measure_many(pmu_counter_t *evt, int count)
374a9f8b16fSGleb Natapov {
375a9f8b16fSGleb Natapov 	int i;
37650f8e27eSDapeng Mi 	u64 cntrs = 0;
37750f8e27eSDapeng Mi 
37850f8e27eSDapeng Mi 	for (i = 0; i < count; i++) {
37950f8e27eSDapeng Mi 		__start_event(&evt[i], 0);
38050f8e27eSDapeng Mi 		cntrs |= BIT_ULL(event_to_global_idx(&evt[i]));
38150f8e27eSDapeng Mi 	}
38250f8e27eSDapeng Mi 	loop(cntrs);
383a9f8b16fSGleb Natapov 	for (i = 0; i < count; i++)
38450f8e27eSDapeng Mi 		__stop_event(&evt[i]);
385a9f8b16fSGleb Natapov }
386a9f8b16fSGleb Natapov 
measure_one(pmu_counter_t * evt)3878554261fSLike Xu static void measure_one(pmu_counter_t *evt)
3888554261fSLike Xu {
3898554261fSLike Xu 	measure_many(evt, 1);
3908554261fSLike Xu }
3918554261fSLike Xu 
__measure(pmu_counter_t * evt,uint64_t count)392e9e7577bSLike Xu static noinline void __measure(pmu_counter_t *evt, uint64_t count)
393e9e7577bSLike Xu {
39450f8e27eSDapeng Mi 	u64 cntrs = BIT_ULL(event_to_global_idx(evt));
39550f8e27eSDapeng Mi 
396e9e7577bSLike Xu 	__start_event(evt, count);
39750f8e27eSDapeng Mi 	loop(cntrs);
39850f8e27eSDapeng Mi 	__stop_event(evt);
399e9e7577bSLike Xu }
400e9e7577bSLike Xu 
verify_event(uint64_t count,struct pmu_event * e)401a9f8b16fSGleb Natapov static bool verify_event(uint64_t count, struct pmu_event *e)
402a9f8b16fSGleb Natapov {
4039c07c92bSDapeng Mi 	bool pass;
404d24d3381SDapeng Mi 
4059c07c92bSDapeng Mi 	if (!e)
4069c07c92bSDapeng Mi 		return false;
4079c07c92bSDapeng Mi 
4089c07c92bSDapeng Mi 	pass = count >= e->min && count <= e->max;
409d24d3381SDapeng Mi 	if (!pass)
410d24d3381SDapeng Mi 		printf("FAIL: %d <= %"PRId64" <= %d\n", e->min, count, e->max);
411d24d3381SDapeng Mi 
412d24d3381SDapeng Mi 	return pass;
413a9f8b16fSGleb Natapov }
414a9f8b16fSGleb Natapov 
verify_counter(pmu_counter_t * cnt)415a9f8b16fSGleb Natapov static bool verify_counter(pmu_counter_t *cnt)
416a9f8b16fSGleb Natapov {
417a9f8b16fSGleb Natapov 	return verify_event(cnt->count, get_counter_event(cnt));
418a9f8b16fSGleb Natapov }
419a9f8b16fSGleb Natapov 
check_gp_counter(struct pmu_event * evt)420a9f8b16fSGleb Natapov static void check_gp_counter(struct pmu_event *evt)
421a9f8b16fSGleb Natapov {
422a9f8b16fSGleb Natapov 	pmu_counter_t cnt = {
423a9f8b16fSGleb Natapov 		.config = EVNTSEL_OS | EVNTSEL_USR | evt->unit_sel,
424a9f8b16fSGleb Natapov 	};
425a9f8b16fSGleb Natapov 	int i;
426a9f8b16fSGleb Natapov 
427cda64e80SLike Xu 	for (i = 0; i < pmu.nr_gp_counters; i++) {
428cda64e80SLike Xu 		cnt.ctr = MSR_GP_COUNTERx(i);
4298554261fSLike Xu 		measure_one(&cnt);
430a299895bSThomas Huth 		report(verify_event(cnt.count, evt), "%s-%d", evt->name, i);
431a9f8b16fSGleb Natapov 	}
432a9f8b16fSGleb Natapov }
433a9f8b16fSGleb Natapov 
check_gp_counters(void)434a9f8b16fSGleb Natapov static void check_gp_counters(void)
435a9f8b16fSGleb Natapov {
436a9f8b16fSGleb Natapov 	int i;
437a9f8b16fSGleb Natapov 
4387c648ce2SLike Xu 	for (i = 0; i < gp_events_size; i++)
4392719b92cSYang Weijiang 		if (pmu_gp_counter_is_available(i))
440a9f8b16fSGleb Natapov 			check_gp_counter(&gp_events[i]);
441a9f8b16fSGleb Natapov 		else
442a9f8b16fSGleb Natapov 			printf("GP event '%s' is disabled\n",
443a9f8b16fSGleb Natapov 					gp_events[i].name);
444a9f8b16fSGleb Natapov }
445a9f8b16fSGleb Natapov 
check_fixed_counters(void)446a9f8b16fSGleb Natapov static void check_fixed_counters(void)
447a9f8b16fSGleb Natapov {
448a9f8b16fSGleb Natapov 	pmu_counter_t cnt = {
449a9f8b16fSGleb Natapov 		.config = EVNTSEL_OS | EVNTSEL_USR,
450a9f8b16fSGleb Natapov 	};
451a9f8b16fSGleb Natapov 	int i;
452a9f8b16fSGleb Natapov 
4539c07c92bSDapeng Mi 	for (i = 0; i < fixed_counters_num; i++) {
454a9f8b16fSGleb Natapov 		cnt.ctr = fixed_events[i].unit_sel;
4558554261fSLike Xu 		measure_one(&cnt);
4562719b92cSYang Weijiang 		report(verify_event(cnt.count, &fixed_events[i]), "fixed-%d", i);
457a9f8b16fSGleb Natapov 	}
458a9f8b16fSGleb Natapov }
459a9f8b16fSGleb Natapov 
check_counters_many(void)460a9f8b16fSGleb Natapov static void check_counters_many(void)
461a9f8b16fSGleb Natapov {
462f21c809eSDapeng Mi 	pmu_counter_t cnt[48];
463a9f8b16fSGleb Natapov 	int i, n;
464a9f8b16fSGleb Natapov 
465414ee7d1SSean Christopherson 	for (i = 0, n = 0; n < pmu.nr_gp_counters; i++) {
4662719b92cSYang Weijiang 		if (!pmu_gp_counter_is_available(i))
467a9f8b16fSGleb Natapov 			continue;
468a9f8b16fSGleb Natapov 
469cda64e80SLike Xu 		cnt[n].ctr = MSR_GP_COUNTERx(n);
4704ac45293SWei Huang 		cnt[n].config = EVNTSEL_OS | EVNTSEL_USR |
4717c648ce2SLike Xu 			gp_events[i % gp_events_size].unit_sel;
472a9f8b16fSGleb Natapov 		n++;
473a9f8b16fSGleb Natapov 	}
4749c07c92bSDapeng Mi 	for (i = 0; i < fixed_counters_num; i++) {
475a9f8b16fSGleb Natapov 		cnt[n].ctr = fixed_events[i].unit_sel;
476a9f8b16fSGleb Natapov 		cnt[n].config = EVNTSEL_OS | EVNTSEL_USR;
477a9f8b16fSGleb Natapov 		n++;
478a9f8b16fSGleb Natapov 	}
479a9f8b16fSGleb Natapov 
480f21c809eSDapeng Mi 	assert(n <= ARRAY_SIZE(cnt));
4818554261fSLike Xu 	measure_many(cnt, n);
482a9f8b16fSGleb Natapov 
483a9f8b16fSGleb Natapov 	for (i = 0; i < n; i++)
484a9f8b16fSGleb Natapov 		if (!verify_counter(&cnt[i]))
485a9f8b16fSGleb Natapov 			break;
486a9f8b16fSGleb Natapov 
487a299895bSThomas Huth 	report(i == n, "all counters");
488a9f8b16fSGleb Natapov }
489a9f8b16fSGleb Natapov 
measure_for_overflow(pmu_counter_t * cnt)4907ec3b67aSLike Xu static uint64_t measure_for_overflow(pmu_counter_t *cnt)
4917ec3b67aSLike Xu {
4927ec3b67aSLike Xu 	__measure(cnt, 0);
4937ec3b67aSLike Xu 	/*
4947ec3b67aSLike Xu 	 * To generate overflow, i.e. roll over to '0', the initial count just
4957ec3b67aSLike Xu 	 * needs to be preset to the negative expected count.  However, as per
4967ec3b67aSLike Xu 	 * Intel's SDM, the preset count needs to be incremented by 1 to ensure
4977ec3b67aSLike Xu 	 * the overflow interrupt is generated immediately instead of possibly
4987ec3b67aSLike Xu 	 * waiting for the overflow to propagate through the counter.
4997ec3b67aSLike Xu 	 */
5007ec3b67aSLike Xu 	assert(cnt->count > 1);
5017ec3b67aSLike Xu 	return 1 - cnt->count;
5027ec3b67aSLike Xu }
5037ec3b67aSLike Xu 
check_counter_overflow(void)504a9f8b16fSGleb Natapov static void check_counter_overflow(void)
505a9f8b16fSGleb Natapov {
506a9f8b16fSGleb Natapov 	int i;
50785c75578SDapeng Mi 	uint64_t overflow_preset;
50885c75578SDapeng Mi 	int instruction_idx = pmu.is_intel ?
50985c75578SDapeng Mi 			      INTEL_INSTRUCTIONS_IDX :
51085c75578SDapeng Mi 			      AMD_INSTRUCTIONS_IDX;
51185c75578SDapeng Mi 
512a9f8b16fSGleb Natapov 	pmu_counter_t cnt = {
513cda64e80SLike Xu 		.ctr = MSR_GP_COUNTERx(0),
51485c75578SDapeng Mi 		.config = EVNTSEL_OS | EVNTSEL_USR |
51585c75578SDapeng Mi 			  gp_events[instruction_idx].unit_sel /* instructions */,
516a9f8b16fSGleb Natapov 	};
5177ec3b67aSLike Xu 	overflow_preset = measure_for_overflow(&cnt);
518a9f8b16fSGleb Natapov 
519a9f8b16fSGleb Natapov 	/* clear status before test */
52062ba5036SLike Xu 	if (this_cpu_has_perf_global_status())
5218a2866d1SLike Xu 		pmu_clear_global_status();
522a9f8b16fSGleb Natapov 
5235bba1769SAndrew Jones 	report_prefix_push("overflow");
5245bba1769SAndrew Jones 
525cda64e80SLike Xu 	for (i = 0; i < pmu.nr_gp_counters + 1; i++) {
526a9f8b16fSGleb Natapov 		uint64_t status;
527a9f8b16fSGleb Natapov 		int idx;
52833cfc1b0SNadav Amit 
5297ec3b67aSLike Xu 		cnt.count = overflow_preset;
530cda64e80SLike Xu 		if (pmu_use_full_writes())
531414ee7d1SSean Christopherson 			cnt.count &= (1ull << pmu.gp_counter_width) - 1;
53233cfc1b0SNadav Amit 
533414ee7d1SSean Christopherson 		if (i == pmu.nr_gp_counters) {
534b883751aSLike Xu 			if (!pmu.is_intel)
535b883751aSLike Xu 				break;
536b883751aSLike Xu 
537a9f8b16fSGleb Natapov 			cnt.ctr = fixed_events[0].unit_sel;
5387ec3b67aSLike Xu 			cnt.count = measure_for_overflow(&cnt);
539cda64e80SLike Xu 			cnt.count &= (1ull << pmu.gp_counter_width) - 1;
540cda64e80SLike Xu 		} else {
541cda64e80SLike Xu 			cnt.ctr = MSR_GP_COUNTERx(i);
54233cfc1b0SNadav Amit 		}
54333cfc1b0SNadav Amit 
544a9f8b16fSGleb Natapov 		if (i % 2)
545a9f8b16fSGleb Natapov 			cnt.config |= EVNTSEL_INT;
546a9f8b16fSGleb Natapov 		else
547a9f8b16fSGleb Natapov 			cnt.config &= ~EVNTSEL_INT;
548a9f8b16fSGleb Natapov 		idx = event_to_global_idx(&cnt);
549e9e7577bSLike Xu 		__measure(&cnt, cnt.count);
550b883751aSLike Xu 		if (pmu.is_intel)
551a299895bSThomas Huth 			report(cnt.count == 1, "cntr-%d", i);
552b883751aSLike Xu 		else
553b883751aSLike Xu 			report(cnt.count == 0xffffffffffff || cnt.count < 7, "cntr-%d", i);
55462ba5036SLike Xu 
55562ba5036SLike Xu 		if (!this_cpu_has_perf_global_status())
55662ba5036SLike Xu 			continue;
55762ba5036SLike Xu 
5588a2866d1SLike Xu 		status = rdmsr(pmu.msr_global_status);
559a299895bSThomas Huth 		report(status & (1ull << idx), "status-%d", i);
5608a2866d1SLike Xu 		wrmsr(pmu.msr_global_status_clr, status);
5618a2866d1SLike Xu 		status = rdmsr(pmu.msr_global_status);
562a299895bSThomas Huth 		report(!(status & (1ull << idx)), "status clear-%d", i);
563a299895bSThomas Huth 		report(check_irq() == (i % 2), "irq-%d", i);
564a9f8b16fSGleb Natapov 	}
5655bba1769SAndrew Jones 
5665bba1769SAndrew Jones 	report_prefix_pop();
567a9f8b16fSGleb Natapov }
568a9f8b16fSGleb Natapov 
check_gp_counter_cmask(void)569a9f8b16fSGleb Natapov static void check_gp_counter_cmask(void)
570a9f8b16fSGleb Natapov {
57185c75578SDapeng Mi 	int instruction_idx = pmu.is_intel ?
57285c75578SDapeng Mi 			      INTEL_INSTRUCTIONS_IDX :
57385c75578SDapeng Mi 			      AMD_INSTRUCTIONS_IDX;
57485c75578SDapeng Mi 
575a9f8b16fSGleb Natapov 	pmu_counter_t cnt = {
576cda64e80SLike Xu 		.ctr = MSR_GP_COUNTERx(0),
57785c75578SDapeng Mi 		.config = EVNTSEL_OS | EVNTSEL_USR |
57885c75578SDapeng Mi 			  gp_events[instruction_idx].unit_sel /* instructions */,
579a9f8b16fSGleb Natapov 	};
580a9f8b16fSGleb Natapov 	cnt.config |= (0x2 << EVNTSEL_CMASK_SHIFT);
5818554261fSLike Xu 	measure_one(&cnt);
58285c75578SDapeng Mi 	report(cnt.count < gp_events[instruction_idx].min, "cmask");
583a9f8b16fSGleb Natapov }
584a9f8b16fSGleb Natapov 
do_rdpmc_fast(void * ptr)585ca1b9de9SNadav Amit static void do_rdpmc_fast(void *ptr)
586ca1b9de9SNadav Amit {
587ca1b9de9SNadav Amit 	pmu_counter_t *cnt = ptr;
588ca1b9de9SNadav Amit 	uint32_t idx = (uint32_t)cnt->idx | (1u << 31);
589ca1b9de9SNadav Amit 
590ca1b9de9SNadav Amit 	if (!is_gp(cnt))
591ca1b9de9SNadav Amit 		idx |= 1 << 30;
592ca1b9de9SNadav Amit 
593ca1b9de9SNadav Amit 	cnt->count = rdpmc(idx);
594ca1b9de9SNadav Amit }
595ca1b9de9SNadav Amit 
596ca1b9de9SNadav Amit 
check_rdpmc(void)597a9f8b16fSGleb Natapov static void check_rdpmc(void)
598a9f8b16fSGleb Natapov {
59922f2901aSLike Xu 	uint64_t val = 0xff0123456789ull;
600ca1b9de9SNadav Amit 	bool exc;
601a9f8b16fSGleb Natapov 	int i;
602a9f8b16fSGleb Natapov 
6035bba1769SAndrew Jones 	report_prefix_push("rdpmc");
6045bba1769SAndrew Jones 
605414ee7d1SSean Christopherson 	for (i = 0; i < pmu.nr_gp_counters; i++) {
60633cfc1b0SNadav Amit 		uint64_t x;
607ca1b9de9SNadav Amit 		pmu_counter_t cnt = {
608cda64e80SLike Xu 			.ctr = MSR_GP_COUNTERx(i),
609ca1b9de9SNadav Amit 			.idx = i
610ca1b9de9SNadav Amit 		};
61133cfc1b0SNadav Amit 
61233cfc1b0SNadav Amit 	        /*
61322f2901aSLike Xu 	         * Without full-width writes, only the low 32 bits are writable,
61422f2901aSLike Xu 	         * and the value is sign-extended.
61533cfc1b0SNadav Amit 	         */
616cda64e80SLike Xu 		if (pmu.msr_gp_counter_base == MSR_IA32_PERFCTR0)
61733cfc1b0SNadav Amit 			x = (uint64_t)(int64_t)(int32_t)val;
61822f2901aSLike Xu 		else
61922f2901aSLike Xu 			x = (uint64_t)(int64_t)val;
62033cfc1b0SNadav Amit 
62133cfc1b0SNadav Amit 		/* Mask according to the number of supported bits */
622414ee7d1SSean Christopherson 		x &= (1ull << pmu.gp_counter_width) - 1;
62333cfc1b0SNadav Amit 
624cda64e80SLike Xu 		wrmsr(MSR_GP_COUNTERx(i), val);
625a299895bSThomas Huth 		report(rdpmc(i) == x, "cntr-%d", i);
626ca1b9de9SNadav Amit 
627ca1b9de9SNadav Amit 		exc = test_for_exception(GP_VECTOR, do_rdpmc_fast, &cnt);
628ca1b9de9SNadav Amit 		if (exc)
629ca1b9de9SNadav Amit 			report_skip("fast-%d", i);
630ca1b9de9SNadav Amit 		else
631a299895bSThomas Huth 			report(cnt.count == (u32)val, "fast-%d", i);
632a9f8b16fSGleb Natapov 	}
6339c07c92bSDapeng Mi 	for (i = 0; i < fixed_counters_num; i++) {
634414ee7d1SSean Christopherson 		uint64_t x = val & ((1ull << pmu.fixed_counter_width) - 1);
635ca1b9de9SNadav Amit 		pmu_counter_t cnt = {
636ca1b9de9SNadav Amit 			.ctr = MSR_CORE_PERF_FIXED_CTR0 + i,
637ca1b9de9SNadav Amit 			.idx = i
638ca1b9de9SNadav Amit 		};
63933cfc1b0SNadav Amit 
6403f914933SLike Xu 		wrmsr(MSR_PERF_FIXED_CTRx(i), x);
641a299895bSThomas Huth 		report(rdpmc(i | (1 << 30)) == x, "fixed cntr-%d", i);
642ca1b9de9SNadav Amit 
643ca1b9de9SNadav Amit 		exc = test_for_exception(GP_VECTOR, do_rdpmc_fast, &cnt);
644ca1b9de9SNadav Amit 		if (exc)
645ca1b9de9SNadav Amit 			report_skip("fixed fast-%d", i);
646ca1b9de9SNadav Amit 		else
647a299895bSThomas Huth 			report(cnt.count == (u32)x, "fixed fast-%d", i);
648a9f8b16fSGleb Natapov 	}
6495bba1769SAndrew Jones 
6505bba1769SAndrew Jones 	report_prefix_pop();
651a9f8b16fSGleb Natapov }
652a9f8b16fSGleb Natapov 
check_running_counter_wrmsr(void)653ddade902SEric Hankland static void check_running_counter_wrmsr(void)
654ddade902SEric Hankland {
65559ca1413SEric Hankland 	uint64_t status;
65622f2901aSLike Xu 	uint64_t count;
65785c75578SDapeng Mi 	unsigned int instruction_idx = pmu.is_intel ?
65885c75578SDapeng Mi 				       INTEL_INSTRUCTIONS_IDX :
65985c75578SDapeng Mi 				       AMD_INSTRUCTIONS_IDX;
66085c75578SDapeng Mi 
661ddade902SEric Hankland 	pmu_counter_t evt = {
662cda64e80SLike Xu 		.ctr = MSR_GP_COUNTERx(0),
66385c75578SDapeng Mi 		.config = EVNTSEL_OS | EVNTSEL_USR |
66485c75578SDapeng Mi 			  gp_events[instruction_idx].unit_sel,
665ddade902SEric Hankland 	};
666ddade902SEric Hankland 
66759ca1413SEric Hankland 	report_prefix_push("running counter wrmsr");
66859ca1413SEric Hankland 
669ddade902SEric Hankland 	start_event(&evt);
67050f8e27eSDapeng Mi 	__loop();
671cda64e80SLike Xu 	wrmsr(MSR_GP_COUNTERx(0), 0);
672ddade902SEric Hankland 	stop_event(&evt);
67385c75578SDapeng Mi 	report(evt.count < gp_events[instruction_idx].min, "cntr");
67459ca1413SEric Hankland 
67559ca1413SEric Hankland 	/* clear status before overflow test */
67662ba5036SLike Xu 	if (this_cpu_has_perf_global_status())
6778a2866d1SLike Xu 		pmu_clear_global_status();
67859ca1413SEric Hankland 
67959ca1413SEric Hankland 	start_event(&evt);
68022f2901aSLike Xu 
68122f2901aSLike Xu 	count = -1;
682cda64e80SLike Xu 	if (pmu_use_full_writes())
683414ee7d1SSean Christopherson 		count &= (1ull << pmu.gp_counter_width) - 1;
68422f2901aSLike Xu 
685cda64e80SLike Xu 	wrmsr(MSR_GP_COUNTERx(0), count);
68622f2901aSLike Xu 
68750f8e27eSDapeng Mi 	__loop();
68859ca1413SEric Hankland 	stop_event(&evt);
68962ba5036SLike Xu 
69062ba5036SLike Xu 	if (this_cpu_has_perf_global_status()) {
6918a2866d1SLike Xu 		status = rdmsr(pmu.msr_global_status);
6928a2866d1SLike Xu 		report(status & 1, "status msr bit");
69362ba5036SLike Xu 	}
69459ca1413SEric Hankland 
69559ca1413SEric Hankland 	report_prefix_pop();
696ddade902SEric Hankland }
697ddade902SEric Hankland 
check_emulated_instr(void)69820cf9147SJim Mattson static void check_emulated_instr(void)
69920cf9147SJim Mattson {
7005dcbe0ddSDapeng Mi 	u32 eax, edx, ecx;
70120cf9147SJim Mattson 	uint64_t status, instr_start, brnch_start;
7028b547cc2SLike Xu 	uint64_t gp_counter_width = (1ull << pmu.gp_counter_width) - 1;
703f4e97f59SDapeng Mi 	unsigned int branch_idx = pmu.is_intel ?
704f4e97f59SDapeng Mi 				  INTEL_BRANCHES_IDX : AMD_BRANCHES_IDX;
70585c75578SDapeng Mi 	unsigned int instruction_idx = pmu.is_intel ?
70685c75578SDapeng Mi 				       INTEL_INSTRUCTIONS_IDX :
70785c75578SDapeng Mi 				       AMD_INSTRUCTIONS_IDX;
7085dcbe0ddSDapeng Mi 
70920cf9147SJim Mattson 	pmu_counter_t brnch_cnt = {
710cda64e80SLike Xu 		.ctr = MSR_GP_COUNTERx(0),
71120cf9147SJim Mattson 		/* branch instructions */
712b883751aSLike Xu 		.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[branch_idx].unit_sel,
71320cf9147SJim Mattson 	};
71420cf9147SJim Mattson 	pmu_counter_t instr_cnt = {
715cda64e80SLike Xu 		.ctr = MSR_GP_COUNTERx(1),
71620cf9147SJim Mattson 		/* instructions */
71785c75578SDapeng Mi 		.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[instruction_idx].unit_sel,
71820cf9147SJim Mattson 	};
71920cf9147SJim Mattson 	report_prefix_push("emulated instruction");
72020cf9147SJim Mattson 
72162ba5036SLike Xu 	if (this_cpu_has_perf_global_status())
7228a2866d1SLike Xu 		pmu_clear_global_status();
72320cf9147SJim Mattson 
7245dcbe0ddSDapeng Mi 	__start_event(&brnch_cnt, 0);
7255dcbe0ddSDapeng Mi 	__start_event(&instr_cnt, 0);
72620cf9147SJim Mattson 
7275dcbe0ddSDapeng Mi 	brnch_start = -KVM_FEP_BRANCHES;
7285dcbe0ddSDapeng Mi 	instr_start = -KVM_FEP_INSNS;
7298b547cc2SLike Xu 	wrmsr(MSR_GP_COUNTERx(0), brnch_start & gp_counter_width);
7308b547cc2SLike Xu 	wrmsr(MSR_GP_COUNTERx(1), instr_start & gp_counter_width);
73120cf9147SJim Mattson 
7325dcbe0ddSDapeng Mi 	if (this_cpu_has_perf_global_ctrl()) {
7335dcbe0ddSDapeng Mi 		eax = BIT(0) | BIT(1);
7345dcbe0ddSDapeng Mi 		ecx = pmu.msr_global_ctl;
7355dcbe0ddSDapeng Mi 		edx = 0;
7365dcbe0ddSDapeng Mi 		kvm_fep_asm("wrmsr");
7375dcbe0ddSDapeng Mi 	} else {
7385dcbe0ddSDapeng Mi 		eax = ecx = edx = 0;
7395dcbe0ddSDapeng Mi 		kvm_fep_asm("nop");
7405dcbe0ddSDapeng Mi 	}
74120cf9147SJim Mattson 
7425dcbe0ddSDapeng Mi 	__stop_event(&brnch_cnt);
7435dcbe0ddSDapeng Mi 	__stop_event(&instr_cnt);
74420cf9147SJim Mattson 
74520cf9147SJim Mattson 	// Check that the end count - start count is at least the expected
74620cf9147SJim Mattson 	// number of instructions and branches.
7475dcbe0ddSDapeng Mi 	if (this_cpu_has_perf_global_ctrl()) {
7485dcbe0ddSDapeng Mi 		report(instr_cnt.count - instr_start == KVM_FEP_INSNS,
74920cf9147SJim Mattson 		       "instruction count");
7505dcbe0ddSDapeng Mi 		report(brnch_cnt.count - brnch_start == KVM_FEP_BRANCHES,
75120cf9147SJim Mattson 		       "branch count");
7525dcbe0ddSDapeng Mi 	} else {
7535dcbe0ddSDapeng Mi 		report(instr_cnt.count - instr_start >= KVM_FEP_INSNS,
7545dcbe0ddSDapeng Mi 		       "instruction count");
7555dcbe0ddSDapeng Mi 		report(brnch_cnt.count - brnch_start >= KVM_FEP_BRANCHES,
7565dcbe0ddSDapeng Mi 		       "branch count");
7575dcbe0ddSDapeng Mi 	}
7585dcbe0ddSDapeng Mi 
75962ba5036SLike Xu 	if (this_cpu_has_perf_global_status()) {
76020cf9147SJim Mattson 		// Additionally check that those counters overflowed properly.
7618a2866d1SLike Xu 		status = rdmsr(pmu.msr_global_status);
7625dcbe0ddSDapeng Mi 		report(status & BIT_ULL(0), "branch counter overflow");
7635dcbe0ddSDapeng Mi 		report(status & BIT_ULL(1), "instruction counter overflow");
76462ba5036SLike Xu 	}
76520cf9147SJim Mattson 
76620cf9147SJim Mattson 	report_prefix_pop();
76720cf9147SJim Mattson }
76820cf9147SJim Mattson 
769006b089dSLike Xu #define XBEGIN_STARTED (~0u)
check_tsx_cycles(void)770006b089dSLike Xu static void check_tsx_cycles(void)
771006b089dSLike Xu {
772006b089dSLike Xu 	pmu_counter_t cnt;
773006b089dSLike Xu 	unsigned int i, ret = 0;
774006b089dSLike Xu 
775006b089dSLike Xu 	if (!this_cpu_has(X86_FEATURE_RTM))
776006b089dSLike Xu 		return;
777006b089dSLike Xu 
778006b089dSLike Xu 	report_prefix_push("TSX cycles");
779006b089dSLike Xu 
780006b089dSLike Xu 	for (i = 0; i < pmu.nr_gp_counters; i++) {
781006b089dSLike Xu 		cnt.ctr = MSR_GP_COUNTERx(i);
782006b089dSLike Xu 
783006b089dSLike Xu 		if (i == 2) {
784d4ae0a71SThomas Huth 			/* Transactional cycles committed only on gp counter 2 */
785006b089dSLike Xu 			cnt.config = EVNTSEL_OS | EVNTSEL_USR | 0x30000003c;
786006b089dSLike Xu 		} else {
787006b089dSLike Xu 			/* Transactional cycles */
788006b089dSLike Xu 			cnt.config = EVNTSEL_OS | EVNTSEL_USR | 0x10000003c;
789006b089dSLike Xu 		}
790006b089dSLike Xu 
791006b089dSLike Xu 		start_event(&cnt);
792006b089dSLike Xu 
793006b089dSLike Xu 		asm volatile("xbegin 1f\n\t"
794006b089dSLike Xu 				"1:\n\t"
795006b089dSLike Xu 				: "+a" (ret) :: "memory");
796006b089dSLike Xu 
797006b089dSLike Xu 		/* Generate a non-canonical #GP to trigger ABORT. */
798006b089dSLike Xu 		if (ret == XBEGIN_STARTED)
799006b089dSLike Xu 			*(int *)NONCANONICAL = 0;
800006b089dSLike Xu 
801006b089dSLike Xu 		stop_event(&cnt);
802006b089dSLike Xu 
803006b089dSLike Xu 		report(cnt.count > 0, "gp cntr-%d with a value of %" PRId64 "", i, cnt.count);
804006b089dSLike Xu 	}
805006b089dSLike Xu 
806006b089dSLike Xu 	report_prefix_pop();
807006b089dSLike Xu }
808006b089dSLike Xu 
warm_up(void)809f2a56148SDapeng Mi static void warm_up(void)
810f2a56148SDapeng Mi {
811f2a56148SDapeng Mi 	int i;
812f2a56148SDapeng Mi 
813f2a56148SDapeng Mi 	/*
814f2a56148SDapeng Mi 	 * Since cycles event is always run as the first event, there would be
815f2a56148SDapeng Mi 	 * a warm-up state to warm up the cache, it leads to the measured cycles
816f2a56148SDapeng Mi 	 * value may exceed the pre-defined cycles upper boundary and cause
817f2a56148SDapeng Mi 	 * false positive. To avoid this, introduce an warm-up state before
818f2a56148SDapeng Mi 	 * the real verification.
819f2a56148SDapeng Mi 	 */
820f2a56148SDapeng Mi 	for (i = 0; i < 10; i++)
82150f8e27eSDapeng Mi 		loop(0);
822f2a56148SDapeng Mi }
823f2a56148SDapeng Mi 
check_counters(void)82422f2901aSLike Xu static void check_counters(void)
82522f2901aSLike Xu {
82600dca75cSLike Xu 	if (is_fep_available())
82700dca75cSLike Xu 		check_emulated_instr();
82800dca75cSLike Xu 
829f2a56148SDapeng Mi 	warm_up();
83022f2901aSLike Xu 	check_gp_counters();
83122f2901aSLike Xu 	check_fixed_counters();
83222f2901aSLike Xu 	check_rdpmc();
83322f2901aSLike Xu 	check_counters_many();
83422f2901aSLike Xu 	check_counter_overflow();
83522f2901aSLike Xu 	check_gp_counter_cmask();
83622f2901aSLike Xu 	check_running_counter_wrmsr();
837006b089dSLike Xu 	check_tsx_cycles();
83822f2901aSLike Xu }
83922f2901aSLike Xu 
do_unsupported_width_counter_write(void * index)84022f2901aSLike Xu static void do_unsupported_width_counter_write(void *index)
84122f2901aSLike Xu {
84222f2901aSLike Xu 	wrmsr(MSR_IA32_PMC0 + *((int *) index), 0xffffff0123456789ull);
84322f2901aSLike Xu }
84422f2901aSLike Xu 
check_gp_counters_write_width(void)84522f2901aSLike Xu static void check_gp_counters_write_width(void)
84622f2901aSLike Xu {
84722f2901aSLike Xu 	u64 val_64 = 0xffffff0123456789ull;
8484b74c718SThomas Huth 	u64 val_32 = val_64 & ((1ull << 32) - 1);
849414ee7d1SSean Christopherson 	u64 val_max_width = val_64 & ((1ull << pmu.gp_counter_width) - 1);
85022f2901aSLike Xu 	int i;
85122f2901aSLike Xu 
85222f2901aSLike Xu 	/*
85322f2901aSLike Xu 	 * MSR_IA32_PERFCTRn supports 64-bit writes,
85422f2901aSLike Xu 	 * but only the lowest 32 bits are valid.
85522f2901aSLike Xu 	 */
856414ee7d1SSean Christopherson 	for (i = 0; i < pmu.nr_gp_counters; i++) {
85722f2901aSLike Xu 		wrmsr(MSR_IA32_PERFCTR0 + i, val_32);
85822f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PERFCTR0 + i) == val_32);
85922f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PMC0 + i) == val_32);
86022f2901aSLike Xu 
86122f2901aSLike Xu 		wrmsr(MSR_IA32_PERFCTR0 + i, val_max_width);
86222f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PERFCTR0 + i) == val_32);
86322f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PMC0 + i) == val_32);
86422f2901aSLike Xu 
86522f2901aSLike Xu 		wrmsr(MSR_IA32_PERFCTR0 + i, val_64);
86622f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PERFCTR0 + i) == val_32);
86722f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PMC0 + i) == val_32);
86822f2901aSLike Xu 	}
86922f2901aSLike Xu 
87022f2901aSLike Xu 	/*
8714340720eSLike Xu 	 * MSR_IA32_PMCn supports writing values up to GP counter width,
87222f2901aSLike Xu 	 * and only the lowest bits of GP counter width are valid.
87322f2901aSLike Xu 	 */
874414ee7d1SSean Christopherson 	for (i = 0; i < pmu.nr_gp_counters; i++) {
87522f2901aSLike Xu 		wrmsr(MSR_IA32_PMC0 + i, val_32);
87622f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PMC0 + i) == val_32);
87722f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PERFCTR0 + i) == val_32);
87822f2901aSLike Xu 
87922f2901aSLike Xu 		wrmsr(MSR_IA32_PMC0 + i, val_max_width);
88022f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PMC0 + i) == val_max_width);
88122f2901aSLike Xu 		assert(rdmsr(MSR_IA32_PERFCTR0 + i) == val_max_width);
88222f2901aSLike Xu 
88322f2901aSLike Xu 		report(test_for_exception(GP_VECTOR,
88422f2901aSLike Xu 			do_unsupported_width_counter_write, &i),
88522f2901aSLike Xu 		"writing unsupported width to MSR_IA32_PMC%d raises #GP", i);
88622f2901aSLike Xu 	}
88722f2901aSLike Xu }
88822f2901aSLike Xu 
889290f4213SJim Mattson /*
890290f4213SJim Mattson  * Per the SDM, reference cycles are currently implemented using the
891290f4213SJim Mattson  * core crystal clock, TSC, or bus clock. Calibrate to the TSC
892290f4213SJim Mattson  * frequency to set reasonable expectations.
893290f4213SJim Mattson  */
set_ref_cycle_expectations(void)894290f4213SJim Mattson static void set_ref_cycle_expectations(void)
895290f4213SJim Mattson {
896290f4213SJim Mattson 	pmu_counter_t cnt = {
897290f4213SJim Mattson 		.ctr = MSR_IA32_PERFCTR0,
89825cc1ea7SDapeng Mi 		.config = EVNTSEL_OS | EVNTSEL_USR |
89925cc1ea7SDapeng Mi 			  intel_gp_events[INTEL_REF_CYCLES_IDX].unit_sel,
900290f4213SJim Mattson 	};
901290f4213SJim Mattson 	uint64_t tsc_delta;
902290f4213SJim Mattson 	uint64_t t0, t1, t2, t3;
903290f4213SJim Mattson 
9042719b92cSYang Weijiang 	/* Bit 2 enumerates the availability of reference cycles events. */
905414ee7d1SSean Christopherson 	if (!pmu.nr_gp_counters || !pmu_gp_counter_is_available(2))
906290f4213SJim Mattson 		return;
907290f4213SJim Mattson 
90862ba5036SLike Xu 	if (this_cpu_has_perf_global_ctrl())
9098a2866d1SLike Xu 		wrmsr(pmu.msr_global_ctl, 0);
910290f4213SJim Mattson 
911290f4213SJim Mattson 	t0 = fenced_rdtsc();
912290f4213SJim Mattson 	start_event(&cnt);
913290f4213SJim Mattson 	t1 = fenced_rdtsc();
914290f4213SJim Mattson 
915290f4213SJim Mattson 	/*
916290f4213SJim Mattson 	 * This loop has to run long enough to dominate the VM-exit
917290f4213SJim Mattson 	 * costs for playing with the PMU MSRs on start and stop.
918290f4213SJim Mattson 	 *
919290f4213SJim Mattson 	 * On a 2.6GHz Ice Lake, with the TSC frequency at 104 times
920290f4213SJim Mattson 	 * the core crystal clock, this function calculated a guest
921290f4213SJim Mattson 	 * TSC : ref cycles ratio of around 105 with ECX initialized
922290f4213SJim Mattson 	 * to one billion.
923290f4213SJim Mattson 	 */
924290f4213SJim Mattson 	asm volatile("loop ." : "+c"((int){1000000000ull}));
925290f4213SJim Mattson 
926290f4213SJim Mattson 	t2 = fenced_rdtsc();
927290f4213SJim Mattson 	stop_event(&cnt);
928290f4213SJim Mattson 	t3 = fenced_rdtsc();
929290f4213SJim Mattson 
930290f4213SJim Mattson 	tsc_delta = ((t2 - t1) + (t3 - t0)) / 2;
931290f4213SJim Mattson 
932290f4213SJim Mattson 	if (!tsc_delta)
933290f4213SJim Mattson 		return;
934290f4213SJim Mattson 
93525cc1ea7SDapeng Mi 	intel_gp_events[INTEL_REF_CYCLES_IDX].min =
93625cc1ea7SDapeng Mi 		(intel_gp_events[INTEL_REF_CYCLES_IDX].min * cnt.count) / tsc_delta;
93725cc1ea7SDapeng Mi 	intel_gp_events[INTEL_REF_CYCLES_IDX].max =
93825cc1ea7SDapeng Mi 		(intel_gp_events[INTEL_REF_CYCLES_IDX].max * cnt.count) / tsc_delta;
939290f4213SJim Mattson }
940290f4213SJim Mattson 
check_invalid_rdpmc_gp(void)94185c21181SLike Xu static void check_invalid_rdpmc_gp(void)
94285c21181SLike Xu {
94385c21181SLike Xu 	uint64_t val;
94485c21181SLike Xu 
94585c21181SLike Xu 	report(rdpmc_safe(64, &val) == GP_VECTOR,
94685c21181SLike Xu 	       "Expected #GP on RDPMC(64)");
94785c21181SLike Xu }
94885c21181SLike Xu 
main(int ac,char ** av)949a9f8b16fSGleb Natapov int main(int ac, char **av)
950a9f8b16fSGleb Natapov {
95189126fa4SDapeng Mi 	int instruction_idx;
95289126fa4SDapeng Mi 	int branch_idx;
95328437cdbSDapeng Mi 	int branch_miss_idx;
95489126fa4SDapeng Mi 
955a9f8b16fSGleb Natapov 	setup_vm();
9565a2cb3e6SLike Xu 	handle_irq(PMI_VECTOR, cnt_overflow);
957dcda215bSPaolo Bonzini 	buf = malloc(N*64);
958a9f8b16fSGleb Natapov 
95985c21181SLike Xu 	check_invalid_rdpmc_gp();
96085c21181SLike Xu 
961b883751aSLike Xu 	if (pmu.is_intel) {
962414ee7d1SSean Christopherson 		if (!pmu.version) {
96303041e97SLike Xu 			report_skip("No Intel Arch PMU is detected!");
96432b9603cSRadim Krčmář 			return report_summary();
965a9f8b16fSGleb Natapov 		}
9667c648ce2SLike Xu 		gp_events = (struct pmu_event *)intel_gp_events;
9677c648ce2SLike Xu 		gp_events_size = sizeof(intel_gp_events)/sizeof(intel_gp_events[0]);
96889126fa4SDapeng Mi 		instruction_idx = INTEL_INSTRUCTIONS_IDX;
96989126fa4SDapeng Mi 		branch_idx = INTEL_BRANCHES_IDX;
97028437cdbSDapeng Mi 		branch_miss_idx = INTEL_BRANCH_MISS_IDX;
971e0d0022fSDapeng Mi 
972e0d0022fSDapeng Mi 		/*
973e0d0022fSDapeng Mi 		 * For legacy Intel CPUS without clflush/clflushopt support,
974e0d0022fSDapeng Mi 		 * there is no way to force to trigger a LLC miss, thus set
975e0d0022fSDapeng Mi 		 * the minimum value to 0 to avoid false positives.
976e0d0022fSDapeng Mi 		 */
977e0d0022fSDapeng Mi 		if (!this_cpu_has(X86_FEATURE_CLFLUSH))
978e0d0022fSDapeng Mi 			gp_events[INTEL_LLC_MISSES_IDX].min = 0;
979e0d0022fSDapeng Mi 
980b883751aSLike Xu 		report_prefix_push("Intel");
981290f4213SJim Mattson 		set_ref_cycle_expectations();
982b883751aSLike Xu 	} else {
983b883751aSLike Xu 		gp_events_size = sizeof(amd_gp_events)/sizeof(amd_gp_events[0]);
984b883751aSLike Xu 		gp_events = (struct pmu_event *)amd_gp_events;
98589126fa4SDapeng Mi 		instruction_idx = AMD_INSTRUCTIONS_IDX;
98689126fa4SDapeng Mi 		branch_idx = AMD_BRANCHES_IDX;
98728437cdbSDapeng Mi 		branch_miss_idx = AMD_BRANCH_MISS_IDX;
988b883751aSLike Xu 		report_prefix_push("AMD");
989b883751aSLike Xu 	}
99028437cdbSDapeng Mi 	adjust_events_range(gp_events, instruction_idx, branch_idx, branch_miss_idx);
991290f4213SJim Mattson 
992414ee7d1SSean Christopherson 	printf("PMU version:         %d\n", pmu.version);
993414ee7d1SSean Christopherson 	printf("GP counters:         %d\n", pmu.nr_gp_counters);
994414ee7d1SSean Christopherson 	printf("GP counter width:    %d\n", pmu.gp_counter_width);
995414ee7d1SSean Christopherson 	printf("Mask length:         %d\n", pmu.gp_counter_mask_length);
996414ee7d1SSean Christopherson 	printf("Fixed counters:      %d\n", pmu.nr_fixed_counters);
997414ee7d1SSean Christopherson 	printf("Fixed counter width: %d\n", pmu.fixed_counter_width);
9980ef1f6a8SPaolo Bonzini 
9999c07c92bSDapeng Mi 	fixed_counters_num = MIN(pmu.nr_fixed_counters, ARRAY_SIZE(fixed_events));
10009c07c92bSDapeng Mi 	if (pmu.nr_fixed_counters > ARRAY_SIZE(fixed_events))
10019c07c92bSDapeng Mi 		report_info("Fixed counters number %d > defined fixed events %u.  "
10029c07c92bSDapeng Mi 			    "Please update test case.", pmu.nr_fixed_counters,
1003*699264f5SPaolo Bonzini 			    (unsigned)ARRAY_SIZE(fixed_events));
10049c07c92bSDapeng Mi 
10055a2cb3e6SLike Xu 	apic_write(APIC_LVTPC, PMI_VECTOR);
1006a9f8b16fSGleb Natapov 
1007afa714b2SPaolo Bonzini 	check_counters();
100820cf9147SJim Mattson 
1009879e7f07SLike Xu 	if (pmu_has_full_writes()) {
1010cda64e80SLike Xu 		pmu.msr_gp_counter_base = MSR_IA32_PMC0;
1011cda64e80SLike Xu 
101222f2901aSLike Xu 		report_prefix_push("full-width writes");
101322f2901aSLike Xu 		check_counters();
101422f2901aSLike Xu 		check_gp_counters_write_width();
1015d7714e16SLike Xu 		report_prefix_pop();
101622f2901aSLike Xu 	}
1017a9f8b16fSGleb Natapov 
1018b883751aSLike Xu 	if (!pmu.is_intel) {
1019b883751aSLike Xu 		report_prefix_push("K7");
1020b883751aSLike Xu 		pmu.nr_gp_counters = AMD64_NUM_COUNTERS;
1021b883751aSLike Xu 		pmu.msr_gp_counter_base = MSR_K7_PERFCTR0;
1022b883751aSLike Xu 		pmu.msr_gp_event_select_base = MSR_K7_EVNTSEL0;
1023b883751aSLike Xu 		check_counters();
1024b883751aSLike Xu 		report_prefix_pop();
1025b883751aSLike Xu 	}
1026b883751aSLike Xu 
1027f3cdd159SJan Kiszka 	return report_summary();
1028a9f8b16fSGleb Natapov }
1029