1 #include <linux/perf_event.h>
2 #include <linux/types.h>
3
4 #include <asm/perf_event.h>
5 #include <asm/msr.h>
6
7 #include "perf_event.h"
8
9 enum {
10 LBR_FORMAT_32 = 0x00,
11 LBR_FORMAT_LIP = 0x01,
12 LBR_FORMAT_EIP = 0x02,
13 LBR_FORMAT_EIP_FLAGS = 0x03,
14 };
15
16 /*
17 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
18 * otherwise it becomes near impossible to get a reliable stack.
19 */
20
__intel_pmu_lbr_enable(void)21 static void __intel_pmu_lbr_enable(void)
22 {
23 u64 debugctl;
24
25 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
26 debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
27 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
28 }
29
__intel_pmu_lbr_disable(void)30 static void __intel_pmu_lbr_disable(void)
31 {
32 u64 debugctl;
33
34 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
35 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
36 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
37 }
38
intel_pmu_lbr_reset_32(void)39 static void intel_pmu_lbr_reset_32(void)
40 {
41 int i;
42
43 for (i = 0; i < x86_pmu.lbr_nr; i++)
44 wrmsrl(x86_pmu.lbr_from + i, 0);
45 }
46
intel_pmu_lbr_reset_64(void)47 static void intel_pmu_lbr_reset_64(void)
48 {
49 int i;
50
51 for (i = 0; i < x86_pmu.lbr_nr; i++) {
52 wrmsrl(x86_pmu.lbr_from + i, 0);
53 wrmsrl(x86_pmu.lbr_to + i, 0);
54 }
55 }
56
intel_pmu_lbr_reset(void)57 void intel_pmu_lbr_reset(void)
58 {
59 if (!x86_pmu.lbr_nr)
60 return;
61
62 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
63 intel_pmu_lbr_reset_32();
64 else
65 intel_pmu_lbr_reset_64();
66 }
67
intel_pmu_lbr_enable(struct perf_event * event)68 void intel_pmu_lbr_enable(struct perf_event *event)
69 {
70 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
71
72 if (!x86_pmu.lbr_nr)
73 return;
74
75 /*
76 * Reset the LBR stack if we changed task context to
77 * avoid data leaks.
78 */
79
80 if (event->ctx->task && cpuc->lbr_context != event->ctx) {
81 intel_pmu_lbr_reset();
82 cpuc->lbr_context = event->ctx;
83 }
84
85 cpuc->lbr_users++;
86 }
87
intel_pmu_lbr_disable(struct perf_event * event)88 void intel_pmu_lbr_disable(struct perf_event *event)
89 {
90 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
91
92 if (!x86_pmu.lbr_nr)
93 return;
94
95 cpuc->lbr_users--;
96 WARN_ON_ONCE(cpuc->lbr_users < 0);
97
98 if (cpuc->enabled && !cpuc->lbr_users)
99 __intel_pmu_lbr_disable();
100 }
101
intel_pmu_lbr_enable_all(void)102 void intel_pmu_lbr_enable_all(void)
103 {
104 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
105
106 if (cpuc->lbr_users)
107 __intel_pmu_lbr_enable();
108 }
109
intel_pmu_lbr_disable_all(void)110 void intel_pmu_lbr_disable_all(void)
111 {
112 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
113
114 if (cpuc->lbr_users)
115 __intel_pmu_lbr_disable();
116 }
117
intel_pmu_lbr_tos(void)118 static inline u64 intel_pmu_lbr_tos(void)
119 {
120 u64 tos;
121
122 rdmsrl(x86_pmu.lbr_tos, tos);
123
124 return tos;
125 }
126
intel_pmu_lbr_read_32(struct cpu_hw_events * cpuc)127 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
128 {
129 unsigned long mask = x86_pmu.lbr_nr - 1;
130 u64 tos = intel_pmu_lbr_tos();
131 int i;
132
133 for (i = 0; i < x86_pmu.lbr_nr; i++) {
134 unsigned long lbr_idx = (tos - i) & mask;
135 union {
136 struct {
137 u32 from;
138 u32 to;
139 };
140 u64 lbr;
141 } msr_lastbranch;
142
143 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
144
145 cpuc->lbr_entries[i].from = msr_lastbranch.from;
146 cpuc->lbr_entries[i].to = msr_lastbranch.to;
147 cpuc->lbr_entries[i].flags = 0;
148 }
149 cpuc->lbr_stack.nr = i;
150 }
151
152 #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
153
154 /*
155 * Due to lack of segmentation in Linux the effective address (offset)
156 * is the same as the linear address, allowing us to merge the LIP and EIP
157 * LBR formats.
158 */
intel_pmu_lbr_read_64(struct cpu_hw_events * cpuc)159 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
160 {
161 unsigned long mask = x86_pmu.lbr_nr - 1;
162 int lbr_format = x86_pmu.intel_cap.lbr_format;
163 u64 tos = intel_pmu_lbr_tos();
164 int i;
165
166 for (i = 0; i < x86_pmu.lbr_nr; i++) {
167 unsigned long lbr_idx = (tos - i) & mask;
168 u64 from, to, flags = 0;
169
170 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
171 rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
172
173 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
174 flags = !!(from & LBR_FROM_FLAG_MISPRED);
175 from = (u64)((((s64)from) << 1) >> 1);
176 }
177
178 cpuc->lbr_entries[i].from = from;
179 cpuc->lbr_entries[i].to = to;
180 cpuc->lbr_entries[i].flags = flags;
181 }
182 cpuc->lbr_stack.nr = i;
183 }
184
intel_pmu_lbr_read(void)185 void intel_pmu_lbr_read(void)
186 {
187 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
188
189 if (!cpuc->lbr_users)
190 return;
191
192 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
193 intel_pmu_lbr_read_32(cpuc);
194 else
195 intel_pmu_lbr_read_64(cpuc);
196 }
197
intel_pmu_lbr_init_core(void)198 void intel_pmu_lbr_init_core(void)
199 {
200 x86_pmu.lbr_nr = 4;
201 x86_pmu.lbr_tos = 0x01c9;
202 x86_pmu.lbr_from = 0x40;
203 x86_pmu.lbr_to = 0x60;
204 }
205
intel_pmu_lbr_init_nhm(void)206 void intel_pmu_lbr_init_nhm(void)
207 {
208 x86_pmu.lbr_nr = 16;
209 x86_pmu.lbr_tos = 0x01c9;
210 x86_pmu.lbr_from = 0x680;
211 x86_pmu.lbr_to = 0x6c0;
212 }
213
intel_pmu_lbr_init_atom(void)214 void intel_pmu_lbr_init_atom(void)
215 {
216 x86_pmu.lbr_nr = 8;
217 x86_pmu.lbr_tos = 0x01c9;
218 x86_pmu.lbr_from = 0x40;
219 x86_pmu.lbr_to = 0x60;
220 }
221