1 #include <linux/bitops.h>
2 #include <linux/types.h>
3 #include <linux/slab.h>
4
5 #include <asm/perf_event.h>
6
7 #include "perf_event.h"
8
9 /* The size of a BTS record in bytes: */
10 #define BTS_RECORD_SIZE 24
11
12 #define BTS_BUFFER_SIZE (PAGE_SIZE << 4)
13 #define PEBS_BUFFER_SIZE PAGE_SIZE
14
15 /*
16 * pebs_record_32 for p4 and core not supported
17
18 struct pebs_record_32 {
19 u32 flags, ip;
20 u32 ax, bc, cx, dx;
21 u32 si, di, bp, sp;
22 };
23
24 */
25
26 struct pebs_record_core {
27 u64 flags, ip;
28 u64 ax, bx, cx, dx;
29 u64 si, di, bp, sp;
30 u64 r8, r9, r10, r11;
31 u64 r12, r13, r14, r15;
32 };
33
34 struct pebs_record_nhm {
35 u64 flags, ip;
36 u64 ax, bx, cx, dx;
37 u64 si, di, bp, sp;
38 u64 r8, r9, r10, r11;
39 u64 r12, r13, r14, r15;
40 u64 status, dla, dse, lat;
41 };
42
init_debug_store_on_cpu(int cpu)43 void init_debug_store_on_cpu(int cpu)
44 {
45 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
46
47 if (!ds)
48 return;
49
50 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
51 (u32)((u64)(unsigned long)ds),
52 (u32)((u64)(unsigned long)ds >> 32));
53 }
54
fini_debug_store_on_cpu(int cpu)55 void fini_debug_store_on_cpu(int cpu)
56 {
57 if (!per_cpu(cpu_hw_events, cpu).ds)
58 return;
59
60 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
61 }
62
alloc_pebs_buffer(int cpu)63 static int alloc_pebs_buffer(int cpu)
64 {
65 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
66 int node = cpu_to_node(cpu);
67 int max, thresh = 1; /* always use a single PEBS record */
68 void *buffer;
69
70 if (!x86_pmu.pebs)
71 return 0;
72
73 buffer = kmalloc_node(PEBS_BUFFER_SIZE, GFP_KERNEL | __GFP_ZERO, node);
74 if (unlikely(!buffer))
75 return -ENOMEM;
76
77 max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
78
79 ds->pebs_buffer_base = (u64)(unsigned long)buffer;
80 ds->pebs_index = ds->pebs_buffer_base;
81 ds->pebs_absolute_maximum = ds->pebs_buffer_base +
82 max * x86_pmu.pebs_record_size;
83
84 ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
85 thresh * x86_pmu.pebs_record_size;
86
87 return 0;
88 }
89
release_pebs_buffer(int cpu)90 static void release_pebs_buffer(int cpu)
91 {
92 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
93
94 if (!ds || !x86_pmu.pebs)
95 return;
96
97 kfree((void *)(unsigned long)ds->pebs_buffer_base);
98 ds->pebs_buffer_base = 0;
99 }
100
alloc_bts_buffer(int cpu)101 static int alloc_bts_buffer(int cpu)
102 {
103 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
104 int node = cpu_to_node(cpu);
105 int max, thresh;
106 void *buffer;
107
108 if (!x86_pmu.bts)
109 return 0;
110
111 buffer = kmalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_ZERO, node);
112 if (unlikely(!buffer))
113 return -ENOMEM;
114
115 max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
116 thresh = max / 16;
117
118 ds->bts_buffer_base = (u64)(unsigned long)buffer;
119 ds->bts_index = ds->bts_buffer_base;
120 ds->bts_absolute_maximum = ds->bts_buffer_base +
121 max * BTS_RECORD_SIZE;
122 ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
123 thresh * BTS_RECORD_SIZE;
124
125 return 0;
126 }
127
release_bts_buffer(int cpu)128 static void release_bts_buffer(int cpu)
129 {
130 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
131
132 if (!ds || !x86_pmu.bts)
133 return;
134
135 kfree((void *)(unsigned long)ds->bts_buffer_base);
136 ds->bts_buffer_base = 0;
137 }
138
alloc_ds_buffer(int cpu)139 static int alloc_ds_buffer(int cpu)
140 {
141 int node = cpu_to_node(cpu);
142 struct debug_store *ds;
143
144 ds = kmalloc_node(sizeof(*ds), GFP_KERNEL | __GFP_ZERO, node);
145 if (unlikely(!ds))
146 return -ENOMEM;
147
148 per_cpu(cpu_hw_events, cpu).ds = ds;
149
150 return 0;
151 }
152
release_ds_buffer(int cpu)153 static void release_ds_buffer(int cpu)
154 {
155 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
156
157 if (!ds)
158 return;
159
160 per_cpu(cpu_hw_events, cpu).ds = NULL;
161 kfree(ds);
162 }
163
release_ds_buffers(void)164 void release_ds_buffers(void)
165 {
166 int cpu;
167
168 if (!x86_pmu.bts && !x86_pmu.pebs)
169 return;
170
171 get_online_cpus();
172 for_each_online_cpu(cpu)
173 fini_debug_store_on_cpu(cpu);
174
175 for_each_possible_cpu(cpu) {
176 release_pebs_buffer(cpu);
177 release_bts_buffer(cpu);
178 release_ds_buffer(cpu);
179 }
180 put_online_cpus();
181 }
182
reserve_ds_buffers(void)183 void reserve_ds_buffers(void)
184 {
185 int bts_err = 0, pebs_err = 0;
186 int cpu;
187
188 x86_pmu.bts_active = 0;
189 x86_pmu.pebs_active = 0;
190
191 if (!x86_pmu.bts && !x86_pmu.pebs)
192 return;
193
194 if (!x86_pmu.bts)
195 bts_err = 1;
196
197 if (!x86_pmu.pebs)
198 pebs_err = 1;
199
200 get_online_cpus();
201
202 for_each_possible_cpu(cpu) {
203 if (alloc_ds_buffer(cpu)) {
204 bts_err = 1;
205 pebs_err = 1;
206 }
207
208 if (!bts_err && alloc_bts_buffer(cpu))
209 bts_err = 1;
210
211 if (!pebs_err && alloc_pebs_buffer(cpu))
212 pebs_err = 1;
213
214 if (bts_err && pebs_err)
215 break;
216 }
217
218 if (bts_err) {
219 for_each_possible_cpu(cpu)
220 release_bts_buffer(cpu);
221 }
222
223 if (pebs_err) {
224 for_each_possible_cpu(cpu)
225 release_pebs_buffer(cpu);
226 }
227
228 if (bts_err && pebs_err) {
229 for_each_possible_cpu(cpu)
230 release_ds_buffer(cpu);
231 } else {
232 if (x86_pmu.bts && !bts_err)
233 x86_pmu.bts_active = 1;
234
235 if (x86_pmu.pebs && !pebs_err)
236 x86_pmu.pebs_active = 1;
237
238 for_each_online_cpu(cpu)
239 init_debug_store_on_cpu(cpu);
240 }
241
242 put_online_cpus();
243 }
244
245 /*
246 * BTS
247 */
248
249 struct event_constraint bts_constraint =
250 EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
251
intel_pmu_enable_bts(u64 config)252 void intel_pmu_enable_bts(u64 config)
253 {
254 unsigned long debugctlmsr;
255
256 debugctlmsr = get_debugctlmsr();
257
258 debugctlmsr |= DEBUGCTLMSR_TR;
259 debugctlmsr |= DEBUGCTLMSR_BTS;
260 debugctlmsr |= DEBUGCTLMSR_BTINT;
261
262 if (!(config & ARCH_PERFMON_EVENTSEL_OS))
263 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
264
265 if (!(config & ARCH_PERFMON_EVENTSEL_USR))
266 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
267
268 update_debugctlmsr(debugctlmsr);
269 }
270
intel_pmu_disable_bts(void)271 void intel_pmu_disable_bts(void)
272 {
273 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
274 unsigned long debugctlmsr;
275
276 if (!cpuc->ds)
277 return;
278
279 debugctlmsr = get_debugctlmsr();
280
281 debugctlmsr &=
282 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
283 DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
284
285 update_debugctlmsr(debugctlmsr);
286 }
287
intel_pmu_drain_bts_buffer(void)288 int intel_pmu_drain_bts_buffer(void)
289 {
290 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
291 struct debug_store *ds = cpuc->ds;
292 struct bts_record {
293 u64 from;
294 u64 to;
295 u64 flags;
296 };
297 struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
298 struct bts_record *at, *top;
299 struct perf_output_handle handle;
300 struct perf_event_header header;
301 struct perf_sample_data data;
302 struct pt_regs regs;
303
304 if (!event)
305 return 0;
306
307 if (!x86_pmu.bts_active)
308 return 0;
309
310 at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
311 top = (struct bts_record *)(unsigned long)ds->bts_index;
312
313 if (top <= at)
314 return 0;
315
316 ds->bts_index = ds->bts_buffer_base;
317
318 perf_sample_data_init(&data, 0);
319 data.period = event->hw.last_period;
320 regs.ip = 0;
321
322 /*
323 * Prepare a generic sample, i.e. fill in the invariant fields.
324 * We will overwrite the from and to address before we output
325 * the sample.
326 */
327 perf_prepare_sample(&header, &data, event, ®s);
328
329 if (perf_output_begin(&handle, event, header.size * (top - at)))
330 return 1;
331
332 for (; at < top; at++) {
333 data.ip = at->from;
334 data.addr = at->to;
335
336 perf_output_sample(&handle, &header, &data, event);
337 }
338
339 perf_output_end(&handle);
340
341 /* There's new data available. */
342 event->hw.interrupts++;
343 event->pending_kill = POLL_IN;
344 return 1;
345 }
346
347 /*
348 * PEBS
349 */
350 struct event_constraint intel_core2_pebs_event_constraints[] = {
351 INTEL_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
352 INTEL_UEVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
353 INTEL_UEVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
354 INTEL_UEVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
355 INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED.* */
356 EVENT_CONSTRAINT_END
357 };
358
359 struct event_constraint intel_atom_pebs_event_constraints[] = {
360 INTEL_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
361 INTEL_UEVENT_CONSTRAINT(0x00c5, 0x1), /* MISPREDICTED_BRANCH_RETIRED */
362 INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED.* */
363 EVENT_CONSTRAINT_END
364 };
365
366 struct event_constraint intel_nehalem_pebs_event_constraints[] = {
367 INTEL_EVENT_CONSTRAINT(0x0b, 0xf), /* MEM_INST_RETIRED.* */
368 INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
369 INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
370 INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INST_RETIRED.ANY */
371 INTEL_EVENT_CONSTRAINT(0xc2, 0xf), /* UOPS_RETIRED.* */
372 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
373 INTEL_UEVENT_CONSTRAINT(0x02c5, 0xf), /* BR_MISP_RETIRED.NEAR_CALL */
374 INTEL_EVENT_CONSTRAINT(0xc7, 0xf), /* SSEX_UOPS_RETIRED.* */
375 INTEL_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
376 INTEL_EVENT_CONSTRAINT(0xcb, 0xf), /* MEM_LOAD_RETIRED.* */
377 INTEL_EVENT_CONSTRAINT(0xf7, 0xf), /* FP_ASSIST.* */
378 EVENT_CONSTRAINT_END
379 };
380
381 struct event_constraint intel_westmere_pebs_event_constraints[] = {
382 INTEL_EVENT_CONSTRAINT(0x0b, 0xf), /* MEM_INST_RETIRED.* */
383 INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
384 INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
385 INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INSTR_RETIRED.* */
386 INTEL_EVENT_CONSTRAINT(0xc2, 0xf), /* UOPS_RETIRED.* */
387 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
388 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
389 INTEL_EVENT_CONSTRAINT(0xc7, 0xf), /* SSEX_UOPS_RETIRED.* */
390 INTEL_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
391 INTEL_EVENT_CONSTRAINT(0xcb, 0xf), /* MEM_LOAD_RETIRED.* */
392 INTEL_EVENT_CONSTRAINT(0xf7, 0xf), /* FP_ASSIST.* */
393 EVENT_CONSTRAINT_END
394 };
395
396 struct event_constraint intel_snb_pebs_event_constraints[] = {
397 INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
398 INTEL_UEVENT_CONSTRAINT(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
399 INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
400 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
401 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
402 INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.* */
403 INTEL_UEVENT_CONSTRAINT(0x11d0, 0xf), /* MEM_UOP_RETIRED.STLB_MISS_LOADS */
404 INTEL_UEVENT_CONSTRAINT(0x12d0, 0xf), /* MEM_UOP_RETIRED.STLB_MISS_STORES */
405 INTEL_UEVENT_CONSTRAINT(0x21d0, 0xf), /* MEM_UOP_RETIRED.LOCK_LOADS */
406 INTEL_UEVENT_CONSTRAINT(0x22d0, 0xf), /* MEM_UOP_RETIRED.LOCK_STORES */
407 INTEL_UEVENT_CONSTRAINT(0x41d0, 0xf), /* MEM_UOP_RETIRED.SPLIT_LOADS */
408 INTEL_UEVENT_CONSTRAINT(0x42d0, 0xf), /* MEM_UOP_RETIRED.SPLIT_STORES */
409 INTEL_UEVENT_CONSTRAINT(0x81d0, 0xf), /* MEM_UOP_RETIRED.ANY_LOADS */
410 INTEL_UEVENT_CONSTRAINT(0x82d0, 0xf), /* MEM_UOP_RETIRED.ANY_STORES */
411 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
412 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
413 INTEL_UEVENT_CONSTRAINT(0x02d4, 0xf), /* MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS */
414 EVENT_CONSTRAINT_END
415 };
416
intel_pebs_constraints(struct perf_event * event)417 struct event_constraint *intel_pebs_constraints(struct perf_event *event)
418 {
419 struct event_constraint *c;
420
421 if (!event->attr.precise_ip)
422 return NULL;
423
424 if (x86_pmu.pebs_constraints) {
425 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
426 if ((event->hw.config & c->cmask) == c->code)
427 return c;
428 }
429 }
430
431 return &emptyconstraint;
432 }
433
intel_pmu_pebs_enable(struct perf_event * event)434 void intel_pmu_pebs_enable(struct perf_event *event)
435 {
436 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
437 struct hw_perf_event *hwc = &event->hw;
438
439 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
440
441 cpuc->pebs_enabled |= 1ULL << hwc->idx;
442
443 if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
444 intel_pmu_lbr_enable(event);
445 }
446
intel_pmu_pebs_disable(struct perf_event * event)447 void intel_pmu_pebs_disable(struct perf_event *event)
448 {
449 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
450 struct hw_perf_event *hwc = &event->hw;
451
452 cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
453 if (cpuc->enabled)
454 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
455
456 hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
457
458 if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
459 intel_pmu_lbr_disable(event);
460 }
461
intel_pmu_pebs_enable_all(void)462 void intel_pmu_pebs_enable_all(void)
463 {
464 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
465
466 if (cpuc->pebs_enabled)
467 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
468 }
469
intel_pmu_pebs_disable_all(void)470 void intel_pmu_pebs_disable_all(void)
471 {
472 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
473
474 if (cpuc->pebs_enabled)
475 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
476 }
477
478 #include <asm/insn.h>
479
kernel_ip(unsigned long ip)480 static inline bool kernel_ip(unsigned long ip)
481 {
482 #ifdef CONFIG_X86_32
483 return ip > PAGE_OFFSET;
484 #else
485 return (long)ip < 0;
486 #endif
487 }
488
intel_pmu_pebs_fixup_ip(struct pt_regs * regs)489 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
490 {
491 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
492 unsigned long from = cpuc->lbr_entries[0].from;
493 unsigned long old_to, to = cpuc->lbr_entries[0].to;
494 unsigned long ip = regs->ip;
495 int is_64bit = 0;
496
497 /*
498 * We don't need to fixup if the PEBS assist is fault like
499 */
500 if (!x86_pmu.intel_cap.pebs_trap)
501 return 1;
502
503 /*
504 * No LBR entry, no basic block, no rewinding
505 */
506 if (!cpuc->lbr_stack.nr || !from || !to)
507 return 0;
508
509 /*
510 * Basic blocks should never cross user/kernel boundaries
511 */
512 if (kernel_ip(ip) != kernel_ip(to))
513 return 0;
514
515 /*
516 * unsigned math, either ip is before the start (impossible) or
517 * the basic block is larger than 1 page (sanity)
518 */
519 if ((ip - to) > PAGE_SIZE)
520 return 0;
521
522 /*
523 * We sampled a branch insn, rewind using the LBR stack
524 */
525 if (ip == to) {
526 regs->ip = from;
527 return 1;
528 }
529
530 do {
531 struct insn insn;
532 u8 buf[MAX_INSN_SIZE];
533 void *kaddr;
534
535 old_to = to;
536 if (!kernel_ip(ip)) {
537 int bytes, size = MAX_INSN_SIZE;
538
539 bytes = copy_from_user_nmi(buf, (void __user *)to, size);
540 if (bytes != size)
541 return 0;
542
543 kaddr = buf;
544 } else
545 kaddr = (void *)to;
546
547 #ifdef CONFIG_X86_64
548 is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
549 #endif
550 insn_init(&insn, kaddr, is_64bit);
551 insn_get_length(&insn);
552 to += insn.length;
553 } while (to < ip);
554
555 if (to == ip) {
556 regs->ip = old_to;
557 return 1;
558 }
559
560 /*
561 * Even though we decoded the basic block, the instruction stream
562 * never matched the given IP, either the TO or the IP got corrupted.
563 */
564 return 0;
565 }
566
__intel_pmu_pebs_event(struct perf_event * event,struct pt_regs * iregs,void * __pebs)567 static void __intel_pmu_pebs_event(struct perf_event *event,
568 struct pt_regs *iregs, void *__pebs)
569 {
570 /*
571 * We cast to pebs_record_core since that is a subset of
572 * both formats and we don't use the other fields in this
573 * routine.
574 */
575 struct pebs_record_core *pebs = __pebs;
576 struct perf_sample_data data;
577 struct pt_regs regs;
578
579 if (!intel_pmu_save_and_restart(event))
580 return;
581
582 perf_sample_data_init(&data, 0);
583 data.period = event->hw.last_period;
584
585 /*
586 * We use the interrupt regs as a base because the PEBS record
587 * does not contain a full regs set, specifically it seems to
588 * lack segment descriptors, which get used by things like
589 * user_mode().
590 *
591 * In the simple case fix up only the IP and BP,SP regs, for
592 * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
593 * A possible PERF_SAMPLE_REGS will have to transfer all regs.
594 */
595 regs = *iregs;
596 regs.ip = pebs->ip;
597 regs.bp = pebs->bp;
598 regs.sp = pebs->sp;
599
600 if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(®s))
601 regs.flags |= PERF_EFLAGS_EXACT;
602 else
603 regs.flags &= ~PERF_EFLAGS_EXACT;
604
605 if (perf_event_overflow(event, &data, ®s))
606 x86_pmu_stop(event, 0);
607 }
608
intel_pmu_drain_pebs_core(struct pt_regs * iregs)609 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
610 {
611 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
612 struct debug_store *ds = cpuc->ds;
613 struct perf_event *event = cpuc->events[0]; /* PMC0 only */
614 struct pebs_record_core *at, *top;
615 int n;
616
617 if (!x86_pmu.pebs_active)
618 return;
619
620 at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
621 top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
622
623 /*
624 * Whatever else happens, drain the thing
625 */
626 ds->pebs_index = ds->pebs_buffer_base;
627
628 if (!test_bit(0, cpuc->active_mask))
629 return;
630
631 WARN_ON_ONCE(!event);
632
633 if (!event->attr.precise_ip)
634 return;
635
636 n = top - at;
637 if (n <= 0)
638 return;
639
640 /*
641 * Should not happen, we program the threshold at 1 and do not
642 * set a reset value.
643 */
644 WARN_ON_ONCE(n > 1);
645 at += n - 1;
646
647 __intel_pmu_pebs_event(event, iregs, at);
648 }
649
intel_pmu_drain_pebs_nhm(struct pt_regs * iregs)650 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
651 {
652 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
653 struct debug_store *ds = cpuc->ds;
654 struct pebs_record_nhm *at, *top;
655 struct perf_event *event = NULL;
656 u64 status = 0;
657 int bit, n;
658
659 if (!x86_pmu.pebs_active)
660 return;
661
662 at = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
663 top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
664
665 ds->pebs_index = ds->pebs_buffer_base;
666
667 n = top - at;
668 if (n <= 0)
669 return;
670
671 /*
672 * Should not happen, we program the threshold at 1 and do not
673 * set a reset value.
674 */
675 WARN_ON_ONCE(n > MAX_PEBS_EVENTS);
676
677 for ( ; at < top; at++) {
678 for_each_set_bit(bit, (unsigned long *)&at->status, MAX_PEBS_EVENTS) {
679 event = cpuc->events[bit];
680 if (!test_bit(bit, cpuc->active_mask))
681 continue;
682
683 WARN_ON_ONCE(!event);
684
685 if (!event->attr.precise_ip)
686 continue;
687
688 if (__test_and_set_bit(bit, (unsigned long *)&status))
689 continue;
690
691 break;
692 }
693
694 if (!event || bit >= MAX_PEBS_EVENTS)
695 continue;
696
697 __intel_pmu_pebs_event(event, iregs, at);
698 }
699 }
700
701 /*
702 * BTS, PEBS probe and setup
703 */
704
intel_ds_init(void)705 void intel_ds_init(void)
706 {
707 /*
708 * No support for 32bit formats
709 */
710 if (!boot_cpu_has(X86_FEATURE_DTES64))
711 return;
712
713 x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
714 x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
715 if (x86_pmu.pebs) {
716 char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-';
717 int format = x86_pmu.intel_cap.pebs_format;
718
719 switch (format) {
720 case 0:
721 printk(KERN_CONT "PEBS fmt0%c, ", pebs_type);
722 x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
723 x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
724 break;
725
726 case 1:
727 printk(KERN_CONT "PEBS fmt1%c, ", pebs_type);
728 x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
729 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
730 break;
731
732 default:
733 printk(KERN_CONT "no PEBS fmt%d%c, ", format, pebs_type);
734 x86_pmu.pebs = 0;
735 }
736 }
737 }
738