1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * BTS PMU driver for perf
4 * Copyright (c) 2013-2014, Intel Corporation.
5 */
6
7 #undef DEBUG
8
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11 #include <linux/bitops.h>
12 #include <linux/types.h>
13 #include <linux/slab.h>
14 #include <linux/debugfs.h>
15 #include <linux/device.h>
16 #include <linux/coredump.h>
17
18 #include <linux/sizes.h>
19 #include <asm/perf_event.h>
20
21 #include "../perf_event.h"
22
23 struct bts_ctx {
24 struct perf_output_handle handle;
25 struct debug_store ds_back;
26 int state;
27 };
28
29 /* BTS context states: */
30 enum {
31 /* no ongoing AUX transactions */
32 BTS_STATE_STOPPED = 0,
33 /* AUX transaction is on, BTS tracing is disabled */
34 BTS_STATE_INACTIVE,
35 /* AUX transaction is on, BTS tracing is running */
36 BTS_STATE_ACTIVE,
37 };
38
39 static struct bts_ctx __percpu *bts_ctx;
40
41 #define BTS_RECORD_SIZE 24
42 #define BTS_SAFETY_MARGIN 4080
43
44 struct bts_phys {
45 struct page *page;
46 unsigned long size;
47 unsigned long offset;
48 unsigned long displacement;
49 };
50
51 struct bts_buffer {
52 size_t real_size; /* multiple of BTS_RECORD_SIZE */
53 unsigned int nr_pages;
54 unsigned int nr_bufs;
55 unsigned int cur_buf;
56 bool snapshot;
57 local_t data_size;
58 local_t head;
59 unsigned long end;
60 void **data_pages;
61 struct bts_phys buf[] __counted_by(nr_bufs);
62 };
63
64 static struct pmu bts_pmu;
65
buf_nr_pages(struct page * page)66 static int buf_nr_pages(struct page *page)
67 {
68 if (!PagePrivate(page))
69 return 1;
70
71 return 1 << page_private(page);
72 }
73
buf_size(struct page * page)74 static size_t buf_size(struct page *page)
75 {
76 return buf_nr_pages(page) * PAGE_SIZE;
77 }
78
79 static void *
bts_buffer_setup_aux(struct perf_event * event,void ** pages,int nr_pages,bool overwrite)80 bts_buffer_setup_aux(struct perf_event *event, void **pages,
81 int nr_pages, bool overwrite)
82 {
83 struct bts_buffer *buf;
84 struct page *page;
85 int cpu = event->cpu;
86 int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
87 unsigned long offset;
88 size_t size = nr_pages << PAGE_SHIFT;
89 int pg, nbuf, pad;
90
91 /* count all the high order buffers */
92 for (pg = 0, nbuf = 0; pg < nr_pages;) {
93 page = virt_to_page(pages[pg]);
94 pg += buf_nr_pages(page);
95 nbuf++;
96 }
97
98 /*
99 * to avoid interrupts in overwrite mode, only allow one physical
100 */
101 if (overwrite && nbuf > 1)
102 return NULL;
103
104 buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
105 if (!buf)
106 return NULL;
107
108 buf->nr_pages = nr_pages;
109 buf->nr_bufs = nbuf;
110 buf->snapshot = overwrite;
111 buf->data_pages = pages;
112 buf->real_size = size - size % BTS_RECORD_SIZE;
113
114 for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
115 unsigned int __nr_pages;
116
117 page = virt_to_page(pages[pg]);
118 __nr_pages = buf_nr_pages(page);
119 buf->buf[nbuf].page = page;
120 buf->buf[nbuf].offset = offset;
121 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
122 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
123 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
124 buf->buf[nbuf].size -= pad;
125
126 pg += __nr_pages;
127 offset += __nr_pages << PAGE_SHIFT;
128 }
129
130 return buf;
131 }
132
bts_buffer_free_aux(void * data)133 static void bts_buffer_free_aux(void *data)
134 {
135 kfree(data);
136 }
137
bts_buffer_offset(struct bts_buffer * buf,unsigned int idx)138 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
139 {
140 return buf->buf[idx].offset + buf->buf[idx].displacement;
141 }
142
143 static void
bts_config_buffer(struct bts_buffer * buf)144 bts_config_buffer(struct bts_buffer *buf)
145 {
146 int cpu = raw_smp_processor_id();
147 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
148 struct bts_phys *phys = &buf->buf[buf->cur_buf];
149 unsigned long index, thresh = 0, end = phys->size;
150 struct page *page = phys->page;
151
152 index = local_read(&buf->head);
153
154 if (!buf->snapshot) {
155 if (buf->end < phys->offset + buf_size(page))
156 end = buf->end - phys->offset - phys->displacement;
157
158 index -= phys->offset + phys->displacement;
159
160 if (end - index > BTS_SAFETY_MARGIN)
161 thresh = end - BTS_SAFETY_MARGIN;
162 else if (end - index > BTS_RECORD_SIZE)
163 thresh = end - BTS_RECORD_SIZE;
164 else
165 thresh = end;
166 }
167
168 ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
169 ds->bts_index = ds->bts_buffer_base + index;
170 ds->bts_absolute_maximum = ds->bts_buffer_base + end;
171 ds->bts_interrupt_threshold = !buf->snapshot
172 ? ds->bts_buffer_base + thresh
173 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
174 }
175
bts_buffer_pad_out(struct bts_phys * phys,unsigned long head)176 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
177 {
178 unsigned long index = head - phys->offset;
179
180 memset(page_address(phys->page) + index, 0, phys->size - index);
181 }
182
bts_update(struct bts_ctx * bts)183 static void bts_update(struct bts_ctx *bts)
184 {
185 int cpu = raw_smp_processor_id();
186 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
187 struct bts_buffer *buf = perf_get_aux(&bts->handle);
188 unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
189
190 if (!buf)
191 return;
192
193 head = index + bts_buffer_offset(buf, buf->cur_buf);
194 old = local_xchg(&buf->head, head);
195
196 if (!buf->snapshot) {
197 if (old == head)
198 return;
199
200 if (ds->bts_index >= ds->bts_absolute_maximum)
201 perf_aux_output_flag(&bts->handle,
202 PERF_AUX_FLAG_TRUNCATED);
203
204 /*
205 * old and head are always in the same physical buffer, so we
206 * can subtract them to get the data size.
207 */
208 local_add(head - old, &buf->data_size);
209 } else {
210 local_set(&buf->data_size, head);
211 }
212
213 /*
214 * Since BTS is coherent, just add compiler barrier to ensure
215 * BTS updating is ordered against bts::handle::event.
216 */
217 barrier();
218 }
219
220 static int
221 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
222
223 /*
224 * Ordering PMU callbacks wrt themselves and the PMI is done by means
225 * of bts::state, which:
226 * - is set when bts::handle::event is valid, that is, between
227 * perf_aux_output_begin() and perf_aux_output_end();
228 * - is zero otherwise;
229 * - is ordered against bts::handle::event with a compiler barrier.
230 */
231
__bts_event_start(struct perf_event * event)232 static void __bts_event_start(struct perf_event *event)
233 {
234 struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
235 struct bts_buffer *buf = perf_get_aux(&bts->handle);
236 u64 config = 0;
237
238 if (!buf->snapshot)
239 config |= ARCH_PERFMON_EVENTSEL_INT;
240 if (!event->attr.exclude_kernel)
241 config |= ARCH_PERFMON_EVENTSEL_OS;
242 if (!event->attr.exclude_user)
243 config |= ARCH_PERFMON_EVENTSEL_USR;
244
245 bts_config_buffer(buf);
246
247 /*
248 * local barrier to make sure that ds configuration made it
249 * before we enable BTS and bts::state goes ACTIVE
250 */
251 wmb();
252
253 /* INACTIVE/STOPPED -> ACTIVE */
254 WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
255
256 intel_pmu_enable_bts(config);
257
258 }
259
bts_event_start(struct perf_event * event,int flags)260 static void bts_event_start(struct perf_event *event, int flags)
261 {
262 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
263 struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
264 struct bts_buffer *buf;
265
266 buf = perf_aux_output_begin(&bts->handle, event);
267 if (!buf)
268 goto fail_stop;
269
270 if (bts_buffer_reset(buf, &bts->handle))
271 goto fail_end_stop;
272
273 bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
274 bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
275 bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
276
277 perf_event_itrace_started(event);
278 event->hw.state = 0;
279
280 __bts_event_start(event);
281
282 return;
283
284 fail_end_stop:
285 perf_aux_output_end(&bts->handle, 0);
286
287 fail_stop:
288 event->hw.state = PERF_HES_STOPPED;
289 }
290
__bts_event_stop(struct perf_event * event,int state)291 static void __bts_event_stop(struct perf_event *event, int state)
292 {
293 struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
294
295 /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
296 WRITE_ONCE(bts->state, state);
297
298 /*
299 * No extra synchronization is mandated by the documentation to have
300 * BTS data stores globally visible.
301 */
302 intel_pmu_disable_bts();
303 }
304
bts_event_stop(struct perf_event * event,int flags)305 static void bts_event_stop(struct perf_event *event, int flags)
306 {
307 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
308 struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
309 struct bts_buffer *buf = NULL;
310 int state = READ_ONCE(bts->state);
311
312 if (state == BTS_STATE_ACTIVE)
313 __bts_event_stop(event, BTS_STATE_STOPPED);
314
315 if (state != BTS_STATE_STOPPED)
316 buf = perf_get_aux(&bts->handle);
317
318 event->hw.state |= PERF_HES_STOPPED;
319
320 if (flags & PERF_EF_UPDATE) {
321 bts_update(bts);
322
323 if (buf) {
324 if (buf->snapshot)
325 bts->handle.head =
326 local_xchg(&buf->data_size,
327 buf->nr_pages << PAGE_SHIFT);
328 perf_aux_output_end(&bts->handle,
329 local_xchg(&buf->data_size, 0));
330 }
331
332 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
333 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
334 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
335 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
336 }
337 }
338
intel_bts_enable_local(void)339 void intel_bts_enable_local(void)
340 {
341 struct bts_ctx *bts;
342 int state;
343
344 if (!bts_ctx)
345 return;
346
347 bts = this_cpu_ptr(bts_ctx);
348 state = READ_ONCE(bts->state);
349 /*
350 * Here we transition from INACTIVE to ACTIVE;
351 * if we instead are STOPPED from the interrupt handler,
352 * stay that way. Can't be ACTIVE here though.
353 */
354 if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
355 return;
356
357 if (state == BTS_STATE_STOPPED)
358 return;
359
360 if (bts->handle.event)
361 __bts_event_start(bts->handle.event);
362 }
363
intel_bts_disable_local(void)364 void intel_bts_disable_local(void)
365 {
366 struct bts_ctx *bts;
367
368 if (!bts_ctx)
369 return;
370
371 bts = this_cpu_ptr(bts_ctx);
372
373 /*
374 * Here we transition from ACTIVE to INACTIVE;
375 * do nothing for STOPPED or INACTIVE.
376 */
377 if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
378 return;
379
380 if (bts->handle.event)
381 __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
382 }
383
384 static int
bts_buffer_reset(struct bts_buffer * buf,struct perf_output_handle * handle)385 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
386 {
387 unsigned long head, space, next_space, pad, gap, skip, wakeup;
388 unsigned int next_buf;
389 struct bts_phys *phys, *next_phys;
390 int ret;
391
392 if (buf->snapshot)
393 return 0;
394
395 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
396
397 phys = &buf->buf[buf->cur_buf];
398 space = phys->offset + phys->displacement + phys->size - head;
399 pad = space;
400 if (space > handle->size) {
401 space = handle->size;
402 space -= space % BTS_RECORD_SIZE;
403 }
404 if (space <= BTS_SAFETY_MARGIN) {
405 /* See if next phys buffer has more space */
406 next_buf = buf->cur_buf + 1;
407 if (next_buf >= buf->nr_bufs)
408 next_buf = 0;
409 next_phys = &buf->buf[next_buf];
410 gap = buf_size(phys->page) - phys->displacement - phys->size +
411 next_phys->displacement;
412 skip = pad + gap;
413 if (handle->size >= skip) {
414 next_space = next_phys->size;
415 if (next_space + skip > handle->size) {
416 next_space = handle->size - skip;
417 next_space -= next_space % BTS_RECORD_SIZE;
418 }
419 if (next_space > space || !space) {
420 if (pad)
421 bts_buffer_pad_out(phys, head);
422 ret = perf_aux_output_skip(handle, skip);
423 if (ret)
424 return ret;
425 /* Advance to next phys buffer */
426 phys = next_phys;
427 space = next_space;
428 head = phys->offset + phys->displacement;
429 /*
430 * After this, cur_buf and head won't match ds
431 * anymore, so we must not be racing with
432 * bts_update().
433 */
434 buf->cur_buf = next_buf;
435 local_set(&buf->head, head);
436 }
437 }
438 }
439
440 /* Don't go far beyond wakeup watermark */
441 wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
442 handle->head;
443 if (space > wakeup) {
444 space = wakeup;
445 space -= space % BTS_RECORD_SIZE;
446 }
447
448 buf->end = head + space;
449
450 /*
451 * If we have no space, the lost notification would have been sent when
452 * we hit absolute_maximum - see bts_update()
453 */
454 if (!space)
455 return -ENOSPC;
456
457 return 0;
458 }
459
intel_bts_interrupt(void)460 int intel_bts_interrupt(void)
461 {
462 struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
463 struct bts_ctx *bts;
464 struct perf_event *event;
465 struct bts_buffer *buf;
466 s64 old_head;
467 int err = -ENOSPC, handled = 0;
468
469 if (!bts_ctx)
470 return 0;
471
472 bts = this_cpu_ptr(bts_ctx);
473 event = bts->handle.event;
474 /*
475 * The only surefire way of knowing if this NMI is ours is by checking
476 * the write ptr against the PMI threshold.
477 */
478 if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
479 handled = 1;
480
481 /*
482 * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
483 * so we can only be INACTIVE or STOPPED
484 */
485 if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
486 return handled;
487
488 buf = perf_get_aux(&bts->handle);
489 if (!buf)
490 return handled;
491
492 /*
493 * Skip snapshot counters: they don't use the interrupt, but
494 * there's no other way of telling, because the pointer will
495 * keep moving
496 */
497 if (buf->snapshot)
498 return 0;
499
500 old_head = local_read(&buf->head);
501 bts_update(bts);
502
503 /* no new data */
504 if (old_head == local_read(&buf->head))
505 return handled;
506
507 perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
508
509 buf = perf_aux_output_begin(&bts->handle, event);
510 if (buf)
511 err = bts_buffer_reset(buf, &bts->handle);
512
513 if (err) {
514 WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
515
516 if (buf) {
517 /*
518 * BTS_STATE_STOPPED should be visible before
519 * cleared handle::event
520 */
521 barrier();
522 perf_aux_output_end(&bts->handle, 0);
523 }
524 }
525
526 return 1;
527 }
528
bts_event_del(struct perf_event * event,int mode)529 static void bts_event_del(struct perf_event *event, int mode)
530 {
531 bts_event_stop(event, PERF_EF_UPDATE);
532 }
533
bts_event_add(struct perf_event * event,int mode)534 static int bts_event_add(struct perf_event *event, int mode)
535 {
536 struct bts_ctx *bts = this_cpu_ptr(bts_ctx);
537 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
538 struct hw_perf_event *hwc = &event->hw;
539
540 event->hw.state = PERF_HES_STOPPED;
541
542 if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
543 return -EBUSY;
544
545 if (bts->handle.event)
546 return -EBUSY;
547
548 if (mode & PERF_EF_START) {
549 bts_event_start(event, 0);
550 if (hwc->state & PERF_HES_STOPPED)
551 return -EINVAL;
552 }
553
554 return 0;
555 }
556
bts_event_destroy(struct perf_event * event)557 static void bts_event_destroy(struct perf_event *event)
558 {
559 x86_release_hardware();
560 x86_del_exclusive(x86_lbr_exclusive_bts);
561 }
562
bts_event_init(struct perf_event * event)563 static int bts_event_init(struct perf_event *event)
564 {
565 int ret;
566
567 if (event->attr.type != bts_pmu.type)
568 return -ENOENT;
569
570 /*
571 * BTS leaks kernel addresses even when CPL0 tracing is
572 * disabled, so disallow intel_bts driver for unprivileged
573 * users on paranoid systems since it provides trace data
574 * to the user in a zero-copy fashion.
575 */
576 if (event->attr.exclude_kernel) {
577 ret = perf_allow_kernel();
578 if (ret)
579 return ret;
580 }
581
582 if (x86_add_exclusive(x86_lbr_exclusive_bts))
583 return -EBUSY;
584
585 ret = x86_reserve_hardware();
586 if (ret) {
587 x86_del_exclusive(x86_lbr_exclusive_bts);
588 return ret;
589 }
590
591 event->destroy = bts_event_destroy;
592
593 return 0;
594 }
595
bts_event_read(struct perf_event * event)596 static void bts_event_read(struct perf_event *event)
597 {
598 }
599
bts_init(void)600 static __init int bts_init(void)
601 {
602 if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
603 return -ENODEV;
604
605 if (boot_cpu_has(X86_FEATURE_PTI)) {
606 /*
607 * BTS hardware writes through a virtual memory map we must
608 * either use the kernel physical map, or the user mapping of
609 * the AUX buffer.
610 *
611 * However, since this driver supports per-CPU and per-task inherit
612 * we cannot use the user mapping since it will not be available
613 * if we're not running the owning process.
614 *
615 * With PTI we can't use the kernel map either, because its not
616 * there when we run userspace.
617 *
618 * For now, disable this driver when using PTI.
619 */
620 return -ENODEV;
621 }
622
623 bts_ctx = alloc_percpu(struct bts_ctx);
624 if (!bts_ctx)
625 return -ENOMEM;
626
627 bts_pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
628 PERF_PMU_CAP_EXCLUSIVE;
629 bts_pmu.task_ctx_nr = perf_sw_context;
630 bts_pmu.event_init = bts_event_init;
631 bts_pmu.add = bts_event_add;
632 bts_pmu.del = bts_event_del;
633 bts_pmu.start = bts_event_start;
634 bts_pmu.stop = bts_event_stop;
635 bts_pmu.read = bts_event_read;
636 bts_pmu.setup_aux = bts_buffer_setup_aux;
637 bts_pmu.free_aux = bts_buffer_free_aux;
638
639 return perf_pmu_register(&bts_pmu, "intel_bts", -1);
640 }
641 arch_initcall(bts_init);
642