1 /* 2 * Performance events - AMD IBS 3 * 4 * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter 5 * 6 * For licencing details see kernel-base/COPYING 7 */ 8 9 #include <linux/perf_event.h> 10 #include <linux/init.h> 11 #include <linux/export.h> 12 #include <linux/pci.h> 13 #include <linux/ptrace.h> 14 #include <linux/syscore_ops.h> 15 #include <linux/sched/clock.h> 16 17 #include <asm/apic.h> 18 19 #include "../perf_event.h" 20 21 static u32 ibs_caps; 22 23 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) 24 25 #include <linux/kprobes.h> 26 #include <linux/hardirq.h> 27 28 #include <asm/nmi.h> 29 #include <asm/amd-ibs.h> 30 31 /* attr.config2 */ 32 #define IBS_SW_FILTER_MASK 1 33 34 /* 35 * IBS states: 36 * 37 * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken 38 * and any further add()s must fail. 39 * 40 * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are 41 * complicated by the fact that the IBS hardware can send late NMIs (ie. after 42 * we've cleared the EN bit). 43 * 44 * In order to consume these late NMIs we have the STOPPED state, any NMI that 45 * happens after we've cleared the EN state will clear this bit and report the 46 * NMI handled (this is fundamentally racy in the face or multiple NMI sources, 47 * someone else can consume our BIT and our NMI will go unhandled). 48 * 49 * And since we cannot set/clear this separate bit together with the EN bit, 50 * there are races; if we cleared STARTED early, an NMI could land in 51 * between clearing STARTED and clearing the EN bit (in fact multiple NMIs 52 * could happen if the period is small enough), and consume our STOPPED bit 53 * and trigger streams of unhandled NMIs. 54 * 55 * If, however, we clear STARTED late, an NMI can hit between clearing the 56 * EN bit and clearing STARTED, still see STARTED set and process the event. 57 * If this event will have the VALID bit clear, we bail properly, but this 58 * is not a given. With VALID set we can end up calling pmu::stop() again 59 * (the throttle logic) and trigger the WARNs in there. 60 * 61 * So what we do is set STOPPING before clearing EN to avoid the pmu::stop() 62 * nesting, and clear STARTED late, so that we have a well defined state over 63 * the clearing of the EN bit. 64 * 65 * XXX: we could probably be using !atomic bitops for all this. 66 */ 67 68 enum ibs_states { 69 IBS_ENABLED = 0, 70 IBS_STARTED = 1, 71 IBS_STOPPING = 2, 72 IBS_STOPPED = 3, 73 74 IBS_MAX_STATES, 75 }; 76 77 struct cpu_perf_ibs { 78 struct perf_event *event; 79 unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)]; 80 }; 81 82 struct perf_ibs { 83 struct pmu pmu; 84 unsigned int msr; 85 u64 config_mask; 86 u64 cnt_mask; 87 u64 enable_mask; 88 u64 valid_mask; 89 u16 min_period; 90 u64 max_period; 91 unsigned long offset_mask[1]; 92 int offset_max; 93 unsigned int fetch_count_reset_broken : 1; 94 unsigned int fetch_ignore_if_zero_rip : 1; 95 struct cpu_perf_ibs __percpu *pcpu; 96 97 u64 (*get_count)(u64 config); 98 }; 99 100 static int 101 perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) 102 { 103 s64 left = local64_read(&hwc->period_left); 104 s64 period = hwc->sample_period; 105 int overflow = 0; 106 107 /* 108 * If we are way outside a reasonable range then just skip forward: 109 */ 110 if (unlikely(left <= -period)) { 111 left = period; 112 local64_set(&hwc->period_left, left); 113 hwc->last_period = period; 114 overflow = 1; 115 } 116 117 if (unlikely(left < (s64)min)) { 118 left += period; 119 local64_set(&hwc->period_left, left); 120 hwc->last_period = period; 121 overflow = 1; 122 } 123 124 /* 125 * If the hw period that triggers the sw overflow is too short 126 * we might hit the irq handler. This biases the results. 127 * Thus we shorten the next-to-last period and set the last 128 * period to the max period. 129 */ 130 if (left > max) { 131 left -= max; 132 if (left > max) 133 left = max; 134 else if (left < min) 135 left = min; 136 } 137 138 *hw_period = (u64)left; 139 140 return overflow; 141 } 142 143 static int 144 perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) 145 { 146 struct hw_perf_event *hwc = &event->hw; 147 int shift = 64 - width; 148 u64 prev_raw_count; 149 u64 delta; 150 151 /* 152 * Careful: an NMI might modify the previous event value. 153 * 154 * Our tactic to handle this is to first atomically read and 155 * exchange a new raw count - then add that new-prev delta 156 * count to the generic event atomically: 157 */ 158 prev_raw_count = local64_read(&hwc->prev_count); 159 if (!local64_try_cmpxchg(&hwc->prev_count, 160 &prev_raw_count, new_raw_count)) 161 return 0; 162 163 /* 164 * Now we have the new raw value and have updated the prev 165 * timestamp already. We can now calculate the elapsed delta 166 * (event-)time and add that to the generic event. 167 * 168 * Careful, not all hw sign-extends above the physical width 169 * of the count. 170 */ 171 delta = (new_raw_count << shift) - (prev_raw_count << shift); 172 delta >>= shift; 173 174 local64_add(delta, &event->count); 175 local64_sub(delta, &hwc->period_left); 176 177 return 1; 178 } 179 180 static struct perf_ibs perf_ibs_fetch; 181 static struct perf_ibs perf_ibs_op; 182 183 static struct perf_ibs *get_ibs_pmu(int type) 184 { 185 if (perf_ibs_fetch.pmu.type == type) 186 return &perf_ibs_fetch; 187 if (perf_ibs_op.pmu.type == type) 188 return &perf_ibs_op; 189 return NULL; 190 } 191 192 /* 193 * core pmu config -> IBS config 194 * 195 * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count 196 * perf record -a -e r076:p ... # same as -e cpu-cycles:p 197 * perf record -a -e r0C1:p ... # use ibs op counting micro-ops 198 * 199 * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, 200 * MSRC001_1033) is used to select either cycle or micro-ops counting 201 * mode. 202 */ 203 static int core_pmu_ibs_config(struct perf_event *event, u64 *config) 204 { 205 switch (event->attr.type) { 206 case PERF_TYPE_HARDWARE: 207 switch (event->attr.config) { 208 case PERF_COUNT_HW_CPU_CYCLES: 209 *config = 0; 210 return 0; 211 } 212 break; 213 case PERF_TYPE_RAW: 214 switch (event->attr.config) { 215 case 0x0076: 216 *config = 0; 217 return 0; 218 case 0x00C1: 219 *config = IBS_OP_CNT_CTL; 220 return 0; 221 } 222 break; 223 default: 224 return -ENOENT; 225 } 226 227 return -EOPNOTSUPP; 228 } 229 230 /* 231 * The rip of IBS samples has skid 0. Thus, IBS supports precise 232 * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the 233 * rip is invalid when IBS was not able to record the rip correctly. 234 * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. 235 */ 236 int forward_event_to_ibs(struct perf_event *event) 237 { 238 u64 config = 0; 239 240 if (!event->attr.precise_ip || event->attr.precise_ip > 2) 241 return -EOPNOTSUPP; 242 243 if (!core_pmu_ibs_config(event, &config)) { 244 event->attr.type = perf_ibs_op.pmu.type; 245 event->attr.config = config; 246 } 247 return -ENOENT; 248 } 249 250 /* 251 * Grouping of IBS events is not possible since IBS can have only 252 * one event active at any point in time. 253 */ 254 static int validate_group(struct perf_event *event) 255 { 256 struct perf_event *sibling; 257 258 if (event->group_leader == event) 259 return 0; 260 261 if (event->group_leader->pmu == event->pmu) 262 return -EINVAL; 263 264 for_each_sibling_event(sibling, event->group_leader) { 265 if (sibling->pmu == event->pmu) 266 return -EINVAL; 267 } 268 return 0; 269 } 270 271 static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs, 272 struct perf_event *event) 273 { 274 return perf_ibs == &perf_ibs_op && 275 (ibs_caps & IBS_CAPS_OPLDLAT) && 276 (event->attr.config1 & 0xFFF); 277 } 278 279 static int perf_ibs_init(struct perf_event *event) 280 { 281 struct hw_perf_event *hwc = &event->hw; 282 struct perf_ibs *perf_ibs; 283 u64 config; 284 int ret; 285 286 perf_ibs = get_ibs_pmu(event->attr.type); 287 if (!perf_ibs) 288 return -ENOENT; 289 290 config = event->attr.config; 291 292 if (event->pmu != &perf_ibs->pmu) 293 return -ENOENT; 294 295 if (config & ~perf_ibs->config_mask) 296 return -EINVAL; 297 298 if (has_branch_stack(event)) 299 return -EOPNOTSUPP; 300 301 /* handle exclude_{user,kernel} in the IRQ handler */ 302 if (event->attr.exclude_host || event->attr.exclude_guest || 303 event->attr.exclude_idle) 304 return -EINVAL; 305 306 if (!(event->attr.config2 & IBS_SW_FILTER_MASK) && 307 (event->attr.exclude_kernel || event->attr.exclude_user || 308 event->attr.exclude_hv)) 309 return -EINVAL; 310 311 ret = validate_group(event); 312 if (ret) 313 return ret; 314 315 if (hwc->sample_period) { 316 if (config & perf_ibs->cnt_mask) 317 /* raw max_cnt may not be set */ 318 return -EINVAL; 319 320 if (event->attr.freq) { 321 hwc->sample_period = perf_ibs->min_period; 322 } else { 323 /* Silently mask off lower nibble. IBS hw mandates it. */ 324 hwc->sample_period &= ~0x0FULL; 325 if (hwc->sample_period < perf_ibs->min_period) 326 return -EINVAL; 327 } 328 } else { 329 u64 period = 0; 330 331 if (event->attr.freq) 332 return -EINVAL; 333 334 if (perf_ibs == &perf_ibs_op) { 335 period = (config & IBS_OP_MAX_CNT) << 4; 336 if (ibs_caps & IBS_CAPS_OPCNTEXT) 337 period |= config & IBS_OP_MAX_CNT_EXT_MASK; 338 } else { 339 period = (config & IBS_FETCH_MAX_CNT) << 4; 340 } 341 342 config &= ~perf_ibs->cnt_mask; 343 event->attr.sample_period = period; 344 hwc->sample_period = period; 345 346 if (hwc->sample_period < perf_ibs->min_period) 347 return -EINVAL; 348 } 349 350 if (perf_ibs_ldlat_event(perf_ibs, event)) { 351 u64 ldlat = event->attr.config1 & 0xFFF; 352 353 if (ldlat < 128 || ldlat > 2048) 354 return -EINVAL; 355 ldlat >>= 7; 356 357 config |= (ldlat - 1) << 59; 358 config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN; 359 } 360 361 /* 362 * If we modify hwc->sample_period, we also need to update 363 * hwc->last_period and hwc->period_left. 364 */ 365 hwc->last_period = hwc->sample_period; 366 local64_set(&hwc->period_left, hwc->sample_period); 367 368 hwc->config_base = perf_ibs->msr; 369 hwc->config = config; 370 371 return 0; 372 } 373 374 static int perf_ibs_set_period(struct perf_ibs *perf_ibs, 375 struct hw_perf_event *hwc, u64 *period) 376 { 377 int overflow; 378 379 /* ignore lower 4 bits in min count: */ 380 overflow = perf_event_set_period(hwc, perf_ibs->min_period, 381 perf_ibs->max_period, period); 382 local64_set(&hwc->prev_count, 0); 383 384 return overflow; 385 } 386 387 static u64 get_ibs_fetch_count(u64 config) 388 { 389 union ibs_fetch_ctl fetch_ctl = (union ibs_fetch_ctl)config; 390 391 return fetch_ctl.fetch_cnt << 4; 392 } 393 394 static u64 get_ibs_op_count(u64 config) 395 { 396 union ibs_op_ctl op_ctl = (union ibs_op_ctl)config; 397 u64 count = 0; 398 399 /* 400 * If the internal 27-bit counter rolled over, the count is MaxCnt 401 * and the lower 7 bits of CurCnt are randomized. 402 * Otherwise CurCnt has the full 27-bit current counter value. 403 */ 404 if (op_ctl.op_val) { 405 count = op_ctl.opmaxcnt << 4; 406 if (ibs_caps & IBS_CAPS_OPCNTEXT) 407 count += op_ctl.opmaxcnt_ext << 20; 408 } else if (ibs_caps & IBS_CAPS_RDWROPCNT) { 409 count = op_ctl.opcurcnt; 410 } 411 412 return count; 413 } 414 415 static void 416 perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, 417 u64 *config) 418 { 419 u64 count = perf_ibs->get_count(*config); 420 421 /* 422 * Set width to 64 since we do not overflow on max width but 423 * instead on max count. In perf_ibs_set_period() we clear 424 * prev count manually on overflow. 425 */ 426 while (!perf_event_try_update(event, count, 64)) { 427 rdmsrl(event->hw.config_base, *config); 428 count = perf_ibs->get_count(*config); 429 } 430 } 431 432 static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, 433 struct hw_perf_event *hwc, u64 config) 434 { 435 u64 tmp = hwc->config | config; 436 437 if (perf_ibs->fetch_count_reset_broken) 438 wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask); 439 440 wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask); 441 } 442 443 /* 444 * Erratum #420 Instruction-Based Sampling Engine May Generate 445 * Interrupt that Cannot Be Cleared: 446 * 447 * Must clear counter mask first, then clear the enable bit. See 448 * Revision Guide for AMD Family 10h Processors, Publication #41322. 449 */ 450 static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, 451 struct hw_perf_event *hwc, u64 config) 452 { 453 config &= ~perf_ibs->cnt_mask; 454 if (boot_cpu_data.x86 == 0x10) 455 wrmsrl(hwc->config_base, config); 456 config &= ~perf_ibs->enable_mask; 457 wrmsrl(hwc->config_base, config); 458 } 459 460 /* 461 * We cannot restore the ibs pmu state, so we always needs to update 462 * the event while stopping it and then reset the state when starting 463 * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in 464 * perf_ibs_start()/perf_ibs_stop() and instead always do it. 465 */ 466 static void perf_ibs_start(struct perf_event *event, int flags) 467 { 468 struct hw_perf_event *hwc = &event->hw; 469 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 470 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 471 u64 period, config = 0; 472 473 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 474 return; 475 476 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 477 hwc->state = 0; 478 479 if (event->attr.freq && hwc->sample_period < perf_ibs->min_period) 480 hwc->sample_period = perf_ibs->min_period; 481 482 perf_ibs_set_period(perf_ibs, hwc, &period); 483 if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) { 484 config |= period & IBS_OP_MAX_CNT_EXT_MASK; 485 period &= ~IBS_OP_MAX_CNT_EXT_MASK; 486 } 487 config |= period >> 4; 488 489 /* 490 * Set STARTED before enabling the hardware, such that a subsequent NMI 491 * must observe it. 492 */ 493 set_bit(IBS_STARTED, pcpu->state); 494 clear_bit(IBS_STOPPING, pcpu->state); 495 perf_ibs_enable_event(perf_ibs, hwc, config); 496 497 perf_event_update_userpage(event); 498 } 499 500 static void perf_ibs_stop(struct perf_event *event, int flags) 501 { 502 struct hw_perf_event *hwc = &event->hw; 503 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 504 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 505 u64 config; 506 int stopping; 507 508 if (test_and_set_bit(IBS_STOPPING, pcpu->state)) 509 return; 510 511 stopping = test_bit(IBS_STARTED, pcpu->state); 512 513 if (!stopping && (hwc->state & PERF_HES_UPTODATE)) 514 return; 515 516 rdmsrl(hwc->config_base, config); 517 518 if (stopping) { 519 /* 520 * Set STOPPED before disabling the hardware, such that it 521 * must be visible to NMIs the moment we clear the EN bit, 522 * at which point we can generate an !VALID sample which 523 * we need to consume. 524 */ 525 set_bit(IBS_STOPPED, pcpu->state); 526 perf_ibs_disable_event(perf_ibs, hwc, config); 527 /* 528 * Clear STARTED after disabling the hardware; if it were 529 * cleared before an NMI hitting after the clear but before 530 * clearing the EN bit might think it a spurious NMI and not 531 * handle it. 532 * 533 * Clearing it after, however, creates the problem of the NMI 534 * handler seeing STARTED but not having a valid sample. 535 */ 536 clear_bit(IBS_STARTED, pcpu->state); 537 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 538 hwc->state |= PERF_HES_STOPPED; 539 } 540 541 if (hwc->state & PERF_HES_UPTODATE) 542 return; 543 544 /* 545 * Clear valid bit to not count rollovers on update, rollovers 546 * are only updated in the irq handler. 547 */ 548 config &= ~perf_ibs->valid_mask; 549 550 perf_ibs_event_update(perf_ibs, event, &config); 551 hwc->state |= PERF_HES_UPTODATE; 552 } 553 554 static int perf_ibs_add(struct perf_event *event, int flags) 555 { 556 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 557 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 558 559 if (test_and_set_bit(IBS_ENABLED, pcpu->state)) 560 return -ENOSPC; 561 562 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 563 564 pcpu->event = event; 565 566 if (flags & PERF_EF_START) 567 perf_ibs_start(event, PERF_EF_RELOAD); 568 569 return 0; 570 } 571 572 static void perf_ibs_del(struct perf_event *event, int flags) 573 { 574 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 575 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 576 577 if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) 578 return; 579 580 perf_ibs_stop(event, PERF_EF_UPDATE); 581 582 pcpu->event = NULL; 583 584 perf_event_update_userpage(event); 585 } 586 587 static void perf_ibs_read(struct perf_event *event) { } 588 589 static int perf_ibs_check_period(struct perf_event *event, u64 value) 590 { 591 struct perf_ibs *perf_ibs; 592 u64 low_nibble; 593 594 if (event->attr.freq) 595 return 0; 596 597 perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 598 low_nibble = value & 0xFULL; 599 600 /* 601 * This contradicts with perf_ibs_init() which allows sample period 602 * with lower nibble bits set but silently masks them off. Whereas 603 * this returns error. 604 */ 605 if (low_nibble || value < perf_ibs->min_period) 606 return -EINVAL; 607 608 return 0; 609 } 610 611 /* 612 * We need to initialize with empty group if all attributes in the 613 * group are dynamic. 614 */ 615 static struct attribute *attrs_empty[] = { 616 NULL, 617 }; 618 619 static struct attribute_group empty_caps_group = { 620 .name = "caps", 621 .attrs = attrs_empty, 622 }; 623 624 PMU_FORMAT_ATTR(rand_en, "config:57"); 625 PMU_FORMAT_ATTR(cnt_ctl, "config:19"); 626 PMU_FORMAT_ATTR(swfilt, "config2:0"); 627 PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); 628 PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); 629 PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11"); 630 PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); 631 PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); 632 PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1"); 633 634 static umode_t 635 zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) 636 { 637 return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; 638 } 639 640 static umode_t 641 ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) 642 { 643 return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0; 644 } 645 646 static umode_t 647 ibs_op_dtlb_pgsize_is_visible(struct kobject *kobj, struct attribute *attr, int i) 648 { 649 return ibs_caps & IBS_CAPS_OPDTLBPGSIZE ? attr->mode : 0; 650 } 651 652 static struct attribute *fetch_attrs[] = { 653 &format_attr_rand_en.attr, 654 &format_attr_swfilt.attr, 655 NULL, 656 }; 657 658 static struct attribute *fetch_l3missonly_attrs[] = { 659 &fetch_l3missonly.attr.attr, 660 NULL, 661 }; 662 663 static struct attribute *zen4_ibs_extensions_attrs[] = { 664 &zen4_ibs_extensions.attr.attr, 665 NULL, 666 }; 667 668 static struct attribute *ibs_op_ldlat_cap_attrs[] = { 669 &ibs_op_ldlat_cap.attr.attr, 670 NULL, 671 }; 672 673 static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = { 674 &ibs_op_dtlb_pgsize_cap.attr.attr, 675 NULL, 676 }; 677 678 static struct attribute_group group_fetch_formats = { 679 .name = "format", 680 .attrs = fetch_attrs, 681 }; 682 683 static struct attribute_group group_fetch_l3missonly = { 684 .name = "format", 685 .attrs = fetch_l3missonly_attrs, 686 .is_visible = zen4_ibs_extensions_is_visible, 687 }; 688 689 static struct attribute_group group_zen4_ibs_extensions = { 690 .name = "caps", 691 .attrs = zen4_ibs_extensions_attrs, 692 .is_visible = zen4_ibs_extensions_is_visible, 693 }; 694 695 static struct attribute_group group_ibs_op_ldlat_cap = { 696 .name = "caps", 697 .attrs = ibs_op_ldlat_cap_attrs, 698 .is_visible = ibs_op_ldlat_is_visible, 699 }; 700 701 static struct attribute_group group_ibs_op_dtlb_pgsize_cap = { 702 .name = "caps", 703 .attrs = ibs_op_dtlb_pgsize_cap_attrs, 704 .is_visible = ibs_op_dtlb_pgsize_is_visible, 705 }; 706 707 static const struct attribute_group *fetch_attr_groups[] = { 708 &group_fetch_formats, 709 &empty_caps_group, 710 NULL, 711 }; 712 713 static const struct attribute_group *fetch_attr_update[] = { 714 &group_fetch_l3missonly, 715 &group_zen4_ibs_extensions, 716 NULL, 717 }; 718 719 static umode_t 720 cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i) 721 { 722 return ibs_caps & IBS_CAPS_OPCNT ? attr->mode : 0; 723 } 724 725 static struct attribute *op_attrs[] = { 726 &format_attr_swfilt.attr, 727 NULL, 728 }; 729 730 static struct attribute *cnt_ctl_attrs[] = { 731 &format_attr_cnt_ctl.attr, 732 NULL, 733 }; 734 735 static struct attribute *op_l3missonly_attrs[] = { 736 &op_l3missonly.attr.attr, 737 NULL, 738 }; 739 740 static struct attribute_group group_op_formats = { 741 .name = "format", 742 .attrs = op_attrs, 743 }; 744 745 static struct attribute *ibs_op_ldlat_format_attrs[] = { 746 &ibs_op_ldlat_format.attr.attr, 747 NULL, 748 }; 749 750 static struct attribute_group group_cnt_ctl = { 751 .name = "format", 752 .attrs = cnt_ctl_attrs, 753 .is_visible = cnt_ctl_is_visible, 754 }; 755 756 static struct attribute_group group_op_l3missonly = { 757 .name = "format", 758 .attrs = op_l3missonly_attrs, 759 .is_visible = zen4_ibs_extensions_is_visible, 760 }; 761 762 static const struct attribute_group *op_attr_groups[] = { 763 &group_op_formats, 764 &empty_caps_group, 765 NULL, 766 }; 767 768 static struct attribute_group group_ibs_op_ldlat_format = { 769 .name = "format", 770 .attrs = ibs_op_ldlat_format_attrs, 771 .is_visible = ibs_op_ldlat_is_visible, 772 }; 773 774 static const struct attribute_group *op_attr_update[] = { 775 &group_cnt_ctl, 776 &group_op_l3missonly, 777 &group_zen4_ibs_extensions, 778 &group_ibs_op_ldlat_cap, 779 &group_ibs_op_ldlat_format, 780 &group_ibs_op_dtlb_pgsize_cap, 781 NULL, 782 }; 783 784 static struct perf_ibs perf_ibs_fetch = { 785 .pmu = { 786 .task_ctx_nr = perf_hw_context, 787 788 .event_init = perf_ibs_init, 789 .add = perf_ibs_add, 790 .del = perf_ibs_del, 791 .start = perf_ibs_start, 792 .stop = perf_ibs_stop, 793 .read = perf_ibs_read, 794 .check_period = perf_ibs_check_period, 795 }, 796 .msr = MSR_AMD64_IBSFETCHCTL, 797 .config_mask = IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN, 798 .cnt_mask = IBS_FETCH_MAX_CNT, 799 .enable_mask = IBS_FETCH_ENABLE, 800 .valid_mask = IBS_FETCH_VAL, 801 .min_period = 0x10, 802 .max_period = IBS_FETCH_MAX_CNT << 4, 803 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, 804 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, 805 806 .get_count = get_ibs_fetch_count, 807 }; 808 809 static struct perf_ibs perf_ibs_op = { 810 .pmu = { 811 .task_ctx_nr = perf_hw_context, 812 813 .event_init = perf_ibs_init, 814 .add = perf_ibs_add, 815 .del = perf_ibs_del, 816 .start = perf_ibs_start, 817 .stop = perf_ibs_stop, 818 .read = perf_ibs_read, 819 .check_period = perf_ibs_check_period, 820 }, 821 .msr = MSR_AMD64_IBSOPCTL, 822 .config_mask = IBS_OP_MAX_CNT, 823 .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT | 824 IBS_OP_CUR_CNT_RAND, 825 .enable_mask = IBS_OP_ENABLE, 826 .valid_mask = IBS_OP_VAL, 827 .min_period = 0x90, 828 .max_period = IBS_OP_MAX_CNT << 4, 829 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, 830 .offset_max = MSR_AMD64_IBSOP_REG_COUNT, 831 832 .get_count = get_ibs_op_count, 833 }; 834 835 static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3, 836 struct perf_sample_data *data) 837 { 838 union perf_mem_data_src *data_src = &data->data_src; 839 840 data_src->mem_op = PERF_MEM_OP_NA; 841 842 if (op_data3->ld_op) 843 data_src->mem_op = PERF_MEM_OP_LOAD; 844 else if (op_data3->st_op) 845 data_src->mem_op = PERF_MEM_OP_STORE; 846 } 847 848 /* 849 * Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has 850 * more fine granular DataSrc encodings. Others have coarse. 851 */ 852 static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2) 853 { 854 if (ibs_caps & IBS_CAPS_ZEN4) 855 return (op_data2->data_src_hi << 3) | op_data2->data_src_lo; 856 857 return op_data2->data_src_lo; 858 } 859 860 #define L(x) (PERF_MEM_S(LVL, x) | PERF_MEM_S(LVL, HIT)) 861 #define LN(x) PERF_MEM_S(LVLNUM, x) 862 #define REM PERF_MEM_S(REMOTE, REMOTE) 863 #define HOPS(x) PERF_MEM_S(HOPS, x) 864 865 static u64 g_data_src[8] = { 866 [IBS_DATA_SRC_LOC_CACHE] = L(L3) | L(REM_CCE1) | LN(ANY_CACHE) | HOPS(0), 867 [IBS_DATA_SRC_DRAM] = L(LOC_RAM) | LN(RAM), 868 [IBS_DATA_SRC_REM_CACHE] = L(REM_CCE2) | LN(ANY_CACHE) | REM | HOPS(1), 869 [IBS_DATA_SRC_IO] = L(IO) | LN(IO), 870 }; 871 872 #define RMT_NODE_BITS (1 << IBS_DATA_SRC_DRAM) 873 #define RMT_NODE_APPLICABLE(x) (RMT_NODE_BITS & (1 << x)) 874 875 static u64 g_zen4_data_src[32] = { 876 [IBS_DATA_SRC_EXT_LOC_CACHE] = L(L3) | LN(L3), 877 [IBS_DATA_SRC_EXT_NEAR_CCX_CACHE] = L(REM_CCE1) | LN(ANY_CACHE) | REM | HOPS(0), 878 [IBS_DATA_SRC_EXT_DRAM] = L(LOC_RAM) | LN(RAM), 879 [IBS_DATA_SRC_EXT_FAR_CCX_CACHE] = L(REM_CCE2) | LN(ANY_CACHE) | REM | HOPS(1), 880 [IBS_DATA_SRC_EXT_PMEM] = LN(PMEM), 881 [IBS_DATA_SRC_EXT_IO] = L(IO) | LN(IO), 882 [IBS_DATA_SRC_EXT_EXT_MEM] = LN(CXL), 883 }; 884 885 #define ZEN4_RMT_NODE_BITS ((1 << IBS_DATA_SRC_EXT_DRAM) | \ 886 (1 << IBS_DATA_SRC_EXT_PMEM) | \ 887 (1 << IBS_DATA_SRC_EXT_EXT_MEM)) 888 #define ZEN4_RMT_NODE_APPLICABLE(x) (ZEN4_RMT_NODE_BITS & (1 << x)) 889 890 static __u64 perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, 891 union ibs_op_data3 *op_data3, 892 struct perf_sample_data *data) 893 { 894 union perf_mem_data_src *data_src = &data->data_src; 895 u8 ibs_data_src = perf_ibs_data_src(op_data2); 896 897 data_src->mem_lvl = 0; 898 data_src->mem_lvl_num = 0; 899 900 /* 901 * DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached 902 * memory accesses. So, check DcUcMemAcc bit early. 903 */ 904 if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO) 905 return L(UNC) | LN(UNC); 906 907 /* L1 Hit */ 908 if (op_data3->dc_miss == 0) 909 return L(L1) | LN(L1); 910 911 /* L2 Hit */ 912 if (op_data3->l2_miss == 0) { 913 /* Erratum #1293 */ 914 if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF || 915 !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) 916 return L(L2) | LN(L2); 917 } 918 919 /* 920 * OP_DATA2 is valid only for load ops. Skip all checks which 921 * uses OP_DATA2[DataSrc]. 922 */ 923 if (data_src->mem_op != PERF_MEM_OP_LOAD) 924 goto check_mab; 925 926 if (ibs_caps & IBS_CAPS_ZEN4) { 927 u64 val = g_zen4_data_src[ibs_data_src]; 928 929 if (!val) 930 goto check_mab; 931 932 /* HOPS_1 because IBS doesn't provide remote socket detail */ 933 if (op_data2->rmt_node && ZEN4_RMT_NODE_APPLICABLE(ibs_data_src)) { 934 if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM) 935 val = L(REM_RAM1) | LN(RAM) | REM | HOPS(1); 936 else 937 val |= REM | HOPS(1); 938 } 939 940 return val; 941 } else { 942 u64 val = g_data_src[ibs_data_src]; 943 944 if (!val) 945 goto check_mab; 946 947 /* HOPS_1 because IBS doesn't provide remote socket detail */ 948 if (op_data2->rmt_node && RMT_NODE_APPLICABLE(ibs_data_src)) { 949 if (ibs_data_src == IBS_DATA_SRC_DRAM) 950 val = L(REM_RAM1) | LN(RAM) | REM | HOPS(1); 951 else 952 val |= REM | HOPS(1); 953 } 954 955 return val; 956 } 957 958 check_mab: 959 /* 960 * MAB (Miss Address Buffer) Hit. MAB keeps track of outstanding 961 * DC misses. However, such data may come from any level in mem 962 * hierarchy. IBS provides detail about both MAB as well as actual 963 * DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set 964 * MAB only when IBS fails to provide DataSrc. 965 */ 966 if (op_data3->dc_miss_no_mab_alloc) 967 return L(LFB) | LN(LFB); 968 969 /* Don't set HIT with NA */ 970 return PERF_MEM_S(LVL, NA) | LN(NA); 971 } 972 973 static bool perf_ibs_cache_hit_st_valid(void) 974 { 975 /* 0: Uninitialized, 1: Valid, -1: Invalid */ 976 static int cache_hit_st_valid; 977 978 if (unlikely(!cache_hit_st_valid)) { 979 if (boot_cpu_data.x86 == 0x19 && 980 (boot_cpu_data.x86_model <= 0xF || 981 (boot_cpu_data.x86_model >= 0x20 && 982 boot_cpu_data.x86_model <= 0x5F))) { 983 cache_hit_st_valid = -1; 984 } else { 985 cache_hit_st_valid = 1; 986 } 987 } 988 989 return cache_hit_st_valid == 1; 990 } 991 992 static void perf_ibs_get_mem_snoop(union ibs_op_data2 *op_data2, 993 struct perf_sample_data *data) 994 { 995 union perf_mem_data_src *data_src = &data->data_src; 996 u8 ibs_data_src; 997 998 data_src->mem_snoop = PERF_MEM_SNOOP_NA; 999 1000 if (!perf_ibs_cache_hit_st_valid() || 1001 data_src->mem_op != PERF_MEM_OP_LOAD || 1002 data_src->mem_lvl & PERF_MEM_LVL_L1 || 1003 data_src->mem_lvl & PERF_MEM_LVL_L2 || 1004 op_data2->cache_hit_st) 1005 return; 1006 1007 ibs_data_src = perf_ibs_data_src(op_data2); 1008 1009 if (ibs_caps & IBS_CAPS_ZEN4) { 1010 if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE || 1011 ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE || 1012 ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) 1013 data_src->mem_snoop = PERF_MEM_SNOOP_HITM; 1014 } else if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) { 1015 data_src->mem_snoop = PERF_MEM_SNOOP_HITM; 1016 } 1017 } 1018 1019 static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, 1020 struct perf_sample_data *data) 1021 { 1022 union perf_mem_data_src *data_src = &data->data_src; 1023 1024 data_src->mem_dtlb = PERF_MEM_TLB_NA; 1025 1026 if (!op_data3->dc_lin_addr_valid) 1027 return; 1028 1029 if ((ibs_caps & IBS_CAPS_OPDTLBPGSIZE) && 1030 !op_data3->dc_phy_addr_valid) 1031 return; 1032 1033 if (!op_data3->dc_l1tlb_miss) { 1034 data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT; 1035 return; 1036 } 1037 1038 if (!op_data3->dc_l2tlb_miss) { 1039 data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT; 1040 return; 1041 } 1042 1043 data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS; 1044 } 1045 1046 static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3, 1047 struct perf_sample_data *data) 1048 { 1049 union perf_mem_data_src *data_src = &data->data_src; 1050 1051 data_src->mem_lock = PERF_MEM_LOCK_NA; 1052 1053 if (op_data3->dc_locked_op) 1054 data_src->mem_lock = PERF_MEM_LOCK_LOCKED; 1055 } 1056 1057 /* Be careful. Works only for contiguous MSRs. */ 1058 #define ibs_fetch_msr_idx(msr) (msr - MSR_AMD64_IBSFETCHCTL) 1059 #define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL) 1060 1061 static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, 1062 struct perf_sample_data *data, 1063 union ibs_op_data2 *op_data2, 1064 union ibs_op_data3 *op_data3) 1065 { 1066 union perf_mem_data_src *data_src = &data->data_src; 1067 1068 data_src->val |= perf_ibs_get_mem_lvl(op_data2, op_data3, data); 1069 perf_ibs_get_mem_snoop(op_data2, data); 1070 perf_ibs_get_tlb_lvl(op_data3, data); 1071 perf_ibs_get_mem_lock(op_data3, data); 1072 } 1073 1074 static __u64 perf_ibs_get_op_data2(struct perf_ibs_data *ibs_data, 1075 union ibs_op_data3 *op_data3) 1076 { 1077 __u64 val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA2)]; 1078 1079 /* Erratum #1293 */ 1080 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model <= 0xF && 1081 (op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) { 1082 /* 1083 * OP_DATA2 has only two fields on Zen3: DataSrc and RmtNode. 1084 * DataSrc=0 is 'No valid status' and RmtNode is invalid when 1085 * DataSrc=0. 1086 */ 1087 val = 0; 1088 } 1089 return val; 1090 } 1091 1092 static void perf_ibs_parse_ld_st_data(__u64 sample_type, 1093 struct perf_ibs_data *ibs_data, 1094 struct perf_sample_data *data) 1095 { 1096 union ibs_op_data3 op_data3; 1097 union ibs_op_data2 op_data2; 1098 union ibs_op_data op_data; 1099 1100 data->data_src.val = PERF_MEM_NA; 1101 op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 1102 1103 perf_ibs_get_mem_op(&op_data3, data); 1104 if (data->data_src.mem_op != PERF_MEM_OP_LOAD && 1105 data->data_src.mem_op != PERF_MEM_OP_STORE) 1106 return; 1107 1108 op_data2.val = perf_ibs_get_op_data2(ibs_data, &op_data3); 1109 1110 if (sample_type & PERF_SAMPLE_DATA_SRC) { 1111 perf_ibs_get_data_src(ibs_data, data, &op_data2, &op_data3); 1112 data->sample_flags |= PERF_SAMPLE_DATA_SRC; 1113 } 1114 1115 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE && op_data3.dc_miss && 1116 data->data_src.mem_op == PERF_MEM_OP_LOAD) { 1117 op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; 1118 1119 if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) { 1120 data->weight.var1_dw = op_data3.dc_miss_lat; 1121 data->weight.var2_w = op_data.tag_to_ret_ctr; 1122 } else if (sample_type & PERF_SAMPLE_WEIGHT) { 1123 data->weight.full = op_data3.dc_miss_lat; 1124 } 1125 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; 1126 } 1127 1128 if (sample_type & PERF_SAMPLE_ADDR && op_data3.dc_lin_addr_valid) { 1129 data->addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; 1130 data->sample_flags |= PERF_SAMPLE_ADDR; 1131 } 1132 1133 if (sample_type & PERF_SAMPLE_PHYS_ADDR && op_data3.dc_phy_addr_valid) { 1134 data->phys_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)]; 1135 data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; 1136 } 1137 } 1138 1139 static bool perf_ibs_is_mem_sample_type(struct perf_ibs *perf_ibs, 1140 struct perf_event *event) 1141 { 1142 u64 sample_type = event->attr.sample_type; 1143 1144 return perf_ibs == &perf_ibs_op && 1145 sample_type & (PERF_SAMPLE_DATA_SRC | 1146 PERF_SAMPLE_WEIGHT_TYPE | 1147 PERF_SAMPLE_ADDR | 1148 PERF_SAMPLE_PHYS_ADDR); 1149 } 1150 1151 static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, 1152 struct perf_event *event, 1153 int check_rip) 1154 { 1155 if (event->attr.sample_type & PERF_SAMPLE_RAW || 1156 perf_ibs_is_mem_sample_type(perf_ibs, event) || 1157 perf_ibs_ldlat_event(perf_ibs, event)) 1158 return perf_ibs->offset_max; 1159 else if (check_rip) 1160 return 3; 1161 return 1; 1162 } 1163 1164 static bool perf_ibs_is_kernel_data_addr(struct perf_event *event, 1165 struct perf_ibs_data *ibs_data) 1166 { 1167 u64 sample_type_mask = PERF_SAMPLE_ADDR | PERF_SAMPLE_RAW; 1168 union ibs_op_data3 op_data3; 1169 u64 dc_lin_addr; 1170 1171 op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 1172 dc_lin_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; 1173 1174 return unlikely((event->attr.sample_type & sample_type_mask) && 1175 op_data3.dc_lin_addr_valid && kernel_ip(dc_lin_addr)); 1176 } 1177 1178 static bool perf_ibs_is_kernel_br_target(struct perf_event *event, 1179 struct perf_ibs_data *ibs_data, 1180 int br_target_idx) 1181 { 1182 union ibs_op_data op_data; 1183 u64 br_target; 1184 1185 op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; 1186 br_target = ibs_data->regs[br_target_idx]; 1187 1188 return unlikely((event->attr.sample_type & PERF_SAMPLE_RAW) && 1189 op_data.op_brn_ret && kernel_ip(br_target)); 1190 } 1191 1192 static bool perf_ibs_swfilt_discard(struct perf_ibs *perf_ibs, struct perf_event *event, 1193 struct pt_regs *regs, struct perf_ibs_data *ibs_data, 1194 int br_target_idx) 1195 { 1196 if (perf_exclude_event(event, regs)) 1197 return true; 1198 1199 if (perf_ibs != &perf_ibs_op || !event->attr.exclude_kernel) 1200 return false; 1201 1202 if (perf_ibs_is_kernel_data_addr(event, ibs_data)) 1203 return true; 1204 1205 if (br_target_idx != -1 && 1206 perf_ibs_is_kernel_br_target(event, ibs_data, br_target_idx)) 1207 return true; 1208 1209 return false; 1210 } 1211 1212 static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs, 1213 struct perf_ibs_data *ibs_data) 1214 { 1215 if (perf_ibs == &perf_ibs_op) { 1216 ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)] &= ~(1ULL << 18); 1217 ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0; 1218 return; 1219 } 1220 1221 ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)] &= ~(1ULL << 52); 1222 ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0; 1223 } 1224 1225 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) 1226 { 1227 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 1228 struct perf_event *event = pcpu->event; 1229 struct hw_perf_event *hwc; 1230 struct perf_sample_data data; 1231 struct perf_raw_record raw; 1232 struct pt_regs regs; 1233 struct perf_ibs_data ibs_data; 1234 int offset, size, check_rip, offset_max, throttle = 0; 1235 unsigned int msr; 1236 u64 *buf, *config, period, new_config = 0; 1237 int br_target_idx = -1; 1238 1239 if (!test_bit(IBS_STARTED, pcpu->state)) { 1240 fail: 1241 /* 1242 * Catch spurious interrupts after stopping IBS: After 1243 * disabling IBS there could be still incoming NMIs 1244 * with samples that even have the valid bit cleared. 1245 * Mark all this NMIs as handled. 1246 */ 1247 if (test_and_clear_bit(IBS_STOPPED, pcpu->state)) 1248 return 1; 1249 1250 return 0; 1251 } 1252 1253 if (WARN_ON_ONCE(!event)) 1254 goto fail; 1255 1256 hwc = &event->hw; 1257 msr = hwc->config_base; 1258 buf = ibs_data.regs; 1259 rdmsrl(msr, *buf); 1260 if (!(*buf++ & perf_ibs->valid_mask)) 1261 goto fail; 1262 1263 config = &ibs_data.regs[0]; 1264 perf_ibs_event_update(perf_ibs, event, config); 1265 perf_sample_data_init(&data, 0, hwc->last_period); 1266 if (!perf_ibs_set_period(perf_ibs, hwc, &period)) 1267 goto out; /* no sw counter overflow */ 1268 1269 ibs_data.caps = ibs_caps; 1270 size = 1; 1271 offset = 1; 1272 check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); 1273 1274 offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip); 1275 1276 do { 1277 rdmsrl(msr + offset, *buf++); 1278 size++; 1279 offset = find_next_bit(perf_ibs->offset_mask, 1280 perf_ibs->offset_max, 1281 offset + 1); 1282 } while (offset < offset_max); 1283 1284 if (perf_ibs_ldlat_event(perf_ibs, event)) { 1285 union ibs_op_data3 op_data3; 1286 1287 op_data3.val = ibs_data.regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 1288 /* 1289 * Opening event is errored out if load latency threshold is 1290 * outside of [128, 2048] range. Since the event has reached 1291 * interrupt handler, we can safely assume the threshold is 1292 * within [128, 2048] range. 1293 */ 1294 if (!op_data3.ld_op || !op_data3.dc_miss || 1295 op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) 1296 goto out; 1297 } 1298 1299 /* 1300 * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately 1301 * depending on their availability. 1302 * Can't add to offset_max as they are staggered 1303 */ 1304 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 1305 if (perf_ibs == &perf_ibs_op) { 1306 if (ibs_caps & IBS_CAPS_BRNTRGT) { 1307 rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); 1308 br_target_idx = size; 1309 size++; 1310 } 1311 if (ibs_caps & IBS_CAPS_OPDATA4) { 1312 rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++); 1313 size++; 1314 } 1315 } 1316 if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) { 1317 rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++); 1318 size++; 1319 } 1320 } 1321 ibs_data.size = sizeof(u64) * size; 1322 1323 regs = *iregs; 1324 if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { 1325 regs.flags &= ~PERF_EFLAGS_EXACT; 1326 } else { 1327 /* Workaround for erratum #1197 */ 1328 if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) 1329 goto out; 1330 1331 set_linear_ip(®s, ibs_data.regs[1]); 1332 regs.flags |= PERF_EFLAGS_EXACT; 1333 } 1334 1335 if ((event->attr.config2 & IBS_SW_FILTER_MASK) && 1336 perf_ibs_swfilt_discard(perf_ibs, event, ®s, &ibs_data, br_target_idx)) { 1337 throttle = perf_event_account_interrupt(event); 1338 goto out; 1339 } 1340 /* 1341 * Prevent leaking physical addresses to unprivileged users. Skip 1342 * PERF_SAMPLE_PHYS_ADDR check since generic code prevents it for 1343 * unprivileged users. 1344 */ 1345 if ((event->attr.sample_type & PERF_SAMPLE_RAW) && 1346 perf_allow_kernel()) { 1347 perf_ibs_phyaddr_clear(perf_ibs, &ibs_data); 1348 } 1349 1350 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 1351 raw = (struct perf_raw_record){ 1352 .frag = { 1353 .size = sizeof(u32) + ibs_data.size, 1354 .data = ibs_data.data, 1355 }, 1356 }; 1357 perf_sample_save_raw_data(&data, event, &raw); 1358 } 1359 1360 if (perf_ibs == &perf_ibs_op) 1361 perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); 1362 1363 /* 1364 * rip recorded by IbsOpRip will not be consistent with rsp and rbp 1365 * recorded as part of interrupt regs. Thus we need to use rip from 1366 * interrupt regs while unwinding call stack. 1367 */ 1368 perf_sample_save_callchain(&data, event, iregs); 1369 1370 throttle = perf_event_overflow(event, &data, ®s); 1371 1372 if (event->attr.freq && hwc->sample_period < perf_ibs->min_period) 1373 hwc->sample_period = perf_ibs->min_period; 1374 1375 out: 1376 if (throttle) { 1377 perf_ibs_stop(event, 0); 1378 } else { 1379 if (perf_ibs == &perf_ibs_op) { 1380 if (ibs_caps & IBS_CAPS_OPCNTEXT) { 1381 new_config = period & IBS_OP_MAX_CNT_EXT_MASK; 1382 period &= ~IBS_OP_MAX_CNT_EXT_MASK; 1383 } 1384 if ((ibs_caps & IBS_CAPS_RDWROPCNT) && (*config & IBS_OP_CNT_CTL)) 1385 new_config |= *config & IBS_OP_CUR_CNT_RAND; 1386 } 1387 new_config |= period >> 4; 1388 1389 perf_ibs_enable_event(perf_ibs, hwc, new_config); 1390 } 1391 1392 perf_event_update_userpage(event); 1393 1394 return 1; 1395 } 1396 1397 static int 1398 perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1399 { 1400 u64 stamp = sched_clock(); 1401 int handled = 0; 1402 1403 handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs); 1404 handled += perf_ibs_handle_irq(&perf_ibs_op, regs); 1405 1406 if (handled) 1407 inc_irq_stat(apic_perf_irqs); 1408 1409 perf_sample_event_took(sched_clock() - stamp); 1410 1411 return handled; 1412 } 1413 NOKPROBE_SYMBOL(perf_ibs_nmi_handler); 1414 1415 static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) 1416 { 1417 struct cpu_perf_ibs __percpu *pcpu; 1418 int ret; 1419 1420 pcpu = alloc_percpu(struct cpu_perf_ibs); 1421 if (!pcpu) 1422 return -ENOMEM; 1423 1424 perf_ibs->pcpu = pcpu; 1425 1426 ret = perf_pmu_register(&perf_ibs->pmu, name, -1); 1427 if (ret) { 1428 perf_ibs->pcpu = NULL; 1429 free_percpu(pcpu); 1430 } 1431 1432 return ret; 1433 } 1434 1435 static __init int perf_ibs_fetch_init(void) 1436 { 1437 /* 1438 * Some chips fail to reset the fetch count when it is written; instead 1439 * they need a 0-1 transition of IbsFetchEn. 1440 */ 1441 if (boot_cpu_data.x86 >= 0x16 && boot_cpu_data.x86 <= 0x18) 1442 perf_ibs_fetch.fetch_count_reset_broken = 1; 1443 1444 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model < 0x10) 1445 perf_ibs_fetch.fetch_ignore_if_zero_rip = 1; 1446 1447 if (ibs_caps & IBS_CAPS_ZEN4) 1448 perf_ibs_fetch.config_mask |= IBS_FETCH_L3MISSONLY; 1449 1450 perf_ibs_fetch.pmu.attr_groups = fetch_attr_groups; 1451 perf_ibs_fetch.pmu.attr_update = fetch_attr_update; 1452 1453 return perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); 1454 } 1455 1456 static __init int perf_ibs_op_init(void) 1457 { 1458 if (ibs_caps & IBS_CAPS_OPCNT) 1459 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; 1460 1461 if (ibs_caps & IBS_CAPS_OPCNTEXT) { 1462 perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK; 1463 perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK; 1464 perf_ibs_op.cnt_mask |= (IBS_OP_MAX_CNT_EXT_MASK | 1465 IBS_OP_CUR_CNT_EXT_MASK); 1466 } 1467 1468 if (ibs_caps & IBS_CAPS_ZEN4) 1469 perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY; 1470 1471 perf_ibs_op.pmu.attr_groups = op_attr_groups; 1472 perf_ibs_op.pmu.attr_update = op_attr_update; 1473 1474 return perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); 1475 } 1476 1477 static __init int perf_event_ibs_init(void) 1478 { 1479 int ret; 1480 1481 ret = perf_ibs_fetch_init(); 1482 if (ret) 1483 return ret; 1484 1485 ret = perf_ibs_op_init(); 1486 if (ret) 1487 goto err_op; 1488 1489 ret = register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); 1490 if (ret) 1491 goto err_nmi; 1492 1493 pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps); 1494 return 0; 1495 1496 err_nmi: 1497 perf_pmu_unregister(&perf_ibs_op.pmu); 1498 free_percpu(perf_ibs_op.pcpu); 1499 perf_ibs_op.pcpu = NULL; 1500 err_op: 1501 perf_pmu_unregister(&perf_ibs_fetch.pmu); 1502 free_percpu(perf_ibs_fetch.pcpu); 1503 perf_ibs_fetch.pcpu = NULL; 1504 1505 return ret; 1506 } 1507 1508 #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ 1509 1510 static __init int perf_event_ibs_init(void) 1511 { 1512 return 0; 1513 } 1514 1515 #endif 1516 1517 /* IBS - apic initialization, for perf and oprofile */ 1518 1519 static __init u32 __get_ibs_caps(void) 1520 { 1521 u32 caps; 1522 unsigned int max_level; 1523 1524 if (!boot_cpu_has(X86_FEATURE_IBS)) 1525 return 0; 1526 1527 /* check IBS cpuid feature flags */ 1528 max_level = cpuid_eax(0x80000000); 1529 if (max_level < IBS_CPUID_FEATURES) 1530 return IBS_CAPS_DEFAULT; 1531 1532 caps = cpuid_eax(IBS_CPUID_FEATURES); 1533 if (!(caps & IBS_CAPS_AVAIL)) 1534 /* cpuid flags not valid */ 1535 return IBS_CAPS_DEFAULT; 1536 1537 return caps; 1538 } 1539 1540 u32 get_ibs_caps(void) 1541 { 1542 return ibs_caps; 1543 } 1544 1545 EXPORT_SYMBOL(get_ibs_caps); 1546 1547 static inline int get_eilvt(int offset) 1548 { 1549 return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); 1550 } 1551 1552 static inline int put_eilvt(int offset) 1553 { 1554 return !setup_APIC_eilvt(offset, 0, 0, 1); 1555 } 1556 1557 /* 1558 * Check and reserve APIC extended interrupt LVT offset for IBS if available. 1559 */ 1560 static inline int ibs_eilvt_valid(void) 1561 { 1562 int offset; 1563 u64 val; 1564 int valid = 0; 1565 1566 preempt_disable(); 1567 1568 rdmsrl(MSR_AMD64_IBSCTL, val); 1569 offset = val & IBSCTL_LVT_OFFSET_MASK; 1570 1571 if (!(val & IBSCTL_LVT_OFFSET_VALID)) { 1572 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", 1573 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 1574 goto out; 1575 } 1576 1577 if (!get_eilvt(offset)) { 1578 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", 1579 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 1580 goto out; 1581 } 1582 1583 valid = 1; 1584 out: 1585 preempt_enable(); 1586 1587 return valid; 1588 } 1589 1590 static int setup_ibs_ctl(int ibs_eilvt_off) 1591 { 1592 struct pci_dev *cpu_cfg; 1593 int nodes; 1594 u32 value = 0; 1595 1596 nodes = 0; 1597 cpu_cfg = NULL; 1598 do { 1599 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, 1600 PCI_DEVICE_ID_AMD_10H_NB_MISC, 1601 cpu_cfg); 1602 if (!cpu_cfg) 1603 break; 1604 ++nodes; 1605 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off 1606 | IBSCTL_LVT_OFFSET_VALID); 1607 pci_read_config_dword(cpu_cfg, IBSCTL, &value); 1608 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { 1609 pci_dev_put(cpu_cfg); 1610 pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n", 1611 value); 1612 return -EINVAL; 1613 } 1614 } while (1); 1615 1616 if (!nodes) { 1617 pr_debug("No CPU node configured for IBS\n"); 1618 return -ENODEV; 1619 } 1620 1621 return 0; 1622 } 1623 1624 /* 1625 * This runs only on the current cpu. We try to find an LVT offset and 1626 * setup the local APIC. For this we must disable preemption. On 1627 * success we initialize all nodes with this offset. This updates then 1628 * the offset in the IBS_CTL per-node msr. The per-core APIC setup of 1629 * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that 1630 * is using the new offset. 1631 */ 1632 static void force_ibs_eilvt_setup(void) 1633 { 1634 int offset; 1635 int ret; 1636 1637 preempt_disable(); 1638 /* find the next free available EILVT entry, skip offset 0 */ 1639 for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { 1640 if (get_eilvt(offset)) 1641 break; 1642 } 1643 preempt_enable(); 1644 1645 if (offset == APIC_EILVT_NR_MAX) { 1646 pr_debug("No EILVT entry available\n"); 1647 return; 1648 } 1649 1650 ret = setup_ibs_ctl(offset); 1651 if (ret) 1652 goto out; 1653 1654 if (!ibs_eilvt_valid()) 1655 goto out; 1656 1657 pr_info("LVT offset %d assigned\n", offset); 1658 1659 return; 1660 out: 1661 preempt_disable(); 1662 put_eilvt(offset); 1663 preempt_enable(); 1664 return; 1665 } 1666 1667 static void ibs_eilvt_setup(void) 1668 { 1669 /* 1670 * Force LVT offset assignment for family 10h: The offsets are 1671 * not assigned by the BIOS for this family, so the OS is 1672 * responsible for doing it. If the OS assignment fails, fall 1673 * back to BIOS settings and try to setup this. 1674 */ 1675 if (boot_cpu_data.x86 == 0x10) 1676 force_ibs_eilvt_setup(); 1677 } 1678 1679 static inline int get_ibs_lvt_offset(void) 1680 { 1681 u64 val; 1682 1683 rdmsrl(MSR_AMD64_IBSCTL, val); 1684 if (!(val & IBSCTL_LVT_OFFSET_VALID)) 1685 return -EINVAL; 1686 1687 return val & IBSCTL_LVT_OFFSET_MASK; 1688 } 1689 1690 static void setup_APIC_ibs(void) 1691 { 1692 int offset; 1693 1694 offset = get_ibs_lvt_offset(); 1695 if (offset < 0) 1696 goto failed; 1697 1698 if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) 1699 return; 1700 failed: 1701 pr_warn("perf: IBS APIC setup failed on cpu #%d\n", 1702 smp_processor_id()); 1703 } 1704 1705 static void clear_APIC_ibs(void) 1706 { 1707 int offset; 1708 1709 offset = get_ibs_lvt_offset(); 1710 if (offset >= 0) 1711 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); 1712 } 1713 1714 static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu) 1715 { 1716 setup_APIC_ibs(); 1717 return 0; 1718 } 1719 1720 #ifdef CONFIG_PM 1721 1722 static int perf_ibs_suspend(void) 1723 { 1724 clear_APIC_ibs(); 1725 return 0; 1726 } 1727 1728 static void perf_ibs_resume(void) 1729 { 1730 ibs_eilvt_setup(); 1731 setup_APIC_ibs(); 1732 } 1733 1734 static struct syscore_ops perf_ibs_syscore_ops = { 1735 .resume = perf_ibs_resume, 1736 .suspend = perf_ibs_suspend, 1737 }; 1738 1739 static void perf_ibs_pm_init(void) 1740 { 1741 register_syscore_ops(&perf_ibs_syscore_ops); 1742 } 1743 1744 #else 1745 1746 static inline void perf_ibs_pm_init(void) { } 1747 1748 #endif 1749 1750 static int x86_pmu_amd_ibs_dying_cpu(unsigned int cpu) 1751 { 1752 clear_APIC_ibs(); 1753 return 0; 1754 } 1755 1756 static __init int amd_ibs_init(void) 1757 { 1758 u32 caps; 1759 1760 caps = __get_ibs_caps(); 1761 if (!caps) 1762 return -ENODEV; /* ibs not supported by the cpu */ 1763 1764 ibs_eilvt_setup(); 1765 1766 if (!ibs_eilvt_valid()) 1767 return -EINVAL; 1768 1769 perf_ibs_pm_init(); 1770 1771 ibs_caps = caps; 1772 /* make ibs_caps visible to other cpus: */ 1773 smp_mb(); 1774 /* 1775 * x86_pmu_amd_ibs_starting_cpu will be called from core on 1776 * all online cpus. 1777 */ 1778 cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_IBS_STARTING, 1779 "perf/x86/amd/ibs:starting", 1780 x86_pmu_amd_ibs_starting_cpu, 1781 x86_pmu_amd_ibs_dying_cpu); 1782 1783 return perf_event_ibs_init(); 1784 } 1785 1786 /* Since we need the pci subsystem to init ibs we can't do this earlier: */ 1787 device_initcall(amd_ibs_init); 1788