1 // SPDX-License-Identifier: GPL-2.0-only 2 #undef DEBUG 3 4 /* 5 * ARM performance counter support. 6 * 7 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles 8 * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com> 9 * 10 * This code is based on the sparc64 perf event code, which is in turn based 11 * on the x86 code. 12 */ 13 #define pr_fmt(fmt) "hw perfevents: " fmt 14 15 #include <linux/bitmap.h> 16 #include <linux/cpumask.h> 17 #include <linux/cpu_pm.h> 18 #include <linux/export.h> 19 #include <linux/kernel.h> 20 #include <linux/perf/arm_pmu.h> 21 #include <linux/slab.h> 22 #include <linux/sched/clock.h> 23 #include <linux/spinlock.h> 24 #include <linux/irq.h> 25 #include <linux/irqdesc.h> 26 27 #include <asm/irq_regs.h> 28 29 static int armpmu_count_irq_users(const int irq); 30 31 struct pmu_irq_ops { 32 void (*enable_pmuirq)(unsigned int irq); 33 void (*disable_pmuirq)(unsigned int irq); 34 void (*free_pmuirq)(unsigned int irq, int cpu, void __percpu *devid); 35 }; 36 37 static void armpmu_free_pmuirq(unsigned int irq, int cpu, void __percpu *devid) 38 { 39 free_irq(irq, per_cpu_ptr(devid, cpu)); 40 } 41 42 static const struct pmu_irq_ops pmuirq_ops = { 43 .enable_pmuirq = enable_irq, 44 .disable_pmuirq = disable_irq_nosync, 45 .free_pmuirq = armpmu_free_pmuirq 46 }; 47 48 static void armpmu_free_pmunmi(unsigned int irq, int cpu, void __percpu *devid) 49 { 50 free_nmi(irq, per_cpu_ptr(devid, cpu)); 51 } 52 53 static const struct pmu_irq_ops pmunmi_ops = { 54 .enable_pmuirq = enable_nmi, 55 .disable_pmuirq = disable_nmi_nosync, 56 .free_pmuirq = armpmu_free_pmunmi 57 }; 58 59 static void armpmu_enable_percpu_pmuirq(unsigned int irq) 60 { 61 enable_percpu_irq(irq, IRQ_TYPE_NONE); 62 } 63 64 static void armpmu_free_percpu_pmuirq(unsigned int irq, int cpu, 65 void __percpu *devid) 66 { 67 if (armpmu_count_irq_users(irq) == 1) 68 free_percpu_irq(irq, devid); 69 } 70 71 static const struct pmu_irq_ops percpu_pmuirq_ops = { 72 .enable_pmuirq = armpmu_enable_percpu_pmuirq, 73 .disable_pmuirq = disable_percpu_irq, 74 .free_pmuirq = armpmu_free_percpu_pmuirq 75 }; 76 77 static void armpmu_enable_percpu_pmunmi(unsigned int irq) 78 { 79 if (!prepare_percpu_nmi(irq)) 80 enable_percpu_nmi(irq, IRQ_TYPE_NONE); 81 } 82 83 static void armpmu_disable_percpu_pmunmi(unsigned int irq) 84 { 85 disable_percpu_nmi(irq); 86 teardown_percpu_nmi(irq); 87 } 88 89 static void armpmu_free_percpu_pmunmi(unsigned int irq, int cpu, 90 void __percpu *devid) 91 { 92 if (armpmu_count_irq_users(irq) == 1) 93 free_percpu_nmi(irq, devid); 94 } 95 96 static const struct pmu_irq_ops percpu_pmunmi_ops = { 97 .enable_pmuirq = armpmu_enable_percpu_pmunmi, 98 .disable_pmuirq = armpmu_disable_percpu_pmunmi, 99 .free_pmuirq = armpmu_free_percpu_pmunmi 100 }; 101 102 static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu); 103 static DEFINE_PER_CPU(int, cpu_irq); 104 static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops); 105 106 static bool has_nmi; 107 108 static inline u64 arm_pmu_event_max_period(struct perf_event *event) 109 { 110 if (event->hw.flags & ARMPMU_EVT_64BIT) 111 return GENMASK_ULL(63, 0); 112 else if (event->hw.flags & ARMPMU_EVT_63BIT) 113 return GENMASK_ULL(62, 0); 114 else if (event->hw.flags & ARMPMU_EVT_47BIT) 115 return GENMASK_ULL(46, 0); 116 else 117 return GENMASK_ULL(31, 0); 118 } 119 120 static int 121 armpmu_map_cache_event(const unsigned (*cache_map) 122 [PERF_COUNT_HW_CACHE_MAX] 123 [PERF_COUNT_HW_CACHE_OP_MAX] 124 [PERF_COUNT_HW_CACHE_RESULT_MAX], 125 u64 config) 126 { 127 unsigned int cache_type, cache_op, cache_result, ret; 128 129 cache_type = (config >> 0) & 0xff; 130 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 131 return -EINVAL; 132 133 cache_op = (config >> 8) & 0xff; 134 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 135 return -EINVAL; 136 137 cache_result = (config >> 16) & 0xff; 138 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 139 return -EINVAL; 140 141 if (!cache_map) 142 return -ENOENT; 143 144 ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; 145 146 if (ret == CACHE_OP_UNSUPPORTED) 147 return -ENOENT; 148 149 return ret; 150 } 151 152 static int 153 armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) 154 { 155 int mapping; 156 157 if (config >= PERF_COUNT_HW_MAX) 158 return -EINVAL; 159 160 if (!event_map) 161 return -ENOENT; 162 163 mapping = (*event_map)[config]; 164 return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; 165 } 166 167 static int 168 armpmu_map_raw_event(u32 raw_event_mask, u64 config) 169 { 170 return (int)(config & raw_event_mask); 171 } 172 173 int 174 armpmu_map_event(struct perf_event *event, 175 const unsigned (*event_map)[PERF_COUNT_HW_MAX], 176 const unsigned (*cache_map) 177 [PERF_COUNT_HW_CACHE_MAX] 178 [PERF_COUNT_HW_CACHE_OP_MAX] 179 [PERF_COUNT_HW_CACHE_RESULT_MAX], 180 u32 raw_event_mask) 181 { 182 u64 config = event->attr.config; 183 int type = event->attr.type; 184 185 if (type == event->pmu->type) 186 return armpmu_map_raw_event(raw_event_mask, config); 187 188 switch (type) { 189 case PERF_TYPE_HARDWARE: 190 return armpmu_map_hw_event(event_map, config); 191 case PERF_TYPE_HW_CACHE: 192 return armpmu_map_cache_event(cache_map, config); 193 case PERF_TYPE_RAW: 194 return armpmu_map_raw_event(raw_event_mask, config); 195 } 196 197 return -ENOENT; 198 } 199 200 int armpmu_event_set_period(struct perf_event *event) 201 { 202 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 203 struct hw_perf_event *hwc = &event->hw; 204 s64 left = local64_read(&hwc->period_left); 205 s64 period = hwc->sample_period; 206 u64 max_period; 207 int ret = 0; 208 209 max_period = arm_pmu_event_max_period(event); 210 if (unlikely(left <= -period)) { 211 left = period; 212 local64_set(&hwc->period_left, left); 213 hwc->last_period = period; 214 ret = 1; 215 } 216 217 if (unlikely(left <= 0)) { 218 left += period; 219 local64_set(&hwc->period_left, left); 220 hwc->last_period = period; 221 ret = 1; 222 } 223 224 /* 225 * Limit the maximum period to prevent the counter value 226 * from overtaking the one we are about to program. In 227 * effect we are reducing max_period to account for 228 * interrupt latency (and we are being very conservative). 229 */ 230 if (left > (max_period >> 1)) 231 left = (max_period >> 1); 232 233 local64_set(&hwc->prev_count, (u64)-left); 234 235 armpmu->write_counter(event, (u64)(-left) & max_period); 236 237 perf_event_update_userpage(event); 238 239 return ret; 240 } 241 242 u64 armpmu_event_update(struct perf_event *event) 243 { 244 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 245 struct hw_perf_event *hwc = &event->hw; 246 u64 delta, prev_raw_count, new_raw_count; 247 u64 max_period = arm_pmu_event_max_period(event); 248 249 again: 250 prev_raw_count = local64_read(&hwc->prev_count); 251 new_raw_count = armpmu->read_counter(event); 252 253 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 254 new_raw_count) != prev_raw_count) 255 goto again; 256 257 delta = (new_raw_count - prev_raw_count) & max_period; 258 259 local64_add(delta, &event->count); 260 local64_sub(delta, &hwc->period_left); 261 262 return new_raw_count; 263 } 264 265 static void 266 armpmu_read(struct perf_event *event) 267 { 268 armpmu_event_update(event); 269 } 270 271 static void 272 armpmu_stop(struct perf_event *event, int flags) 273 { 274 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 275 struct hw_perf_event *hwc = &event->hw; 276 277 /* 278 * ARM pmu always has to update the counter, so ignore 279 * PERF_EF_UPDATE, see comments in armpmu_start(). 280 */ 281 if (!(hwc->state & PERF_HES_STOPPED)) { 282 armpmu->disable(event); 283 armpmu_event_update(event); 284 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 285 } 286 } 287 288 static void armpmu_start(struct perf_event *event, int flags) 289 { 290 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 291 struct hw_perf_event *hwc = &event->hw; 292 293 /* 294 * ARM pmu always has to reprogram the period, so ignore 295 * PERF_EF_RELOAD, see the comment below. 296 */ 297 if (flags & PERF_EF_RELOAD) 298 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 299 300 hwc->state = 0; 301 /* 302 * Set the period again. Some counters can't be stopped, so when we 303 * were stopped we simply disabled the IRQ source and the counter 304 * may have been left counting. If we don't do this step then we may 305 * get an interrupt too soon or *way* too late if the overflow has 306 * happened since disabling. 307 */ 308 armpmu_event_set_period(event); 309 armpmu->enable(event); 310 } 311 312 static void 313 armpmu_del(struct perf_event *event, int flags) 314 { 315 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 316 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 317 struct hw_perf_event *hwc = &event->hw; 318 int idx = hwc->idx; 319 320 armpmu_stop(event, PERF_EF_UPDATE); 321 hw_events->events[idx] = NULL; 322 armpmu->clear_event_idx(hw_events, event); 323 perf_event_update_userpage(event); 324 /* Clear the allocated counter */ 325 hwc->idx = -1; 326 } 327 328 static int 329 armpmu_add(struct perf_event *event, int flags) 330 { 331 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 332 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 333 struct hw_perf_event *hwc = &event->hw; 334 int idx; 335 336 /* An event following a process won't be stopped earlier */ 337 if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 338 return -ENOENT; 339 340 /* If we don't have a space for the counter then finish early. */ 341 idx = armpmu->get_event_idx(hw_events, event); 342 if (idx < 0) 343 return idx; 344 345 /* The newly-allocated counter should be empty */ 346 WARN_ON_ONCE(hw_events->events[idx]); 347 348 event->hw.idx = idx; 349 hw_events->events[idx] = event; 350 351 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 352 if (flags & PERF_EF_START) 353 armpmu_start(event, PERF_EF_RELOAD); 354 355 /* Propagate our changes to the userspace mapping. */ 356 perf_event_update_userpage(event); 357 358 return 0; 359 } 360 361 static int 362 validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, 363 struct perf_event *event) 364 { 365 struct arm_pmu *armpmu; 366 367 if (is_software_event(event)) 368 return 1; 369 370 /* 371 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The 372 * core perf code won't check that the pmu->ctx == leader->ctx 373 * until after pmu->event_init(event). 374 */ 375 if (event->pmu != pmu) 376 return 0; 377 378 if (event->state < PERF_EVENT_STATE_OFF) 379 return 1; 380 381 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 382 return 1; 383 384 armpmu = to_arm_pmu(event->pmu); 385 return armpmu->get_event_idx(hw_events, event) >= 0; 386 } 387 388 static int 389 validate_group(struct perf_event *event) 390 { 391 struct perf_event *sibling, *leader = event->group_leader; 392 struct pmu_hw_events fake_pmu; 393 394 /* 395 * Initialise the fake PMU. We only need to populate the 396 * used_mask for the purposes of validation. 397 */ 398 memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask)); 399 400 if (!validate_event(event->pmu, &fake_pmu, leader)) 401 return -EINVAL; 402 403 if (event == leader) 404 return 0; 405 406 for_each_sibling_event(sibling, leader) { 407 if (!validate_event(event->pmu, &fake_pmu, sibling)) 408 return -EINVAL; 409 } 410 411 if (!validate_event(event->pmu, &fake_pmu, event)) 412 return -EINVAL; 413 414 return 0; 415 } 416 417 static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) 418 { 419 struct arm_pmu *armpmu; 420 int ret; 421 u64 start_clock, finish_clock; 422 423 /* 424 * we request the IRQ with a (possibly percpu) struct arm_pmu**, but 425 * the handlers expect a struct arm_pmu*. The percpu_irq framework will 426 * do any necessary shifting, we just need to perform the first 427 * dereference. 428 */ 429 armpmu = *(void **)dev; 430 if (WARN_ON_ONCE(!armpmu)) 431 return IRQ_NONE; 432 433 start_clock = sched_clock(); 434 ret = armpmu->handle_irq(armpmu); 435 finish_clock = sched_clock(); 436 437 perf_sample_event_took(finish_clock - start_clock); 438 return ret; 439 } 440 441 static int 442 __hw_perf_event_init(struct perf_event *event) 443 { 444 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 445 struct hw_perf_event *hwc = &event->hw; 446 int mapping, ret; 447 448 hwc->flags = 0; 449 mapping = armpmu->map_event(event); 450 451 if (mapping < 0) { 452 pr_debug("event %x:%llx not supported\n", event->attr.type, 453 event->attr.config); 454 return mapping; 455 } 456 457 /* 458 * We don't assign an index until we actually place the event onto 459 * hardware. Use -1 to signify that we haven't decided where to put it 460 * yet. For SMP systems, each core has it's own PMU so we can't do any 461 * clever allocation or constraints checking at this point. 462 */ 463 hwc->idx = -1; 464 hwc->config_base = 0; 465 hwc->config = 0; 466 hwc->event_base = 0; 467 468 /* 469 * Check whether we need to exclude the counter from certain modes. 470 */ 471 if (armpmu->set_event_filter) { 472 ret = armpmu->set_event_filter(hwc, &event->attr); 473 if (ret) 474 return ret; 475 } 476 477 /* 478 * Store the event encoding into the config_base field. 479 */ 480 hwc->config_base |= (unsigned long)mapping; 481 482 if (!is_sampling_event(event)) { 483 /* 484 * For non-sampling runs, limit the sample_period to half 485 * of the counter width. That way, the new counter value 486 * is far less likely to overtake the previous one unless 487 * you have some serious IRQ latency issues. 488 */ 489 hwc->sample_period = arm_pmu_event_max_period(event) >> 1; 490 hwc->last_period = hwc->sample_period; 491 local64_set(&hwc->period_left, hwc->sample_period); 492 } 493 494 return validate_group(event); 495 } 496 497 static int armpmu_event_init(struct perf_event *event) 498 { 499 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 500 501 /* 502 * Reject CPU-affine events for CPUs that are of a different class to 503 * that which this PMU handles. Process-following events (where 504 * event->cpu == -1) can be migrated between CPUs, and thus we have to 505 * reject them later (in armpmu_add) if they're scheduled on a 506 * different class of CPU. 507 */ 508 if (event->cpu != -1 && 509 !cpumask_test_cpu(event->cpu, &armpmu->supported_cpus)) 510 return -ENOENT; 511 512 /* does not support taken branch sampling */ 513 if (has_branch_stack(event)) 514 return -EOPNOTSUPP; 515 516 return __hw_perf_event_init(event); 517 } 518 519 static void armpmu_enable(struct pmu *pmu) 520 { 521 struct arm_pmu *armpmu = to_arm_pmu(pmu); 522 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 523 bool enabled = !bitmap_empty(hw_events->used_mask, ARMPMU_MAX_HWEVENTS); 524 525 /* For task-bound events we may be called on other CPUs */ 526 if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 527 return; 528 529 if (enabled) 530 armpmu->start(armpmu); 531 } 532 533 static void armpmu_disable(struct pmu *pmu) 534 { 535 struct arm_pmu *armpmu = to_arm_pmu(pmu); 536 537 /* For task-bound events we may be called on other CPUs */ 538 if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 539 return; 540 541 armpmu->stop(armpmu); 542 } 543 544 /* 545 * In heterogeneous systems, events are specific to a particular 546 * microarchitecture, and aren't suitable for another. Thus, only match CPUs of 547 * the same microarchitecture. 548 */ 549 static bool armpmu_filter(struct pmu *pmu, int cpu) 550 { 551 struct arm_pmu *armpmu = to_arm_pmu(pmu); 552 return !cpumask_test_cpu(cpu, &armpmu->supported_cpus); 553 } 554 555 static ssize_t cpus_show(struct device *dev, 556 struct device_attribute *attr, char *buf) 557 { 558 struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev)); 559 return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus); 560 } 561 562 static DEVICE_ATTR_RO(cpus); 563 564 static struct attribute *armpmu_common_attrs[] = { 565 &dev_attr_cpus.attr, 566 NULL, 567 }; 568 569 static const struct attribute_group armpmu_common_attr_group = { 570 .attrs = armpmu_common_attrs, 571 }; 572 573 static int armpmu_count_irq_users(const int irq) 574 { 575 int cpu, count = 0; 576 577 for_each_possible_cpu(cpu) { 578 if (per_cpu(cpu_irq, cpu) == irq) 579 count++; 580 } 581 582 return count; 583 } 584 585 static const struct pmu_irq_ops *armpmu_find_irq_ops(int irq) 586 { 587 const struct pmu_irq_ops *ops = NULL; 588 int cpu; 589 590 for_each_possible_cpu(cpu) { 591 if (per_cpu(cpu_irq, cpu) != irq) 592 continue; 593 594 ops = per_cpu(cpu_irq_ops, cpu); 595 if (ops) 596 break; 597 } 598 599 return ops; 600 } 601 602 void armpmu_free_irq(int irq, int cpu) 603 { 604 if (per_cpu(cpu_irq, cpu) == 0) 605 return; 606 if (WARN_ON(irq != per_cpu(cpu_irq, cpu))) 607 return; 608 609 per_cpu(cpu_irq_ops, cpu)->free_pmuirq(irq, cpu, &cpu_armpmu); 610 611 per_cpu(cpu_irq, cpu) = 0; 612 per_cpu(cpu_irq_ops, cpu) = NULL; 613 } 614 615 int armpmu_request_irq(int irq, int cpu) 616 { 617 int err = 0; 618 const irq_handler_t handler = armpmu_dispatch_irq; 619 const struct pmu_irq_ops *irq_ops; 620 621 if (!irq) 622 return 0; 623 624 if (!irq_is_percpu_devid(irq)) { 625 unsigned long irq_flags; 626 627 err = irq_force_affinity(irq, cpumask_of(cpu)); 628 629 if (err && num_possible_cpus() > 1) { 630 pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n", 631 irq, cpu); 632 goto err_out; 633 } 634 635 irq_flags = IRQF_PERCPU | 636 IRQF_NOBALANCING | IRQF_NO_AUTOEN | 637 IRQF_NO_THREAD; 638 639 err = request_nmi(irq, handler, irq_flags, "arm-pmu", 640 per_cpu_ptr(&cpu_armpmu, cpu)); 641 642 /* If cannot get an NMI, get a normal interrupt */ 643 if (err) { 644 err = request_irq(irq, handler, irq_flags, "arm-pmu", 645 per_cpu_ptr(&cpu_armpmu, cpu)); 646 irq_ops = &pmuirq_ops; 647 } else { 648 has_nmi = true; 649 irq_ops = &pmunmi_ops; 650 } 651 } else if (armpmu_count_irq_users(irq) == 0) { 652 err = request_percpu_nmi(irq, handler, "arm-pmu", &cpu_armpmu); 653 654 /* If cannot get an NMI, get a normal interrupt */ 655 if (err) { 656 err = request_percpu_irq(irq, handler, "arm-pmu", 657 &cpu_armpmu); 658 irq_ops = &percpu_pmuirq_ops; 659 } else { 660 has_nmi = true; 661 irq_ops = &percpu_pmunmi_ops; 662 } 663 } else { 664 /* Per cpudevid irq was already requested by another CPU */ 665 irq_ops = armpmu_find_irq_ops(irq); 666 667 if (WARN_ON(!irq_ops)) 668 err = -EINVAL; 669 } 670 671 if (err) 672 goto err_out; 673 674 per_cpu(cpu_irq, cpu) = irq; 675 per_cpu(cpu_irq_ops, cpu) = irq_ops; 676 return 0; 677 678 err_out: 679 pr_err("unable to request IRQ%d for ARM PMU counters\n", irq); 680 return err; 681 } 682 683 static int armpmu_get_cpu_irq(struct arm_pmu *pmu, int cpu) 684 { 685 struct pmu_hw_events __percpu *hw_events = pmu->hw_events; 686 return per_cpu(hw_events->irq, cpu); 687 } 688 689 bool arm_pmu_irq_is_nmi(void) 690 { 691 return has_nmi; 692 } 693 694 /* 695 * PMU hardware loses all context when a CPU goes offline. 696 * When a CPU is hotplugged back in, since some hardware registers are 697 * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading 698 * junk values out of them. 699 */ 700 static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) 701 { 702 struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); 703 int irq; 704 705 if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) 706 return 0; 707 if (pmu->reset) 708 pmu->reset(pmu); 709 710 per_cpu(cpu_armpmu, cpu) = pmu; 711 712 irq = armpmu_get_cpu_irq(pmu, cpu); 713 if (irq) 714 per_cpu(cpu_irq_ops, cpu)->enable_pmuirq(irq); 715 716 return 0; 717 } 718 719 static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) 720 { 721 struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); 722 int irq; 723 724 if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) 725 return 0; 726 727 irq = armpmu_get_cpu_irq(pmu, cpu); 728 if (irq) 729 per_cpu(cpu_irq_ops, cpu)->disable_pmuirq(irq); 730 731 per_cpu(cpu_armpmu, cpu) = NULL; 732 733 return 0; 734 } 735 736 #ifdef CONFIG_CPU_PM 737 static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd) 738 { 739 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 740 struct perf_event *event; 741 int idx; 742 743 for_each_set_bit(idx, armpmu->cntr_mask, ARMPMU_MAX_HWEVENTS) { 744 event = hw_events->events[idx]; 745 if (!event) 746 continue; 747 748 switch (cmd) { 749 case CPU_PM_ENTER: 750 /* 751 * Stop and update the counter 752 */ 753 armpmu_stop(event, PERF_EF_UPDATE); 754 break; 755 case CPU_PM_EXIT: 756 case CPU_PM_ENTER_FAILED: 757 /* 758 * Restore and enable the counter. 759 */ 760 armpmu_start(event, PERF_EF_RELOAD); 761 break; 762 default: 763 break; 764 } 765 } 766 } 767 768 static int cpu_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, 769 void *v) 770 { 771 struct arm_pmu *armpmu = container_of(b, struct arm_pmu, cpu_pm_nb); 772 struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 773 bool enabled = !bitmap_empty(hw_events->used_mask, ARMPMU_MAX_HWEVENTS); 774 775 if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 776 return NOTIFY_DONE; 777 778 /* 779 * Always reset the PMU registers on power-up even if 780 * there are no events running. 781 */ 782 if (cmd == CPU_PM_EXIT && armpmu->reset) 783 armpmu->reset(armpmu); 784 785 if (!enabled) 786 return NOTIFY_OK; 787 788 switch (cmd) { 789 case CPU_PM_ENTER: 790 armpmu->stop(armpmu); 791 cpu_pm_pmu_setup(armpmu, cmd); 792 break; 793 case CPU_PM_EXIT: 794 case CPU_PM_ENTER_FAILED: 795 cpu_pm_pmu_setup(armpmu, cmd); 796 armpmu->start(armpmu); 797 break; 798 default: 799 return NOTIFY_DONE; 800 } 801 802 return NOTIFY_OK; 803 } 804 805 static int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) 806 { 807 cpu_pmu->cpu_pm_nb.notifier_call = cpu_pm_pmu_notify; 808 return cpu_pm_register_notifier(&cpu_pmu->cpu_pm_nb); 809 } 810 811 static void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) 812 { 813 cpu_pm_unregister_notifier(&cpu_pmu->cpu_pm_nb); 814 } 815 #else 816 static inline int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) { return 0; } 817 static inline void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) { } 818 #endif 819 820 static int cpu_pmu_init(struct arm_pmu *cpu_pmu) 821 { 822 int err; 823 824 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_STARTING, 825 &cpu_pmu->node); 826 if (err) 827 goto out; 828 829 err = cpu_pm_pmu_register(cpu_pmu); 830 if (err) 831 goto out_unregister; 832 833 return 0; 834 835 out_unregister: 836 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, 837 &cpu_pmu->node); 838 out: 839 return err; 840 } 841 842 static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu) 843 { 844 cpu_pm_pmu_unregister(cpu_pmu); 845 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, 846 &cpu_pmu->node); 847 } 848 849 struct arm_pmu *armpmu_alloc(void) 850 { 851 struct arm_pmu *pmu; 852 int cpu; 853 854 pmu = kzalloc(sizeof(*pmu), GFP_KERNEL); 855 if (!pmu) 856 goto out; 857 858 pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, GFP_KERNEL); 859 if (!pmu->hw_events) { 860 pr_info("failed to allocate per-cpu PMU data.\n"); 861 goto out_free_pmu; 862 } 863 864 pmu->pmu = (struct pmu) { 865 .pmu_enable = armpmu_enable, 866 .pmu_disable = armpmu_disable, 867 .event_init = armpmu_event_init, 868 .add = armpmu_add, 869 .del = armpmu_del, 870 .start = armpmu_start, 871 .stop = armpmu_stop, 872 .read = armpmu_read, 873 .filter = armpmu_filter, 874 .attr_groups = pmu->attr_groups, 875 /* 876 * This is a CPU PMU potentially in a heterogeneous 877 * configuration (e.g. big.LITTLE) so 878 * PERF_PMU_CAP_EXTENDED_HW_TYPE is required to open 879 * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE events on a 880 * specific PMU. 881 */ 882 .capabilities = PERF_PMU_CAP_EXTENDED_REGS | 883 PERF_PMU_CAP_EXTENDED_HW_TYPE, 884 }; 885 886 pmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] = 887 &armpmu_common_attr_group; 888 889 for_each_possible_cpu(cpu) { 890 struct pmu_hw_events *events; 891 892 events = per_cpu_ptr(pmu->hw_events, cpu); 893 events->percpu_pmu = pmu; 894 } 895 896 return pmu; 897 898 out_free_pmu: 899 kfree(pmu); 900 out: 901 return NULL; 902 } 903 904 void armpmu_free(struct arm_pmu *pmu) 905 { 906 free_percpu(pmu->hw_events); 907 kfree(pmu); 908 } 909 910 int armpmu_register(struct arm_pmu *pmu) 911 { 912 int ret; 913 914 ret = cpu_pmu_init(pmu); 915 if (ret) 916 return ret; 917 918 if (!pmu->set_event_filter) 919 pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; 920 921 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); 922 if (ret) 923 goto out_destroy; 924 925 pr_info("enabled with %s PMU driver, %d (%*pb) counters available%s\n", 926 pmu->name, bitmap_weight(pmu->cntr_mask, ARMPMU_MAX_HWEVENTS), 927 ARMPMU_MAX_HWEVENTS, &pmu->cntr_mask, 928 has_nmi ? ", using NMIs" : ""); 929 930 kvm_host_pmu_init(pmu); 931 932 return 0; 933 934 out_destroy: 935 cpu_pmu_destroy(pmu); 936 return ret; 937 } 938 939 static int arm_pmu_hp_init(void) 940 { 941 int ret; 942 943 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_STARTING, 944 "perf/arm/pmu:starting", 945 arm_perf_starting_cpu, 946 arm_perf_teardown_cpu); 947 if (ret) 948 pr_err("CPU hotplug notifier for ARM PMU could not be registered: %d\n", 949 ret); 950 return ret; 951 } 952 subsys_initcall(arm_pmu_hp_init); 953