Lines Matching +full:ignore +full:- +full:power +full:- +full:on +full:- +full:sel
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2016-2020 Arm Limited
3 // CMN-600 Coherent Mesh Network PMU driver
11 #include <linux/io-64-nonatomic-lo-hi.h>
36 #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4)
77 /* Technically this is 4 bits wide on DNs, but we only use 2 there anyway */
128 /* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */
130 #define CMN_DT_PMEVCNT(dtc, n) ((dtc)->pmu_base + _CMN_DT_CNT_REG(n))
131 #define CMN_DT_PMCCNTR(dtc) ((dtc)->pmu_base + 0x40)
133 #define CMN_DT_PMEVCNTSR(dtc, n) ((dtc)->pmu_base + 0x50 + _CMN_DT_CNT_REG(n))
134 #define CMN_DT_PMCCNTRSR(dtc) ((dtc)->pmu_base + 0x90)
136 #define CMN_DT_PMCR(dtc) ((dtc)->pmu_base + 0x100)
141 #define CMN_DT_PMOVSR(dtc) ((dtc)->pmu_base + 0x118)
142 #define CMN_DT_PMOVSR_CLR(dtc) ((dtc)->pmu_base + 0x120)
144 #define CMN_DT_PMSSR(dtc) ((dtc)->pmu_base + 0x128)
147 #define CMN_DT_PMSRR(dtc) ((dtc)->pmu_base + 0x130)
158 /* Similarly for the 40-bit cycle counter */
169 #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)
170 #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)
171 #define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)
172 #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
173 #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
183 #define CMN_EVENT_WP_COMBINE(event) FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)
184 #define CMN_EVENT_WP_DEV_SEL(event) FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)
185 #define CMN_EVENT_WP_CHN_SEL(event) FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)
186 #define CMN_EVENT_WP_GRP(event) FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)
187 #define CMN_EVENT_WP_EXCLUSIVE(event) FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)
188 #define CMN_EVENT_WP_VAL(event) FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)
189 #define CMN_EVENT_WP_MASK(event) FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)
191 /* Made-up event IDs for watchpoint direction */
204 CMN_ANY = -1,
205 NOT_CMN600 = -2,
218 /* CMN-600 r0px shouldn't exist in silicon, thankfully */
273 SEL_NONE = -1,
292 /* DN/HN-F/CXHA */
374 return fls((cmn->mesh_x - 1) | (cmn->mesh_y - 1)); in arm_cmn_xyidbits()
381 nid.dev = dn->id & ((1U << dn->deviceid_bits) - 1); in arm_cmn_nid()
382 nid.port = (dn->id >> dn->deviceid_bits) & ((1U << dn->portid_bits) - 1); in arm_cmn_nid()
389 int id = dn->id >> (dn->portid_bits + dn->deviceid_bits); in arm_cmn_node_to_xp()
392 int y = id & ((1U << bits) - 1); in arm_cmn_node_to_xp()
394 return cmn->xps + cmn->mesh_x * y + x; in arm_cmn_node_to_xp()
401 for (dn = cmn->dns; dn->type; dn++) in arm_cmn_node()
402 if (dn->type == type) in arm_cmn_node()
409 switch (cmn->part) { in arm_cmn_model()
427 if (cmn->part == PART_CMN_S3) { in arm_cmn_pmu_offset()
428 if (dn->type == CMN_TYPE_XP) in arm_cmn_pmu_offset()
438 int offset = CMN_MXP__CONNECT_INFO(port) - arm_cmn_pmu_offset(cmn, xp); in arm_cmn_device_connect_info()
441 if (cmn->part == PART_CMN600 || cmn->part == PART_CMN650) in arm_cmn_device_connect_info()
444 * CI-700 may have extra ports, but still has the in arm_cmn_device_connect_info()
447 if (cmn->part == PART_CI700) in arm_cmn_device_connect_info()
451 return readl_relaxed(xp->pmu_base + offset); in arm_cmn_device_connect_info()
461 case 0x01: return " RN-I |"; in arm_cmn_device_type()
462 case 0x02: return " RN-D |"; in arm_cmn_device_type()
463 case 0x04: return " RN-F_B |"; in arm_cmn_device_type()
464 case 0x05: return "RN-F_B_E|"; in arm_cmn_device_type()
465 case 0x06: return " RN-F_A |"; in arm_cmn_device_type()
466 case 0x07: return "RN-F_A_E|"; in arm_cmn_device_type()
467 case 0x08: return " HN-T |"; in arm_cmn_device_type()
468 case 0x09: return " HN-I |"; in arm_cmn_device_type()
469 case 0x0a: return " HN-D |"; in arm_cmn_device_type()
470 case 0x0b: return " HN-P |"; in arm_cmn_device_type()
471 case 0x0c: return " SN-F |"; in arm_cmn_device_type()
473 case 0x0e: return " HN-F |"; in arm_cmn_device_type()
474 case 0x0f: return " SN-F_E |"; in arm_cmn_device_type()
475 case 0x10: return " SN-F_D |"; in arm_cmn_device_type()
479 case 0x14: return " RN-F_D |"; in arm_cmn_device_type()
480 case 0x15: return "RN-F_D_E|"; in arm_cmn_device_type()
481 case 0x16: return " RN-F_C |"; in arm_cmn_device_type()
482 case 0x17: return "RN-F_C_E|"; in arm_cmn_device_type()
483 case 0x18: return " RN-F_E |"; in arm_cmn_device_type()
484 case 0x19: return "RN-F_E_E|"; in arm_cmn_device_type()
485 case 0x1a: return " HN-S |"; in arm_cmn_device_type()
488 case 0x1d: return " HN-V |"; in arm_cmn_device_type()
490 case 0x20: return " RN-F_F |"; in arm_cmn_device_type()
491 case 0x21: return "RN-F_F_E|"; in arm_cmn_device_type()
492 case 0x22: return " SN-F_F |"; in arm_cmn_device_type()
499 struct arm_cmn *cmn = s->private; in arm_cmn_show_logid()
501 u16 id = xp->id | d | (p << xp->deviceid_bits); in arm_cmn_show_logid()
503 for (dn = cmn->dns; dn->type; dn++) { in arm_cmn_show_logid()
504 int pad = dn->logid < 10; in arm_cmn_show_logid()
506 if (dn->type == CMN_TYPE_XP) in arm_cmn_show_logid()
508 /* Ignore the extra components that will overlap on some ports */ in arm_cmn_show_logid()
509 if (dn->type < CMN_TYPE_HNI) in arm_cmn_show_logid()
512 if (dn->id != id) in arm_cmn_show_logid()
515 seq_printf(s, " %*c#%-*d |", pad + 1, ' ', 3 - pad, dn->logid); in arm_cmn_show_logid()
523 struct arm_cmn *cmn = s->private; in arm_cmn_map_show()
524 int x, y, p, pmax = fls(cmn->ports_used); in arm_cmn_map_show()
527 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
528 seq_printf(s, " %-2d ", x); in arm_cmn_map_show()
530 y = cmn->mesh_y; in arm_cmn_map_show()
531 while (y--) { in arm_cmn_map_show()
532 int xp_base = cmn->mesh_x * y; in arm_cmn_map_show()
533 struct arm_cmn_node *xp = cmn->xps + xp_base; in arm_cmn_map_show()
536 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
537 seq_puts(s, "--------+"); in arm_cmn_map_show()
539 seq_printf(s, "\n%-2d |", y); in arm_cmn_map_show()
540 for (x = 0; x < cmn->mesh_x; x++) { in arm_cmn_map_show()
543 seq_printf(s, " XP #%-3d|", xp_base + x); in arm_cmn_map_show()
547 for (x = 0; x < cmn->mesh_x; x++) { in arm_cmn_map_show()
556 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
561 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
564 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
567 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
570 seq_puts(s, "\n-----+"); in arm_cmn_map_show()
572 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
573 seq_puts(s, "--------+"); in arm_cmn_map_show()
584 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id); in arm_cmn_debugfs_init()
588 cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops); in arm_cmn_debugfs_init()
613 for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)
615 /* @i is the DTC number, @idx is the counter index on that DTC */
617 for (int i = 0, idx; i < CMN_MAX_DTCS; i++) if ((idx = hw->dtc_idx[i]) >= 0)
621 return (struct arm_cmn_hw_event *)&event->hw; in to_cmn_hw()
679 if (eattr->type == CMN_TYPE_DTC) in arm_cmn_event_show()
680 return sysfs_emit(buf, "type=0x%x\n", eattr->type); in arm_cmn_event_show()
682 if (eattr->type == CMN_TYPE_WP) in arm_cmn_event_show()
685 eattr->type, eattr->eventid); in arm_cmn_event_show()
687 if (eattr->fsel > SEL_NONE) in arm_cmn_event_show()
689 eattr->type, eattr->eventid, eattr->occupid); in arm_cmn_event_show()
691 return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, in arm_cmn_event_show()
692 eattr->eventid); in arm_cmn_event_show()
707 if (!(eattr->model & arm_cmn_model(cmn))) in arm_cmn_event_attr_is_visible()
710 type = eattr->type; in arm_cmn_event_attr_is_visible()
711 eventid = eattr->eventid; in arm_cmn_event_attr_is_visible()
715 return attr->mode; in arm_cmn_event_attr_is_visible()
722 if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3))) in arm_cmn_event_attr_is_visible()
725 if (chan == 4 && cmn->part == PART_CMN600) in arm_cmn_event_attr_is_visible()
728 if ((chan == 5 && cmn->rsp_vc_num < 2) || in arm_cmn_event_attr_is_visible()
729 (chan == 6 && cmn->dat_vc_num < 2) || in arm_cmn_event_attr_is_visible()
730 (chan == 7 && cmn->snp_vc_num < 2) || in arm_cmn_event_attr_is_visible()
731 (chan == 8 && cmn->req_vc_num < 2)) in arm_cmn_event_attr_is_visible()
735 /* Revision-specific differences */ in arm_cmn_event_attr_is_visible()
736 if (cmn->part == PART_CMN600) { in arm_cmn_event_attr_is_visible()
737 if (cmn->rev < REV_CMN600_R1P3) { in arm_cmn_event_attr_is_visible()
741 if (cmn->rev < REV_CMN600_R1P2) { in arm_cmn_event_attr_is_visible()
747 } else if (cmn->part == PART_CMN650) { in arm_cmn_event_attr_is_visible()
748 if (cmn->rev < REV_CMN650_R2P0 || cmn->rev == REV_CMN650_R1P2) { in arm_cmn_event_attr_is_visible()
756 } else if (cmn->part == PART_CMN700) { in arm_cmn_event_attr_is_visible()
757 if (cmn->rev < REV_CMN700_R2P0) { in arm_cmn_event_attr_is_visible()
765 if (cmn->rev < REV_CMN700_R1P0) { in arm_cmn_event_attr_is_visible()
774 return attr->mode; in arm_cmn_event_attr_is_visible()
897 * DVM node events conflict with HN-I events in the equivalent PMU
898 * slot, but our lazy short-cut of using the DTM counter index for
996 * HN-P events squat on top of the HN-I similarly to DVM events, except
1019 /* We treat watchpoints as a special made-up class of XP events */
1282 if (!fmt->config) in arm_cmn_format_show()
1283 return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field); in arm_cmn_format_show()
1285 return sysfs_emit(buf, "config%d:%*pbl\n", fmt->config, 64, &fmt->field); in arm_cmn_format_show()
1325 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu)); in arm_cmn_cpumask_show()
1336 return sysfs_emit(buf, "%03x%02x\n", cmn->part, cmn->rev); in arm_cmn_identifier_show()
1364 if (dtm->wp_event[wp_idx] >= 0) in arm_cmn_find_free_wp_idx()
1365 if (dtm->wp_event[++wp_idx] >= 0) in arm_cmn_find_free_wp_idx()
1366 return -ENOSPC; in arm_cmn_find_free_wp_idx()
1375 return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos); in arm_cmn_get_assigned_wp_idx()
1385 dtm->wp_event[wp_idx] = hw->dtc_idx[dtc]; in arm_cmn_claim_wp_idx()
1386 arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event)); in arm_cmn_claim_wp_idx()
1397 bool is_cmn600 = to_cmn(event->pmu)->part == PART_CMN600; in arm_cmn_wp_config()
1399 /* CMN-600 supports only primary and secondary matching groups */ in arm_cmn_wp_config()
1411 /* wp_combine is available only on WP0 and WP2 */ in arm_cmn_wp_config()
1420 if (!cmn->state) in arm_cmn_set_state()
1421 writel_relaxed(0, CMN_DT_PMCR(&cmn->dtc[0])); in arm_cmn_set_state()
1422 cmn->state |= state; in arm_cmn_set_state()
1427 cmn->state &= ~state; in arm_cmn_clear_state()
1428 if (!cmn->state) in arm_cmn_clear_state()
1430 CMN_DT_PMCR(&cmn->dtc[0])); in arm_cmn_clear_state()
1453 if (dtm != &cmn->dtms[dn->dtm]) { in arm_cmn_read_dtm()
1454 dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; in arm_cmn_read_dtm()
1455 reg = readq_relaxed(dtm->base + offset); in arm_cmn_read_dtm()
1457 dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_read_dtm()
1469 return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1); in arm_cmn_read_cc()
1478 return val - CMN_COUNTER_INIT; in arm_cmn_read_counter()
1483 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_init_counter()
1488 writel_relaxed(CMN_COUNTER_INIT, CMN_DT_PMEVCNT(&cmn->dtc[i], idx)); in arm_cmn_init_counter()
1489 cmn->dtc[i].counters[idx] = event; in arm_cmn_init_counter()
1493 local64_set(&event->hw.prev_count, count); in arm_cmn_init_counter()
1498 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_read()
1504 delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]); in arm_cmn_event_read()
1505 local64_add(delta, &event->count); in arm_cmn_event_read()
1509 prev = local64_xchg(&event->hw.prev_count, new); in arm_cmn_event_read()
1511 delta = new - prev; in arm_cmn_event_read()
1515 new = arm_cmn_read_counter(cmn->dtc + i, idx); in arm_cmn_event_read()
1519 local64_add(delta, &event->count); in arm_cmn_event_read()
1530 if (!dn->occupid[fsel].count) { in arm_cmn_set_event_sel_hi()
1531 dn->occupid[fsel].val = occupid; in arm_cmn_set_event_sel_hi()
1533 dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) | in arm_cmn_set_event_sel_hi()
1535 dn->occupid[SEL_SN_HOME_SEL].val) | in arm_cmn_set_event_sel_hi()
1537 dn->occupid[SEL_HBT_LBT_SEL].val) | in arm_cmn_set_event_sel_hi()
1539 dn->occupid[SEL_CLASS_OCCUP_ID].val) | in arm_cmn_set_event_sel_hi()
1541 dn->occupid[SEL_OCCUP1ID].val); in arm_cmn_set_event_sel_hi()
1542 writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4); in arm_cmn_set_event_sel_hi()
1543 } else if (dn->occupid[fsel].val != occupid) { in arm_cmn_set_event_sel_hi()
1544 return -EBUSY; in arm_cmn_set_event_sel_hi()
1546 dn->occupid[fsel].count++; in arm_cmn_set_event_sel_hi()
1554 dn->event_w[dtm_idx] = eventid; in arm_cmn_set_event_sel_lo()
1555 writeq_relaxed(le64_to_cpu(dn->event_sel_w), dn->pmu_base + CMN_PMU_EVENT_SEL); in arm_cmn_set_event_sel_lo()
1557 dn->event[dtm_idx] = eventid; in arm_cmn_set_event_sel_lo()
1558 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); in arm_cmn_set_event_sel_lo()
1564 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_start()
1571 struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; in arm_cmn_event_start()
1574 dtc->base + CMN_DT_DTC_CTL); in arm_cmn_event_start()
1576 dtc->cc_active = true; in arm_cmn_event_start()
1582 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); in arm_cmn_event_start()
1589 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_start()
1592 hw->wide_sel); in arm_cmn_event_start()
1598 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_stop()
1605 struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; in arm_cmn_event_stop()
1607 dtc->cc_active = false; in arm_cmn_event_stop()
1608 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); in arm_cmn_event_stop()
1611 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); in arm_cmn_event_stop()
1618 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_stop()
1620 arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel); in arm_cmn_event_stop()
1640 if (val->wp[dtm][wp_idx]) in arm_cmn_val_find_free_wp_config()
1641 if (val->wp[dtm][++wp_idx]) in arm_cmn_val_find_free_wp_config()
1642 return -ENOSPC; in arm_cmn_val_find_free_wp_config()
1660 val->cycles = true; in arm_cmn_val_add_event()
1665 val->dtc_count[dtc]++; in arm_cmn_val_add_event()
1668 int wp_idx, dtm = dn->dtm, sel = hw->filter_sel; in arm_cmn_val_add_event() local
1670 val->dtm_count[dtm]++; in arm_cmn_val_add_event()
1672 if (sel > SEL_NONE) in arm_cmn_val_add_event()
1673 val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1; in arm_cmn_val_add_event()
1679 val->wp[dtm][wp_idx] = 1; in arm_cmn_val_add_event()
1680 val->wp_combine[dtm][wp_idx >> 1] += !!CMN_EVENT_WP_COMBINE(event); in arm_cmn_val_add_event()
1688 struct perf_event *sibling, *leader = event->group_leader; in arm_cmn_validate_group()
1691 int i, ret = -EINVAL; in arm_cmn_validate_group()
1696 if (event->pmu != leader->pmu && !is_software_event(leader)) in arm_cmn_validate_group()
1697 return -EINVAL; in arm_cmn_validate_group()
1701 return -ENOMEM; in arm_cmn_validate_group()
1710 ret = val->cycles ? -EINVAL : 0; in arm_cmn_validate_group()
1715 if (val->dtc_count[dtc] == CMN_DT_NUM_COUNTERS) in arm_cmn_validate_group()
1719 int wp_idx, dtm = dn->dtm, sel = hw->filter_sel; in arm_cmn_validate_group() local
1721 if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) in arm_cmn_validate_group()
1724 if (sel > SEL_NONE && val->occupid[dtm][sel] && in arm_cmn_validate_group()
1725 val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1) in arm_cmn_validate_group()
1736 val->wp_combine[dtm][wp_idx >> 1] != !!CMN_EVENT_WP_COMBINE(event)) in arm_cmn_validate_group()
1753 for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) { in arm_cmn_filter_sel()
1755 if (e->model & model && e->type == type && e->eventid == eventid) in arm_cmn_filter_sel()
1756 return e->fsel; in arm_cmn_filter_sel()
1764 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_init()
1771 if (event->attr.type != event->pmu->type) in arm_cmn_event_init()
1772 return -ENOENT; in arm_cmn_event_init()
1774 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) in arm_cmn_event_init()
1775 return -EINVAL; in arm_cmn_event_init()
1777 event->cpu = cmn->cpu; in arm_cmn_event_init()
1778 if (event->cpu < 0) in arm_cmn_event_init()
1779 return -EINVAL; in arm_cmn_event_init()
1792 return -EINVAL; in arm_cmn_event_init()
1793 /* ...but the DTM may depend on which port we're watching */ in arm_cmn_event_init()
1794 if (cmn->multi_dtm) in arm_cmn_event_init()
1795 hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; in arm_cmn_event_init()
1797 (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) { in arm_cmn_event_init()
1798 hw->wide_sel = true; in arm_cmn_event_init()
1805 hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid); in arm_cmn_event_init()
1810 hw->dn = arm_cmn_node(cmn, type); in arm_cmn_event_init()
1811 if (!hw->dn) in arm_cmn_event_init()
1812 return -EINVAL; in arm_cmn_event_init()
1814 memset(hw->dtc_idx, -1, sizeof(hw->dtc_idx)); in arm_cmn_event_init()
1815 for (dn = hw->dn; dn->type == type; dn++) { in arm_cmn_event_init()
1816 if (bynodeid && dn->id != nodeid) { in arm_cmn_event_init()
1817 hw->dn++; in arm_cmn_event_init()
1820 hw->num_dns++; in arm_cmn_event_init()
1821 if (dn->dtc < 0) in arm_cmn_event_init()
1822 memset(hw->dtc_idx, 0, cmn->num_dtcs); in arm_cmn_event_init()
1824 hw->dtc_idx[dn->dtc] = 0; in arm_cmn_event_init()
1830 if (!hw->num_dns) { in arm_cmn_event_init()
1831 dev_dbg(cmn->dev, "invalid node 0x%x type 0x%x\n", nodeid, type); in arm_cmn_event_init()
1832 return -EINVAL; in arm_cmn_event_init()
1844 while (i--) { in arm_cmn_event_clear()
1845 struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; in arm_cmn_event_clear()
1846 unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_clear()
1851 dtm->wp_event[wp_idx] = -1; in arm_cmn_event_clear()
1854 if (hw->filter_sel > SEL_NONE) in arm_cmn_event_clear()
1855 hw->dn[i].occupid[hw->filter_sel].count--; in arm_cmn_event_clear()
1857 dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); in arm_cmn_event_clear()
1858 writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_event_clear()
1860 memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); in arm_cmn_event_clear()
1861 memset(hw->wp_idx, 0, sizeof(hw->wp_idx)); in arm_cmn_event_clear()
1864 cmn->dtc[j].counters[idx] = NULL; in arm_cmn_event_clear()
1869 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_add()
1876 while (cmn->dtc[i].cycles) in arm_cmn_event_add()
1877 if (++i == cmn->num_dtcs) in arm_cmn_event_add()
1878 return -ENOSPC; in arm_cmn_event_add()
1880 cmn->dtc[i].cycles = event; in arm_cmn_event_add()
1881 hw->dtc_idx[0] = i; in arm_cmn_event_add()
1890 if (cmn->part == PART_CMN600 && j > 0) { in arm_cmn_event_add()
1891 idx = hw->dtc_idx[0]; in arm_cmn_event_add()
1894 while (cmn->dtc[j].counters[idx]) in arm_cmn_event_add()
1896 return -ENOSPC; in arm_cmn_event_add()
1898 hw->dtc_idx[j] = idx; in arm_cmn_event_add()
1903 struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; in arm_cmn_event_add()
1904 unsigned int dtm_idx, shift, d = max_t(int, dn->dtc, 0); in arm_cmn_event_add()
1908 while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx)) in arm_cmn_event_add()
1924 tmp = dtm->wp_event[wp_idx ^ 1]; in arm_cmn_event_add()
1926 CMN_EVENT_WP_COMBINE(cmn->dtc[d].counters[tmp])) in arm_cmn_event_add()
1932 writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx)); in arm_cmn_event_add()
1936 if (cmn->multi_dtm) in arm_cmn_event_add()
1942 if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event))) in arm_cmn_event_add()
1946 arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); in arm_cmn_event_add()
1948 dtm->input_sel[dtm_idx] = input_sel; in arm_cmn_event_add()
1950 dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift); in arm_cmn_event_add()
1951 dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, hw->dtc_idx[d]) << shift; in arm_cmn_event_add()
1952 dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx); in arm_cmn_event_add()
1953 reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low; in arm_cmn_event_add()
1954 writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_event_add()
1967 return -ENOSPC; in arm_cmn_event_add()
1972 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_del()
1979 cmn->dtc[hw->dtc_idx[0]].cycles = NULL; in arm_cmn_event_del()
1981 arm_cmn_event_clear(cmn, event, hw->num_dns); in arm_cmn_event_del()
1988 * plus it seems they don't work properly on some hardware anyway :(
2010 perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu); in arm_cmn_migrate()
2011 for (i = 0; i < cmn->num_dtcs; i++) in arm_cmn_migrate()
2012 irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu)); in arm_cmn_migrate()
2013 cmn->cpu = cpu; in arm_cmn_migrate()
2022 node = dev_to_node(cmn->dev); in arm_cmn_pmu_online_cpu()
2023 if (cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node) in arm_cmn_pmu_online_cpu()
2035 if (cpu != cmn->cpu) in arm_cmn_pmu_offline_cpu()
2038 node = dev_to_node(cmn->dev); in arm_cmn_pmu_offline_cpu()
2063 if (WARN_ON(!dtc->counters[i])) in arm_cmn_handle_irq()
2066 local64_add(delta, &dtc->counters[i]->count); in arm_cmn_handle_irq()
2072 if (dtc->cc_active && !WARN_ON(!dtc->cycles)) { in arm_cmn_handle_irq()
2074 local64_add(delta, &dtc->cycles->count); in arm_cmn_handle_irq()
2080 if (!dtc->irq_friend) in arm_cmn_handle_irq()
2082 dtc += dtc->irq_friend; in arm_cmn_handle_irq()
2091 for (i = 0; i < cmn->num_dtcs; i++) { in arm_cmn_init_irqs()
2092 irq = cmn->dtc[i].irq; in arm_cmn_init_irqs()
2093 for (j = i; j--; ) { in arm_cmn_init_irqs()
2094 if (cmn->dtc[j].irq == irq) { in arm_cmn_init_irqs()
2095 cmn->dtc[j].irq_friend = i - j; in arm_cmn_init_irqs()
2099 err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq, in arm_cmn_init_irqs()
2101 dev_name(cmn->dev), &cmn->dtc[i]); in arm_cmn_init_irqs()
2105 err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); in arm_cmn_init_irqs()
2118 dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx); in arm_cmn_init_dtm()
2119 dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN; in arm_cmn_init_dtm()
2120 writeq_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_init_dtm()
2122 dtm->wp_event[i] = -1; in arm_cmn_init_dtm()
2123 writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i)); in arm_cmn_init_dtm()
2124 writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i)); in arm_cmn_init_dtm()
2130 struct arm_cmn_dtc *dtc = cmn->dtc + idx; in arm_cmn_init_dtc()
2132 dtc->pmu_base = dn->pmu_base; in arm_cmn_init_dtc()
2133 dtc->base = dtc->pmu_base - arm_cmn_pmu_offset(cmn, dn); in arm_cmn_init_dtc()
2134 dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx); in arm_cmn_init_dtc()
2135 if (dtc->irq < 0) in arm_cmn_init_dtc()
2136 return dtc->irq; in arm_cmn_init_dtc()
2138 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); in arm_cmn_init_dtc()
2151 cmp = dna->type - dnb->type; in arm_cmn_node_cmp()
2153 cmp = dna->logid - dnb->logid; in arm_cmn_node_cmp()
2162 cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL); in arm_cmn_init_dtcs()
2163 if (!cmn->dtc) in arm_cmn_init_dtcs()
2164 return -ENOMEM; in arm_cmn_init_dtcs()
2166 sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL); in arm_cmn_init_dtcs()
2168 cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP); in arm_cmn_init_dtcs()
2170 if (cmn->part == PART_CMN600 && cmn->num_dtcs > 1) { in arm_cmn_init_dtcs()
2173 for (int i = 0; i < cmn->num_dtcs; i++) in arm_cmn_init_dtcs()
2174 arm_cmn_node_to_xp(cmn, dn + i)->dtc = i; in arm_cmn_init_dtcs()
2177 for (dn = cmn->dns; dn->type; dn++) { in arm_cmn_init_dtcs()
2178 if (dn->type == CMN_TYPE_XP) in arm_cmn_init_dtcs()
2182 dn->dtc = xp->dtc; in arm_cmn_init_dtcs()
2183 dn->dtm = xp->dtm; in arm_cmn_init_dtcs()
2184 if (cmn->multi_dtm) in arm_cmn_init_dtcs()
2185 dn->dtm += arm_cmn_nid(dn).port / 2; in arm_cmn_init_dtcs()
2187 if (dn->type == CMN_TYPE_DTC) { in arm_cmn_init_dtcs()
2194 /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */ in arm_cmn_init_dtcs()
2195 if (dn->type == CMN_TYPE_RND) in arm_cmn_init_dtcs()
2196 dn->type = CMN_TYPE_RNI; in arm_cmn_init_dtcs()
2198 /* We split the RN-I off already, so let the CCLA part match CCLA events */ in arm_cmn_init_dtcs()
2199 if (dn->type == CMN_TYPE_CCLA_RNI) in arm_cmn_init_dtcs()
2200 dn->type = CMN_TYPE_CCLA; in arm_cmn_init_dtcs()
2212 if (cmn->part == PART_CMN650 || cmn->part == PART_CI700) in arm_cmn_dtc_domain()
2221 u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO); in arm_cmn_init_node_info()
2223 node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg); in arm_cmn_init_node_info()
2224 node->id = FIELD_GET(CMN_NI_NODE_ID, reg); in arm_cmn_init_node_info()
2225 node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg); in arm_cmn_init_node_info()
2227 node->pmu_base = cmn->base + offset + arm_cmn_pmu_offset(cmn, node); in arm_cmn_init_node_info()
2229 if (node->type == CMN_TYPE_CFG) in arm_cmn_init_node_info()
2231 else if (node->type == CMN_TYPE_XP) in arm_cmn_init_node_info()
2236 dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n", in arm_cmn_init_node_info()
2237 (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ', in arm_cmn_init_node_info()
2238 node->type, node->logid, offset); in arm_cmn_init_node_info()
2267 return -ENODEV; in arm_cmn_discover()
2269 cfg_region = cmn->base + rgn_offset; in arm_cmn_discover()
2274 if (cmn->part && cmn->part != part) in arm_cmn_discover()
2275 dev_warn(cmn->dev, in arm_cmn_discover()
2277 cmn->part, part); in arm_cmn_discover()
2278 cmn->part = part; in arm_cmn_discover()
2280 dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); in arm_cmn_discover()
2283 cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); in arm_cmn_discover()
2288 * it; however we also have no way to tell from Non-Secure whether any in arm_cmn_discover()
2293 dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n"); in arm_cmn_discover()
2294 return -ENODEV; in arm_cmn_discover()
2296 cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; in arm_cmn_discover()
2297 cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); in arm_cmn_discover()
2298 cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); in arm_cmn_discover()
2301 cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); in arm_cmn_discover()
2302 cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); in arm_cmn_discover()
2308 cmn->num_xps = child_count; in arm_cmn_discover()
2309 cmn->num_dns = cmn->num_xps; in arm_cmn_discover()
2312 for (i = 0; i < cmn->num_xps; i++) { in arm_cmn_discover()
2316 reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO); in arm_cmn_discover()
2317 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg); in arm_cmn_discover()
2323 * bound, account for double the number of non-XP nodes. in arm_cmn_discover()
2325 dn = devm_kcalloc(cmn->dev, cmn->num_dns * 2 - cmn->num_xps, in arm_cmn_discover()
2328 return -ENOMEM; in arm_cmn_discover()
2330 /* Initial safe upper bound on DTMs for any possible mesh layout */ in arm_cmn_discover()
2331 i = cmn->num_xps; in arm_cmn_discover()
2332 if (cmn->multi_dtm) in arm_cmn_discover()
2333 i += cmn->num_xps + 1; in arm_cmn_discover()
2334 dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL); in arm_cmn_discover()
2336 return -ENOMEM; in arm_cmn_discover()
2339 cmn->dns = dn; in arm_cmn_discover()
2340 cmn->dtms = dtm; in arm_cmn_discover()
2341 for (i = 0; i < cmn->num_xps; i++) { in arm_cmn_discover()
2342 void __iomem *xp_region = cmn->base + xp_offset[i]; in arm_cmn_discover()
2353 if (xp->id == (1 << 3)) in arm_cmn_discover()
2354 cmn->mesh_x = xp->logid; in arm_cmn_discover()
2356 if (cmn->part == PART_CMN600) in arm_cmn_discover()
2357 xp->dtc = -1; in arm_cmn_discover()
2359 xp->dtc = arm_cmn_dtc_domain(cmn, xp_region); in arm_cmn_discover()
2361 xp->dtm = dtm - cmn->dtms; in arm_cmn_discover()
2365 * unnecessary XP events easily, and also infer the per-XP in arm_cmn_discover()
2372 if (cmn->num_xps == 1) { in arm_cmn_discover()
2373 xp->portid_bits = 3; in arm_cmn_discover()
2374 xp->deviceid_bits = 2; in arm_cmn_discover()
2376 xp->portid_bits = 2; in arm_cmn_discover()
2377 xp->deviceid_bits = 1; in arm_cmn_discover()
2379 xp->portid_bits = 1; in arm_cmn_discover()
2380 xp->deviceid_bits = 2; in arm_cmn_discover()
2383 if (cmn->multi_dtm && (xp_ports > 0x3)) in arm_cmn_discover()
2385 if (cmn->multi_dtm && (xp_ports > 0xf)) in arm_cmn_discover()
2388 cmn->ports_used |= xp_ports; in arm_cmn_discover()
2398 * we haven't a clue how to power up arbitrary CHI requesters. in arm_cmn_discover()
2399 * As of CMN-600r1 these could only be RN-SAMs or CXLAs, in arm_cmn_discover()
2402 * but they don't go to regular XP DTMs, and they depend on in arm_cmn_discover()
2406 dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); in arm_cmn_discover()
2412 * A child offset of 0 can only occur on CMN-600; otherwise it in arm_cmn_discover()
2416 if (reg == 0 && cmn->part != PART_CMN600) { in arm_cmn_discover()
2417 dev_dbg(cmn->dev, "bogus child pointer?\n"); in arm_cmn_discover()
2422 dn->portid_bits = xp->portid_bits; in arm_cmn_discover()
2423 dn->deviceid_bits = xp->deviceid_bits; in arm_cmn_discover()
2425 switch (dn->type) { in arm_cmn_discover()
2427 cmn->num_dtcs++; in arm_cmn_discover()
2446 dn->pmu_base += CMN_CCLA_PMU_EVENT_SEL; in arm_cmn_discover()
2468 dn[1].type = arm_cmn_subtype(dn->type); in arm_cmn_discover()
2473 dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); in arm_cmn_discover()
2474 return -ENODEV; in arm_cmn_discover()
2480 cmn->num_dns = dn - cmn->dns; in arm_cmn_discover()
2482 /* Cheeky +1 to help terminate pointer-based iteration later */ in arm_cmn_discover()
2483 sz = (void *)(dn + 1) - (void *)cmn->dns; in arm_cmn_discover()
2484 dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL); in arm_cmn_discover()
2486 cmn->dns = dn; in arm_cmn_discover()
2488 sz = (void *)dtm - (void *)cmn->dtms; in arm_cmn_discover()
2489 dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL); in arm_cmn_discover()
2491 cmn->dtms = dtm; in arm_cmn_discover()
2497 if (!cmn->mesh_x) in arm_cmn_discover()
2498 cmn->mesh_x = cmn->num_xps; in arm_cmn_discover()
2499 cmn->mesh_y = cmn->num_xps / cmn->mesh_x; in arm_cmn_discover()
2502 if (cmn->num_xps == 1) in arm_cmn_discover()
2503 dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); in arm_cmn_discover()
2505 dev_dbg(cmn->dev, "periph_id part 0x%03x revision %d\n", cmn->part, cmn->rev); in arm_cmn_discover()
2506 reg = cmn->ports_used; in arm_cmn_discover()
2507 dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n", in arm_cmn_discover()
2508 cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), ®, in arm_cmn_discover()
2509 cmn->multi_dtm ? ", multi-DTM" : ""); in arm_cmn_discover()
2520 return -EINVAL; in arm_cmn600_acpi_probe()
2524 return -EINVAL; in arm_cmn600_acpi_probe()
2534 cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg)); in arm_cmn600_acpi_probe()
2535 if (!cmn->base) in arm_cmn600_acpi_probe()
2536 return -ENOMEM; in arm_cmn600_acpi_probe()
2538 return root->start - cfg->start; in arm_cmn600_acpi_probe()
2545 return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode; in arm_cmn600_of_probe()
2555 cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL); in arm_cmn_probe()
2557 return -ENOMEM; in arm_cmn_probe()
2559 cmn->dev = &pdev->dev; in arm_cmn_probe()
2560 cmn->part = (unsigned long)device_get_match_data(cmn->dev); in arm_cmn_probe()
2563 if (cmn->part == PART_CMN600 && has_acpi_companion(cmn->dev)) { in arm_cmn_probe()
2567 cmn->base = devm_platform_ioremap_resource(pdev, 0); in arm_cmn_probe()
2568 if (IS_ERR(cmn->base)) in arm_cmn_probe()
2569 return PTR_ERR(cmn->base); in arm_cmn_probe()
2570 if (cmn->part == PART_CMN600) in arm_cmn_probe()
2571 rootnode = arm_cmn600_of_probe(pdev->dev.of_node); in arm_cmn_probe()
2588 cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); in arm_cmn_probe()
2589 cmn->pmu = (struct pmu) { in arm_cmn_probe()
2591 .parent = cmn->dev, in arm_cmn_probe()
2609 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id); in arm_cmn_probe()
2611 return -ENOMEM; in arm_cmn_probe()
2613 err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_probe()
2617 err = perf_pmu_register(&cmn->pmu, name, -1); in arm_cmn_probe()
2619 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_probe()
2630 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); in arm_cmn_remove()
2632 perf_pmu_unregister(&cmn->pmu); in arm_cmn_remove()
2633 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_remove()
2634 debugfs_remove(cmn->debug); in arm_cmn_remove()
2639 { .compatible = "arm,cmn-600", .data = (void *)PART_CMN600 },
2640 { .compatible = "arm,cmn-650" },
2641 { .compatible = "arm,cmn-700" },
2642 { .compatible = "arm,cmn-s3" },
2643 { .compatible = "arm,ci-700" },
2661 .name = "arm-cmn",
2681 arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL); in arm_cmn_init()
2702 MODULE_DESCRIPTION("Arm CMN-600 PMU driver");