1 // SPDX-License-Identifier: GPL-2.0
3  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
11  * Cortex-A8 has up to 4 configurable performance counters and
13  * Cortex-A9 has up to 31 configurable performance counters and
55  *  - all (taken) branch instructions,
56  *  - instructions that explicitly write the PC,
57  *  - exception generating instructions.
82 /* ARMv7 Cortex-A8 specific event types */
88 /* ARMv7 Cortex-A9 specific event types */
93 /* ARMv7 Cortex-A5 specific event types */
97 /* ARMv7 Cortex-A15 specific event types */
113 /* ARMv7 Cortex-A12 specific event types */
152  * Cortex-A8 HW events mapping
205  * Cortex-A9 HW events mapping
249  * Cortex-A5 HW events mapping
295  * Cortex-A15 HW events mapping
344  * Cortex-A7 HW events mapping
393  * Cortex-A12 HW events mapping
535 PMU_FORMAT_ATTR(event, "config:0-7");
657 	(ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
660 #define	ARMV7_COUNTER_MASK	(ARMV7_MAX_COUNTERS - 1)
670 	(((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
673  * Per-CPU PMNC: config reg
680 #define ARMV7_PMNC_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
707 #define ARMV7_SDER_SUNIDEN	BIT(1) /* Permit non-invasive debug */
748 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_read_counter()
749 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_read_counter()
750 	int idx = hwc->idx;  in armv7pmu_read_counter()
768 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_write_counter()
769 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_write_counter()
770 	int idx = hwc->idx;  in armv7pmu_write_counter()
873 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_enable_event()
874 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_enable_event()
875 	int idx = hwc->idx;  in armv7pmu_enable_event()
898 	if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)  in armv7pmu_enable_event()
899 		armv7_pmnc_write_evtsel(idx, hwc->config_base);  in armv7pmu_enable_event()
914 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_disable_event()
915 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_disable_event()
916 	int idx = hwc->idx;  in armv7pmu_disable_event()
943 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv7pmu_handle_irq()
963 	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {  in armv7pmu_handle_irq()
964 		struct perf_event *event = cpuc->events[idx];  in armv7pmu_handle_irq()
978 		hwc = &event->hw;  in armv7pmu_handle_irq()
980 		perf_sample_data_init(&data, 0, hwc->last_period);  in armv7pmu_handle_irq()
985 			cpu_pmu->disable(event);  in armv7pmu_handle_irq()
1016 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv7pmu_get_event_idx()
1017 	struct hw_perf_event *hwc = &event->hw;  in armv7pmu_get_event_idx()
1018 	unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;  in armv7pmu_get_event_idx()
1022 		if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))  in armv7pmu_get_event_idx()
1023 			return -EAGAIN;  in armv7pmu_get_event_idx()
1032 	for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {  in armv7pmu_get_event_idx()
1033 		if (!test_and_set_bit(idx, cpuc->used_mask))  in armv7pmu_get_event_idx()
1038 	return -EAGAIN;  in armv7pmu_get_event_idx()
1044 	clear_bit(event->hw.idx, cpuc->used_mask);  in armv7pmu_clear_event_idx()
1055 	if (attr->exclude_idle) {  in armv7pmu_set_event_filter()
1057 		return -EOPNOTSUPP;  in armv7pmu_set_event_filter()
1059 	if (attr->exclude_user)  in armv7pmu_set_event_filter()
1061 	if (attr->exclude_kernel)  in armv7pmu_set_event_filter()
1063 	if (!attr->exclude_hv)  in armv7pmu_set_event_filter()
1070 	event->config_base = config_base;  in armv7pmu_set_event_filter()
1078 	u32 idx, nb_cnt = cpu_pmu->num_events, val;  in armv7pmu_reset()
1080 	if (cpu_pmu->secure_access) {  in armv7pmu_reset()
1152 	cpu_pmu->handle_irq	= armv7pmu_handle_irq;  in armv7pmu_init()
1153 	cpu_pmu->enable		= armv7pmu_enable_event;  in armv7pmu_init()
1154 	cpu_pmu->disable	= armv7pmu_disable_event;  in armv7pmu_init()
1155 	cpu_pmu->read_counter	= armv7pmu_read_counter;  in armv7pmu_init()
1156 	cpu_pmu->write_counter	= armv7pmu_write_counter;  in armv7pmu_init()
1157 	cpu_pmu->get_event_idx	= armv7pmu_get_event_idx;  in armv7pmu_init()
1158 	cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx;  in armv7pmu_init()
1159 	cpu_pmu->start		= armv7pmu_start;  in armv7pmu_init()
1160 	cpu_pmu->stop		= armv7pmu_stop;  in armv7pmu_init()
1161 	cpu_pmu->reset		= armv7pmu_reset;  in armv7pmu_init()
1177 	return smp_call_function_any(&arm_pmu->supported_cpus,  in armv7_probe_num_events()
1179 				     &arm_pmu->num_events, 1);  in armv7_probe_num_events()
1185 	cpu_pmu->name		= "armv7_cortex_a8";  in armv7_a8_pmu_init()
1186 	cpu_pmu->map_event	= armv7_a8_map_event;  in armv7_a8_pmu_init()
1187 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a8_pmu_init()
1189 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a8_pmu_init()
1197 	cpu_pmu->name		= "armv7_cortex_a9";  in armv7_a9_pmu_init()
1198 	cpu_pmu->map_event	= armv7_a9_map_event;  in armv7_a9_pmu_init()
1199 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a9_pmu_init()
1201 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a9_pmu_init()
1209 	cpu_pmu->name		= "armv7_cortex_a5";  in armv7_a5_pmu_init()
1210 	cpu_pmu->map_event	= armv7_a5_map_event;  in armv7_a5_pmu_init()
1211 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a5_pmu_init()
1213 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a5_pmu_init()
1221 	cpu_pmu->name		= "armv7_cortex_a15";  in armv7_a15_pmu_init()
1222 	cpu_pmu->map_event	= armv7_a15_map_event;  in armv7_a15_pmu_init()
1223 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in armv7_a15_pmu_init()
1224 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a15_pmu_init()
1226 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a15_pmu_init()
1234 	cpu_pmu->name		= "armv7_cortex_a7";  in armv7_a7_pmu_init()
1235 	cpu_pmu->map_event	= armv7_a7_map_event;  in armv7_a7_pmu_init()
1236 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in armv7_a7_pmu_init()
1237 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a7_pmu_init()
1239 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a7_pmu_init()
1247 	cpu_pmu->name		= "armv7_cortex_a12";  in armv7_a12_pmu_init()
1248 	cpu_pmu->map_event	= armv7_a12_map_event;  in armv7_a12_pmu_init()
1249 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in armv7_a12_pmu_init()
1250 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a12_pmu_init()
1252 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a12_pmu_init()
1260 	cpu_pmu->name = "armv7_cortex_a17";  in armv7_a17_pmu_init()
1261 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =  in armv7_a17_pmu_init()
1263 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =  in armv7_a17_pmu_init()
1272  *            +--------------------------------+
1274  *            +--------------------------------+
1276  *            +--------------------------------+
1278  *            +--------------------------------+
1280  *            +--------------------------------+
1285  *      hwc->config_base = 0xNRCCG
1417 	/* Mix in mode-exclusion bits */  in krait_evt_setup()
1477 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_disable_event()
1478 	int idx = hwc->idx;  in krait_pmu_disable_event()
1488 	if (hwc->config_base & KRAIT_EVENT_MASK)  in krait_pmu_disable_event()
1489 		krait_clearpmu(hwc->config_base);  in krait_pmu_disable_event()
1497 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_enable_event()
1498 	int idx = hwc->idx;  in krait_pmu_enable_event()
1513 	if (hwc->config_base & KRAIT_EVENT_MASK)  in krait_pmu_enable_event()
1514 		krait_evt_setup(idx, hwc->config_base);  in krait_pmu_enable_event()
1516 		armv7_pmnc_write_evtsel(idx, hwc->config_base);  in krait_pmu_enable_event()
1529 	u32 idx, nb_cnt = cpu_pmu->num_events;  in krait_pmu_reset()
1554 	struct hw_perf_event *hwc = &event->hw;  in krait_event_to_bit()
1555 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in krait_event_to_bit()
1557 	if (hwc->config_base & VENUM_EVENT)  in krait_event_to_bit()
1561 	bit -= krait_get_pmresrn_event(0);  in krait_event_to_bit()
1580 	int bit = -1;  in krait_pmu_get_event_idx()
1581 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_get_event_idx()
1582 	unsigned int region = EVENT_REGION(hwc->config_base);  in krait_pmu_get_event_idx()
1583 	unsigned int code = EVENT_CODE(hwc->config_base);  in krait_pmu_get_event_idx()
1584 	unsigned int group = EVENT_GROUP(hwc->config_base);  in krait_pmu_get_event_idx()
1585 	bool venum_event = EVENT_VENUM(hwc->config_base);  in krait_pmu_get_event_idx()
1586 	bool krait_event = EVENT_CPU(hwc->config_base);  in krait_pmu_get_event_idx()
1591 			return -EINVAL;  in krait_pmu_get_event_idx()
1593 			return -EINVAL;  in krait_pmu_get_event_idx()
1596 		if (test_and_set_bit(bit, cpuc->used_mask))  in krait_pmu_get_event_idx()
1597 			return -EAGAIN;  in krait_pmu_get_event_idx()
1602 		clear_bit(bit, cpuc->used_mask);  in krait_pmu_get_event_idx()
1611 	struct hw_perf_event *hwc = &event->hw;  in krait_pmu_clear_event_idx()
1612 	unsigned int region = EVENT_REGION(hwc->config_base);  in krait_pmu_clear_event_idx()
1613 	unsigned int group = EVENT_GROUP(hwc->config_base);  in krait_pmu_clear_event_idx()
1614 	bool venum_event = EVENT_VENUM(hwc->config_base);  in krait_pmu_clear_event_idx()
1615 	bool krait_event = EVENT_CPU(hwc->config_base);  in krait_pmu_clear_event_idx()
1620 		clear_bit(bit, cpuc->used_mask);  in krait_pmu_clear_event_idx()
1627 	cpu_pmu->name		= "armv7_krait";  in krait_pmu_init()
1629 	if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,  in krait_pmu_init()
1630 				  "qcom,no-pc-write"))  in krait_pmu_init()
1631 		cpu_pmu->map_event = krait_map_event_no_branch;  in krait_pmu_init()
1633 		cpu_pmu->map_event = krait_map_event;  in krait_pmu_init()
1634 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;  in krait_pmu_init()
1635 	cpu_pmu->reset		= krait_pmu_reset;  in krait_pmu_init()
1636 	cpu_pmu->enable		= krait_pmu_enable_event;  in krait_pmu_init()
1637 	cpu_pmu->disable	= krait_pmu_disable_event;  in krait_pmu_init()
1638 	cpu_pmu->get_event_idx	= krait_pmu_get_event_idx;  in krait_pmu_init()
1639 	cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;  in krait_pmu_init()
1647  *            +--------------------------------+
1649  *            +--------------------------------+
1651  *            +--------------------------------+
1653  *            +--------------------------------+
1655  *            +--------------------------------+
1657  *            +--------------------------------+
1663  *      hwc->config_base = 0xNRCCG
1752 	/* Mix in mode-exclusion bits */  in scorpion_evt_setup()
1798 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_disable_event()
1799 	int idx = hwc->idx;  in scorpion_pmu_disable_event()
1809 	if (hwc->config_base & KRAIT_EVENT_MASK)  in scorpion_pmu_disable_event()
1810 		scorpion_clearpmu(hwc->config_base);  in scorpion_pmu_disable_event()
1818 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_enable_event()
1819 	int idx = hwc->idx;  in scorpion_pmu_enable_event()
1834 	if (hwc->config_base & KRAIT_EVENT_MASK)  in scorpion_pmu_enable_event()
1835 		scorpion_evt_setup(idx, hwc->config_base);  in scorpion_pmu_enable_event()
1837 		armv7_pmnc_write_evtsel(idx, hwc->config_base);  in scorpion_pmu_enable_event()
1850 	u32 idx, nb_cnt = cpu_pmu->num_events;  in scorpion_pmu_reset()
1875 	struct hw_perf_event *hwc = &event->hw;  in scorpion_event_to_bit()
1876 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in scorpion_event_to_bit()
1878 	if (hwc->config_base & VENUM_EVENT)  in scorpion_event_to_bit()
1882 	bit -= scorpion_get_pmresrn_event(0);  in scorpion_event_to_bit()
1901 	int bit = -1;  in scorpion_pmu_get_event_idx()
1902 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_get_event_idx()
1903 	unsigned int region = EVENT_REGION(hwc->config_base);  in scorpion_pmu_get_event_idx()
1904 	unsigned int group = EVENT_GROUP(hwc->config_base);  in scorpion_pmu_get_event_idx()
1905 	bool venum_event = EVENT_VENUM(hwc->config_base);  in scorpion_pmu_get_event_idx()
1906 	bool scorpion_event = EVENT_CPU(hwc->config_base);  in scorpion_pmu_get_event_idx()
1911 			return -EINVAL;  in scorpion_pmu_get_event_idx()
1914 		if (test_and_set_bit(bit, cpuc->used_mask))  in scorpion_pmu_get_event_idx()
1915 			return -EAGAIN;  in scorpion_pmu_get_event_idx()
1920 		clear_bit(bit, cpuc->used_mask);  in scorpion_pmu_get_event_idx()
1929 	struct hw_perf_event *hwc = &event->hw;  in scorpion_pmu_clear_event_idx()
1930 	unsigned int region = EVENT_REGION(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1931 	unsigned int group = EVENT_GROUP(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1932 	bool venum_event = EVENT_VENUM(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1933 	bool scorpion_event = EVENT_CPU(hwc->config_base);  in scorpion_pmu_clear_event_idx()
1938 		clear_bit(bit, cpuc->used_mask);  in scorpion_pmu_clear_event_idx()
1945 	cpu_pmu->name		= "armv7_scorpion";  in scorpion_pmu_init()
1946 	cpu_pmu->map_event	= scorpion_map_event;  in scorpion_pmu_init()
1947 	cpu_pmu->reset		= scorpion_pmu_reset;  in scorpion_pmu_init()
1948 	cpu_pmu->enable		= scorpion_pmu_enable_event;  in scorpion_pmu_init()
1949 	cpu_pmu->disable	= scorpion_pmu_disable_event;  in scorpion_pmu_init()
1950 	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;  in scorpion_pmu_init()
1951 	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;  in scorpion_pmu_init()
1958 	cpu_pmu->name		= "armv7_scorpion_mp";  in scorpion_mp_pmu_init()
1959 	cpu_pmu->map_event	= scorpion_map_event;  in scorpion_mp_pmu_init()
1960 	cpu_pmu->reset		= scorpion_pmu_reset;  in scorpion_mp_pmu_init()
1961 	cpu_pmu->enable		= scorpion_pmu_enable_event;  in scorpion_mp_pmu_init()
1962 	cpu_pmu->disable	= scorpion_pmu_disable_event;  in scorpion_mp_pmu_init()
1963 	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;  in scorpion_mp_pmu_init()
1964 	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;  in scorpion_mp_pmu_init()
1969 	{.compatible = "arm,cortex-a17-pmu",	.data = armv7_a17_pmu_init},
1970 	{.compatible = "arm,cortex-a15-pmu",	.data = armv7_a15_pmu_init},
1971 	{.compatible = "arm,cortex-a12-pmu",	.data = armv7_a12_pmu_init},
1972 	{.compatible = "arm,cortex-a9-pmu",	.data = armv7_a9_pmu_init},
1973 	{.compatible = "arm,cortex-a8-pmu",	.data = armv7_a8_pmu_init},
1974 	{.compatible = "arm,cortex-a7-pmu",	.data = armv7_a7_pmu_init},
1975 	{.compatible = "arm,cortex-a5-pmu",	.data = armv7_a5_pmu_init},
1976 	{.compatible = "qcom,krait-pmu",	.data = krait_pmu_init},
1977 	{.compatible = "qcom,scorpion-pmu",	.data = scorpion_pmu_init},
1978 	{.compatible = "qcom,scorpion-mp-pmu",	.data = scorpion_mp_pmu_init},
1997 		.name	= "armv7-pmu",