1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe PMU driver 4 * 5 * Copyright (C) 2021-2023 Alibaba Inc. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/cpuhotplug.h> 11 #include <linux/cpumask.h> 12 #include <linux/device.h> 13 #include <linux/errno.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/pcie-dwc.h> 17 #include <linux/perf_event.h> 18 #include <linux/pci.h> 19 #include <linux/platform_device.h> 20 #include <linux/smp.h> 21 #include <linux/sysfs.h> 22 #include <linux/types.h> 23 24 #define DWC_PCIE_EVENT_CNT_CTL 0x8 25 26 /* 27 * Event Counter Data Select includes two parts: 28 * - 27-24: Group number(4-bit: 0..0x7) 29 * - 23-16: Event number(8-bit: 0..0x13) within the Group 30 * 31 * Put them together as in TRM. 32 */ 33 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) 34 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) 35 #define DWC_PCIE_CNT_STATUS BIT(7) 36 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) 37 #define DWC_PCIE_PER_EVENT_OFF 0x1 38 #define DWC_PCIE_PER_EVENT_ON 0x3 39 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) 40 #define DWC_PCIE_EVENT_PER_CLEAR 0x1 41 42 #define DWC_PCIE_EVENT_CNT_DATA 0xC 43 44 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 45 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) 46 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) 47 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0 48 #define DWC_PCIE_DURATION_1MS 0x1 49 #define DWC_PCIE_DURATION_10MS 0x2 50 #define DWC_PCIE_DURATION_100MS 0x3 51 #define DWC_PCIE_DURATION_1S 0x4 52 #define DWC_PCIE_DURATION_2S 0x5 53 #define DWC_PCIE_DURATION_4S 0x6 54 #define DWC_PCIE_DURATION_4US 0xFF 55 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) 56 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 57 58 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 59 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 60 61 /* Event attributes */ 62 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) 63 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) 64 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) 65 66 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) 67 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) 68 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) 69 70 enum dwc_pcie_event_type { 71 DWC_PCIE_TIME_BASE_EVENT, 72 DWC_PCIE_LANE_EVENT, 73 DWC_PCIE_EVENT_TYPE_MAX, 74 }; 75 76 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) 77 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) 78 79 struct dwc_pcie_pmu { 80 struct pmu pmu; 81 struct pci_dev *pdev; /* Root Port device */ 82 u16 ras_des_offset; 83 u32 nr_lanes; 84 85 struct hlist_node cpuhp_node; 86 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX]; 87 int on_cpu; 88 }; 89 90 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) 91 92 static int dwc_pcie_pmu_hp_state; 93 static struct list_head dwc_pcie_dev_info_head = 94 LIST_HEAD_INIT(dwc_pcie_dev_info_head); 95 static bool notify; 96 97 struct dwc_pcie_dev_info { 98 struct platform_device *plat_dev; 99 struct pci_dev *pdev; 100 struct list_head dev_node; 101 }; 102 103 static ssize_t cpumask_show(struct device *dev, 104 struct device_attribute *attr, 105 char *buf) 106 { 107 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); 108 109 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); 110 } 111 static DEVICE_ATTR_RO(cpumask); 112 113 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { 114 &dev_attr_cpumask.attr, 115 NULL 116 }; 117 118 static struct attribute_group dwc_pcie_cpumask_attr_group = { 119 .attrs = dwc_pcie_pmu_cpumask_attrs, 120 }; 121 122 struct dwc_pcie_format_attr { 123 struct device_attribute attr; 124 u64 field; 125 int config; 126 }; 127 128 PMU_FORMAT_ATTR(eventid, "config:0-15"); 129 PMU_FORMAT_ATTR(type, "config:16-19"); 130 PMU_FORMAT_ATTR(lane, "config:20-27"); 131 132 static struct attribute *dwc_pcie_format_attrs[] = { 133 &format_attr_type.attr, 134 &format_attr_eventid.attr, 135 &format_attr_lane.attr, 136 NULL, 137 }; 138 139 static struct attribute_group dwc_pcie_format_attrs_group = { 140 .name = "format", 141 .attrs = dwc_pcie_format_attrs, 142 }; 143 144 struct dwc_pcie_event_attr { 145 struct device_attribute attr; 146 enum dwc_pcie_event_type type; 147 u16 eventid; 148 u8 lane; 149 }; 150 151 static ssize_t dwc_pcie_event_show(struct device *dev, 152 struct device_attribute *attr, char *buf) 153 { 154 struct dwc_pcie_event_attr *eattr; 155 156 eattr = container_of(attr, typeof(*eattr), attr); 157 158 if (eattr->type == DWC_PCIE_LANE_EVENT) 159 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n", 160 eattr->eventid, eattr->type); 161 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) 162 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n", 163 eattr->eventid, eattr->type); 164 165 return 0; 166 } 167 168 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ 169 (&((struct dwc_pcie_event_attr[]) {{ \ 170 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ 171 .type = _type, \ 172 .eventid = _eventid, \ 173 .lane = _lane, \ 174 }})[0].attr.attr) 175 176 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ 177 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) 178 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ 179 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) 180 181 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { 182 /* Group #0 */ 183 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), 184 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), 185 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), 186 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), 187 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), 188 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), 189 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), 190 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), 191 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08), 192 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09), 193 194 /* Group #1 */ 195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20), 196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21), 197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22), 198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23), 199 200 /* 201 * Leave it to the user to specify the lane ID to avoid generating 202 * a list of hundreds of events. 203 */ 204 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), 205 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), 206 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), 207 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), 208 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604), 209 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605), 210 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606), 211 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), 212 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), 213 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), 214 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), 215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), 216 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), 217 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), 218 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), 219 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), 220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), 221 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), 222 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), 223 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), 224 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), 225 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), 226 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), 227 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), 228 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), 229 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), 230 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), 231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), 232 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), 233 NULL 234 }; 235 236 static const struct attribute_group dwc_pcie_event_attrs_group = { 237 .name = "events", 238 .attrs = dwc_pcie_pmu_time_event_attrs, 239 }; 240 241 static const struct attribute_group *dwc_pcie_attr_groups[] = { 242 &dwc_pcie_event_attrs_group, 243 &dwc_pcie_format_attrs_group, 244 &dwc_pcie_cpumask_attr_group, 245 NULL 246 }; 247 248 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, 249 bool enable) 250 { 251 struct pci_dev *pdev = pcie_pmu->pdev; 252 u16 ras_des_offset = pcie_pmu->ras_des_offset; 253 254 if (enable) 255 pci_clear_and_set_config_dword(pdev, 256 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 257 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 258 else 259 pci_clear_and_set_config_dword(pdev, 260 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 261 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); 262 } 263 264 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, 265 bool enable) 266 { 267 struct pci_dev *pdev = pcie_pmu->pdev; 268 u16 ras_des_offset = pcie_pmu->ras_des_offset; 269 270 pci_clear_and_set_config_dword(pdev, 271 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, 272 DWC_PCIE_TIME_BASED_TIMER_START, enable); 273 } 274 275 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) 276 { 277 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 278 struct pci_dev *pdev = pcie_pmu->pdev; 279 u16 ras_des_offset = pcie_pmu->ras_des_offset; 280 u32 val; 281 282 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val); 283 284 return val; 285 } 286 287 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) 288 { 289 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 290 struct pci_dev *pdev = pcie_pmu->pdev; 291 int event_id = DWC_PCIE_EVENT_ID(event); 292 u16 ras_des_offset = pcie_pmu->ras_des_offset; 293 u32 lo, hi, ss; 294 u64 val; 295 296 /* 297 * The 64-bit value of the data counter is spread across two 298 * registers that are not synchronized. In order to read them 299 * atomically, ensure that the high 32 bits match before and after 300 * reading the low 32 bits. 301 */ 302 pci_read_config_dword(pdev, 303 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi); 304 do { 305 /* snapshot the high 32 bits */ 306 ss = hi; 307 308 pci_read_config_dword( 309 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, 310 &lo); 311 pci_read_config_dword( 312 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, 313 &hi); 314 } while (hi != ss); 315 316 val = ((u64)hi << 32) | lo; 317 /* 318 * The Group#1 event measures the amount of data processed in 16-byte 319 * units. Simplify the end-user interface by multiplying the counter 320 * at the point of read. 321 */ 322 if (event_id >= 0x20 && event_id <= 0x23) 323 val *= 16; 324 325 return val; 326 } 327 328 static void dwc_pcie_pmu_event_update(struct perf_event *event) 329 { 330 struct hw_perf_event *hwc = &event->hw; 331 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 332 u64 delta, prev, now = 0; 333 334 do { 335 prev = local64_read(&hwc->prev_count); 336 337 if (type == DWC_PCIE_LANE_EVENT) 338 now = dwc_pcie_pmu_read_lane_event_counter(event); 339 else if (type == DWC_PCIE_TIME_BASE_EVENT) 340 now = dwc_pcie_pmu_read_time_based_counter(event); 341 342 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 343 344 delta = (now - prev) & DWC_PCIE_MAX_PERIOD; 345 /* 32-bit counter for Lane Event Counting */ 346 if (type == DWC_PCIE_LANE_EVENT) 347 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD; 348 349 local64_add(delta, &event->count); 350 } 351 352 static int dwc_pcie_pmu_event_init(struct perf_event *event) 353 { 354 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 355 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 356 struct perf_event *sibling; 357 u32 lane; 358 359 if (event->attr.type != event->pmu->type) 360 return -ENOENT; 361 362 /* We don't support sampling */ 363 if (is_sampling_event(event)) 364 return -EINVAL; 365 366 /* We cannot support task bound events */ 367 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) 368 return -EINVAL; 369 370 if (event->group_leader != event && 371 !is_software_event(event->group_leader)) 372 return -EINVAL; 373 374 for_each_sibling_event(sibling, event->group_leader) { 375 if (sibling->pmu != event->pmu && !is_software_event(sibling)) 376 return -EINVAL; 377 } 378 379 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) 380 return -EINVAL; 381 382 if (type == DWC_PCIE_LANE_EVENT) { 383 lane = DWC_PCIE_EVENT_LANE(event); 384 if (lane < 0 || lane >= pcie_pmu->nr_lanes) 385 return -EINVAL; 386 } 387 388 event->cpu = pcie_pmu->on_cpu; 389 390 return 0; 391 } 392 393 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) 394 { 395 struct hw_perf_event *hwc = &event->hw; 396 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 397 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 398 399 hwc->state = 0; 400 local64_set(&hwc->prev_count, 0); 401 402 if (type == DWC_PCIE_LANE_EVENT) 403 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true); 404 else if (type == DWC_PCIE_TIME_BASE_EVENT) 405 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); 406 } 407 408 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) 409 { 410 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 411 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 412 struct hw_perf_event *hwc = &event->hw; 413 414 if (event->hw.state & PERF_HES_STOPPED) 415 return; 416 417 if (type == DWC_PCIE_LANE_EVENT) 418 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false); 419 else if (type == DWC_PCIE_TIME_BASE_EVENT) 420 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); 421 422 dwc_pcie_pmu_event_update(event); 423 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 424 } 425 426 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 427 { 428 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 429 struct pci_dev *pdev = pcie_pmu->pdev; 430 struct hw_perf_event *hwc = &event->hw; 431 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 432 int event_id = DWC_PCIE_EVENT_ID(event); 433 int lane = DWC_PCIE_EVENT_LANE(event); 434 u16 ras_des_offset = pcie_pmu->ras_des_offset; 435 u32 ctrl; 436 437 /* one counter for each type and it is in use */ 438 if (pcie_pmu->event[type]) 439 return -ENOSPC; 440 441 pcie_pmu->event[type] = event; 442 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 443 444 if (type == DWC_PCIE_LANE_EVENT) { 445 /* EVENT_COUNTER_DATA_REG needs clear manually */ 446 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 447 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 448 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | 449 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 450 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 451 ctrl); 452 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 453 /* 454 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely 455 * use it with any manually controlled duration. And it is 456 * cleared when next measurement starts. 457 */ 458 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | 459 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, 460 DWC_PCIE_DURATION_MANUAL_CTL) | 461 DWC_PCIE_TIME_BASED_CNT_ENABLE; 462 pci_write_config_dword( 463 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl); 464 } 465 466 if (flags & PERF_EF_START) 467 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); 468 469 perf_event_update_userpage(event); 470 471 return 0; 472 } 473 474 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) 475 { 476 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 477 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 478 479 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE); 480 perf_event_update_userpage(event); 481 pcie_pmu->event[type] = NULL; 482 } 483 484 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) 485 { 486 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node); 487 } 488 489 /* 490 * Find the binded DES capability device info of a PCI device. 491 * @pdev: The PCI device. 492 */ 493 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) 494 { 495 struct dwc_pcie_dev_info *dev_info; 496 497 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) 498 if (dev_info->pdev == pdev) 499 return dev_info; 500 501 return NULL; 502 } 503 504 static void dwc_pcie_unregister_pmu(void *data) 505 { 506 struct dwc_pcie_pmu *pcie_pmu = data; 507 508 perf_pmu_unregister(&pcie_pmu->pmu); 509 } 510 511 static u16 dwc_pcie_des_cap(struct pci_dev *pdev) 512 { 513 const struct dwc_pcie_vsec_id *vid; 514 u16 vsec; 515 u32 val; 516 517 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)) 518 return 0; 519 520 for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) { 521 vsec = pci_find_vsec_capability(pdev, vid->vendor_id, 522 vid->vsec_id); 523 if (vsec) { 524 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, 525 &val); 526 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) { 527 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n"); 528 return vsec; 529 } 530 } 531 } 532 return 0; 533 } 534 535 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) 536 { 537 platform_device_unregister(dev_info->plat_dev); 538 list_del(&dev_info->dev_node); 539 kfree(dev_info); 540 } 541 542 static int dwc_pcie_register_dev(struct pci_dev *pdev) 543 { 544 struct platform_device *plat_dev; 545 struct dwc_pcie_dev_info *dev_info; 546 u32 sbdf; 547 548 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn); 549 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0); 550 if (IS_ERR(plat_dev)) 551 return PTR_ERR(plat_dev); 552 553 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL); 554 if (!dev_info) { 555 platform_device_unregister(plat_dev); 556 return -ENOMEM; 557 } 558 559 /* Cache platform device to handle pci device hotplug */ 560 dev_info->plat_dev = plat_dev; 561 dev_info->pdev = pdev; 562 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head); 563 564 return 0; 565 } 566 567 static int dwc_pcie_pmu_notifier(struct notifier_block *nb, 568 unsigned long action, void *data) 569 { 570 struct device *dev = data; 571 struct pci_dev *pdev = to_pci_dev(dev); 572 struct dwc_pcie_dev_info *dev_info; 573 574 switch (action) { 575 case BUS_NOTIFY_ADD_DEVICE: 576 if (!dwc_pcie_des_cap(pdev)) 577 return NOTIFY_DONE; 578 if (dwc_pcie_register_dev(pdev)) 579 return NOTIFY_BAD; 580 break; 581 case BUS_NOTIFY_DEL_DEVICE: 582 dev_info = dwc_pcie_find_dev_info(pdev); 583 if (!dev_info) 584 return NOTIFY_DONE; 585 dwc_pcie_unregister_dev(dev_info); 586 break; 587 } 588 589 return NOTIFY_OK; 590 } 591 592 static struct notifier_block dwc_pcie_pmu_nb = { 593 .notifier_call = dwc_pcie_pmu_notifier, 594 }; 595 596 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) 597 { 598 struct pci_dev *pdev; 599 struct dwc_pcie_pmu *pcie_pmu; 600 char *name; 601 u32 sbdf; 602 u16 vsec; 603 int ret; 604 605 sbdf = plat_dev->id; 606 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff), 607 sbdf & 0xff); 608 if (!pdev) { 609 pr_err("No pdev found for the sbdf 0x%x\n", sbdf); 610 return -ENODEV; 611 } 612 613 vsec = dwc_pcie_des_cap(pdev); 614 if (!vsec) 615 return -ENODEV; 616 617 pci_dev_put(pdev); 618 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf); 619 if (!name) 620 return -ENOMEM; 621 622 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL); 623 if (!pcie_pmu) 624 return -ENOMEM; 625 626 pcie_pmu->pdev = pdev; 627 pcie_pmu->ras_des_offset = vsec; 628 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); 629 pcie_pmu->on_cpu = -1; 630 pcie_pmu->pmu = (struct pmu){ 631 .name = name, 632 .parent = &plat_dev->dev, 633 .module = THIS_MODULE, 634 .attr_groups = dwc_pcie_attr_groups, 635 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 636 .task_ctx_nr = perf_invalid_context, 637 .event_init = dwc_pcie_pmu_event_init, 638 .add = dwc_pcie_pmu_event_add, 639 .del = dwc_pcie_pmu_event_del, 640 .start = dwc_pcie_pmu_event_start, 641 .stop = dwc_pcie_pmu_event_stop, 642 .read = dwc_pcie_pmu_event_update, 643 }; 644 645 /* Add this instance to the list used by the offline callback */ 646 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state, 647 &pcie_pmu->cpuhp_node); 648 if (ret) { 649 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf); 650 return ret; 651 } 652 653 /* Unwind when platform driver removes */ 654 ret = devm_add_action_or_reset(&plat_dev->dev, 655 dwc_pcie_pmu_remove_cpuhp_instance, 656 &pcie_pmu->cpuhp_node); 657 if (ret) 658 return ret; 659 660 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1); 661 if (ret) { 662 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf); 663 return ret; 664 } 665 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, 666 pcie_pmu); 667 if (ret) 668 return ret; 669 670 return 0; 671 } 672 673 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 674 { 675 struct dwc_pcie_pmu *pcie_pmu; 676 677 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 678 if (pcie_pmu->on_cpu == -1) 679 pcie_pmu->on_cpu = cpumask_local_spread( 680 0, dev_to_node(&pcie_pmu->pdev->dev)); 681 682 return 0; 683 } 684 685 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 686 { 687 struct dwc_pcie_pmu *pcie_pmu; 688 struct pci_dev *pdev; 689 unsigned int target; 690 int node; 691 692 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 693 /* Nothing to do if this CPU doesn't own the PMU */ 694 if (cpu != pcie_pmu->on_cpu) 695 return 0; 696 697 pcie_pmu->on_cpu = -1; 698 pdev = pcie_pmu->pdev; 699 node = dev_to_node(&pdev->dev); 700 701 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); 702 if (target >= nr_cpu_ids) 703 target = cpumask_any_but(cpu_online_mask, cpu); 704 705 if (target >= nr_cpu_ids) { 706 pci_err(pdev, "There is no CPU to set\n"); 707 return 0; 708 } 709 710 /* This PMU does NOT support interrupt, just migrate context. */ 711 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); 712 pcie_pmu->on_cpu = target; 713 714 return 0; 715 } 716 717 static struct platform_driver dwc_pcie_pmu_driver = { 718 .probe = dwc_pcie_pmu_probe, 719 .driver = {.name = "dwc_pcie_pmu",}, 720 }; 721 722 static void dwc_pcie_cleanup_devices(void) 723 { 724 struct dwc_pcie_dev_info *dev_info, *tmp; 725 726 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) { 727 dwc_pcie_unregister_dev(dev_info); 728 } 729 } 730 731 static int __init dwc_pcie_pmu_init(void) 732 { 733 struct pci_dev *pdev = NULL; 734 int ret; 735 736 for_each_pci_dev(pdev) { 737 if (!dwc_pcie_des_cap(pdev)) 738 continue; 739 740 ret = dwc_pcie_register_dev(pdev); 741 if (ret) { 742 pci_dev_put(pdev); 743 goto err_cleanup; 744 } 745 } 746 747 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 748 "perf/dwc_pcie_pmu:online", 749 dwc_pcie_pmu_online_cpu, 750 dwc_pcie_pmu_offline_cpu); 751 if (ret < 0) 752 goto err_cleanup; 753 754 dwc_pcie_pmu_hp_state = ret; 755 756 ret = platform_driver_register(&dwc_pcie_pmu_driver); 757 if (ret) 758 goto err_remove_cpuhp; 759 760 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 761 if (ret) 762 goto err_unregister_driver; 763 notify = true; 764 765 return 0; 766 767 err_unregister_driver: 768 platform_driver_unregister(&dwc_pcie_pmu_driver); 769 err_remove_cpuhp: 770 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 771 err_cleanup: 772 dwc_pcie_cleanup_devices(); 773 return ret; 774 } 775 776 static void __exit dwc_pcie_pmu_exit(void) 777 { 778 if (notify) 779 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 780 dwc_pcie_cleanup_devices(); 781 platform_driver_unregister(&dwc_pcie_pmu_driver); 782 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 783 } 784 785 module_init(dwc_pcie_pmu_init); 786 module_exit(dwc_pcie_pmu_exit); 787 788 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller"); 789 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 790 MODULE_LICENSE("GPL v2"); 791