1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/pci.h> 7 8 #include <linux/pds/pds_common.h> 9 10 #include "core.h" 11 12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION); 13 MODULE_AUTHOR("Advanced Micro Devices, Inc"); 14 MODULE_LICENSE("GPL"); 15 16 /* Supported devices */ 17 static const struct pci_device_id pdsc_id_table[] = { 18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) }, 19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) }, 20 { 0, } /* end of table */ 21 }; 22 MODULE_DEVICE_TABLE(pci, pdsc_id_table); 23 24 static void pdsc_wdtimer_cb(struct timer_list *t) 25 { 26 struct pdsc *pdsc = timer_container_of(pdsc, t, wdtimer); 27 28 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies); 29 mod_timer(&pdsc->wdtimer, 30 round_jiffies(jiffies + pdsc->wdtimer_period)); 31 32 queue_work(pdsc->wq, &pdsc->health_work); 33 } 34 35 static void pdsc_unmap_bars(struct pdsc *pdsc) 36 { 37 struct pdsc_dev_bar *bars = pdsc->bars; 38 unsigned int i; 39 40 pdsc->info_regs = NULL; 41 pdsc->cmd_regs = NULL; 42 pdsc->intr_status = NULL; 43 pdsc->intr_ctrl = NULL; 44 45 for (i = 0; i < PDS_CORE_BARS_MAX; i++) { 46 if (bars[i].vaddr) 47 pci_iounmap(pdsc->pdev, bars[i].vaddr); 48 bars[i].vaddr = NULL; 49 } 50 } 51 52 static int pdsc_map_bars(struct pdsc *pdsc) 53 { 54 struct pdsc_dev_bar *bar = pdsc->bars; 55 struct pci_dev *pdev = pdsc->pdev; 56 struct device *dev = pdsc->dev; 57 struct pdsc_dev_bar *bars; 58 unsigned int i, j; 59 int num_bars = 0; 60 int err; 61 u32 sig; 62 63 bars = pdsc->bars; 64 65 /* Since the PCI interface in the hardware is configurable, 66 * we need to poke into all the bars to find the set we're 67 * expecting. 68 */ 69 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) { 70 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 71 continue; 72 73 bars[j].len = pci_resource_len(pdev, i); 74 bars[j].bus_addr = pci_resource_start(pdev, i); 75 bars[j].res_index = i; 76 77 /* only map the whole bar 0 */ 78 if (j > 0) { 79 bars[j].vaddr = NULL; 80 } else { 81 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 82 if (!bars[j].vaddr) { 83 dev_err(dev, "Cannot map BAR %d, aborting\n", i); 84 return -ENODEV; 85 } 86 } 87 88 j++; 89 } 90 num_bars = j; 91 92 /* BAR0: dev_cmd and interrupts */ 93 if (num_bars < 1) { 94 dev_err(dev, "No bars found\n"); 95 err = -EFAULT; 96 goto err_out; 97 } 98 99 if (bar->len < PDS_CORE_BAR0_SIZE) { 100 dev_err(dev, "Resource bar size %lu too small\n", bar->len); 101 err = -EFAULT; 102 goto err_out; 103 } 104 105 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET; 106 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET; 107 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET; 108 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET; 109 110 sig = ioread32(&pdsc->info_regs->signature); 111 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) { 112 dev_err(dev, "Incompatible firmware signature %x", sig); 113 err = -EFAULT; 114 goto err_out; 115 } 116 117 /* BAR1: doorbells */ 118 bar++; 119 if (num_bars < 2) { 120 dev_err(dev, "Doorbell bar missing\n"); 121 err = -EFAULT; 122 goto err_out; 123 } 124 125 pdsc->db_pages = bar->vaddr; 126 pdsc->phy_db_pages = bar->bus_addr; 127 128 return 0; 129 130 err_out: 131 pdsc_unmap_bars(pdsc); 132 return err; 133 } 134 135 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num) 136 { 137 return pci_iomap_range(pdsc->pdev, 138 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index, 139 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 140 } 141 142 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs) 143 { 144 struct pdsc *pdsc = pci_get_drvdata(pdev); 145 struct device *dev = pdsc->dev; 146 int ret = 0; 147 148 if (num_vfs > 0) { 149 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf), 150 GFP_KERNEL); 151 if (!pdsc->vfs) 152 return -ENOMEM; 153 pdsc->num_vfs = num_vfs; 154 155 ret = pci_enable_sriov(pdev, num_vfs); 156 if (ret) { 157 dev_err(dev, "Cannot enable SRIOV: %pe\n", 158 ERR_PTR(ret)); 159 goto no_vfs; 160 } 161 162 return num_vfs; 163 } 164 165 no_vfs: 166 pci_disable_sriov(pdev); 167 168 kfree(pdsc->vfs); 169 pdsc->vfs = NULL; 170 pdsc->num_vfs = 0; 171 172 return ret; 173 } 174 175 static int pdsc_init_vf(struct pdsc *vf) 176 { 177 struct devlink *dl; 178 struct pdsc *pf; 179 int err; 180 181 pf = pdsc_get_pf_struct(vf->pdev); 182 if (IS_ERR_OR_NULL(pf)) 183 return PTR_ERR(pf) ?: -1; 184 185 vf->vf_id = pci_iov_vf_id(vf->pdev); 186 187 dl = priv_to_devlink(vf); 188 devl_lock(dl); 189 devl_register(dl); 190 devl_unlock(dl); 191 192 pf->vfs[vf->vf_id].vf = vf; 193 err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA, 194 &pf->vfs[vf->vf_id].padev); 195 if (err) { 196 devl_lock(dl); 197 devl_unregister(dl); 198 devl_unlock(dl); 199 } 200 201 return err; 202 } 203 204 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = { 205 .name = "fw", 206 .diagnose = pdsc_fw_reporter_diagnose, 207 }; 208 209 static const struct devlink_param pdsc_dl_params[] = { 210 DEVLINK_PARAM_GENERIC(ENABLE_VNET, 211 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 212 pdsc_dl_enable_get, 213 pdsc_dl_enable_set, 214 pdsc_dl_enable_validate), 215 }; 216 217 #define PDSC_WQ_NAME_LEN 24 218 219 static int pdsc_init_pf(struct pdsc *pdsc) 220 { 221 struct devlink_health_reporter *hr; 222 char wq_name[PDSC_WQ_NAME_LEN]; 223 struct devlink *dl; 224 int err; 225 226 pcie_print_link_status(pdsc->pdev); 227 228 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 229 if (err) { 230 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 231 ERR_PTR(err)); 232 return err; 233 } 234 235 err = pdsc_map_bars(pdsc); 236 if (err) 237 goto err_out_release_regions; 238 239 /* General workqueue and timer, but don't start timer yet */ 240 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); 241 pdsc->wq = create_singlethread_workqueue(wq_name); 242 INIT_WORK(&pdsc->health_work, pdsc_health_thread); 243 INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread); 244 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 245 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ; 246 247 mutex_init(&pdsc->devcmd_lock); 248 mutex_init(&pdsc->config_lock); 249 spin_lock_init(&pdsc->adminq_lock); 250 251 mutex_lock(&pdsc->config_lock); 252 set_bit(PDSC_S_FW_DEAD, &pdsc->state); 253 254 err = pdsc_setup(pdsc, PDSC_SETUP_INIT); 255 if (err) { 256 mutex_unlock(&pdsc->config_lock); 257 goto err_out_unmap_bars; 258 } 259 260 err = pdsc_start(pdsc); 261 if (err) { 262 mutex_unlock(&pdsc->config_lock); 263 goto err_out_teardown; 264 } 265 266 mutex_unlock(&pdsc->config_lock); 267 268 err = pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, &pdsc->padev); 269 if (err) 270 goto err_out_stop; 271 272 dl = priv_to_devlink(pdsc); 273 devl_lock(dl); 274 err = devl_params_register(dl, pdsc_dl_params, 275 ARRAY_SIZE(pdsc_dl_params)); 276 if (err) { 277 devl_unlock(dl); 278 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n", 279 ERR_PTR(err)); 280 goto err_out_del_dev; 281 } 282 283 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc); 284 if (IS_ERR(hr)) { 285 devl_unlock(dl); 286 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr); 287 err = PTR_ERR(hr); 288 goto err_out_unreg_params; 289 } 290 pdsc->fw_reporter = hr; 291 292 devl_register(dl); 293 devl_unlock(dl); 294 295 /* Lastly, start the health check timer */ 296 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period)); 297 298 return 0; 299 300 err_out_unreg_params: 301 devlink_params_unregister(dl, pdsc_dl_params, 302 ARRAY_SIZE(pdsc_dl_params)); 303 err_out_del_dev: 304 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); 305 err_out_stop: 306 pdsc_stop(pdsc); 307 err_out_teardown: 308 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 309 err_out_unmap_bars: 310 timer_shutdown_sync(&pdsc->wdtimer); 311 if (pdsc->wq) 312 destroy_workqueue(pdsc->wq); 313 mutex_destroy(&pdsc->config_lock); 314 mutex_destroy(&pdsc->devcmd_lock); 315 pci_free_irq_vectors(pdsc->pdev); 316 pdsc_unmap_bars(pdsc); 317 err_out_release_regions: 318 pci_release_regions(pdsc->pdev); 319 320 return err; 321 } 322 323 static const struct devlink_ops pdsc_dl_ops = { 324 .info_get = pdsc_dl_info_get, 325 .flash_update = pdsc_dl_flash_update, 326 }; 327 328 static const struct devlink_ops pdsc_dl_vf_ops = { 329 }; 330 331 static DEFINE_IDA(pdsc_ida); 332 333 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 334 { 335 struct device *dev = &pdev->dev; 336 const struct devlink_ops *ops; 337 struct devlink *dl; 338 struct pdsc *pdsc; 339 bool is_pf; 340 int err; 341 342 is_pf = !pdev->is_virtfn; 343 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops; 344 dl = devlink_alloc(ops, sizeof(struct pdsc), dev); 345 if (!dl) 346 return -ENOMEM; 347 pdsc = devlink_priv(dl); 348 349 pdsc->pdev = pdev; 350 pdsc->dev = &pdev->dev; 351 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 352 pci_set_drvdata(pdev, pdsc); 353 pdsc_debugfs_add_dev(pdsc); 354 355 err = ida_alloc(&pdsc_ida, GFP_KERNEL); 356 if (err < 0) { 357 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n", 358 __func__, ERR_PTR(err)); 359 goto err_out_free_devlink; 360 } 361 pdsc->uid = err; 362 363 /* Query system for DMA addressing limitation for the device. */ 364 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN)); 365 if (err) { 366 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n", 367 ERR_PTR(err)); 368 goto err_out_free_ida; 369 } 370 371 err = pci_enable_device(pdev); 372 if (err) { 373 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 374 goto err_out_free_ida; 375 } 376 pci_set_master(pdev); 377 378 if (is_pf) 379 err = pdsc_init_pf(pdsc); 380 else 381 err = pdsc_init_vf(pdsc); 382 if (err) { 383 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err)); 384 goto err_out_disable_device; 385 } 386 387 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 388 return 0; 389 390 err_out_disable_device: 391 pci_disable_device(pdev); 392 err_out_free_ida: 393 ida_free(&pdsc_ida, pdsc->uid); 394 err_out_free_devlink: 395 pdsc_debugfs_del_dev(pdsc); 396 devlink_free(dl); 397 398 return err; 399 } 400 401 static void pdsc_remove(struct pci_dev *pdev) 402 { 403 struct pdsc *pdsc = pci_get_drvdata(pdev); 404 struct devlink *dl; 405 406 /* Unhook the registrations first to be sure there 407 * are no requests while we're stopping. 408 */ 409 dl = priv_to_devlink(pdsc); 410 devl_lock(dl); 411 devl_unregister(dl); 412 if (!pdev->is_virtfn) { 413 if (pdsc->fw_reporter) { 414 devl_health_reporter_destroy(pdsc->fw_reporter); 415 pdsc->fw_reporter = NULL; 416 } 417 devl_params_unregister(dl, pdsc_dl_params, 418 ARRAY_SIZE(pdsc_dl_params)); 419 } 420 devl_unlock(dl); 421 422 if (pdev->is_virtfn) { 423 struct pdsc *pf; 424 425 pf = pdsc_get_pf_struct(pdsc->pdev); 426 if (!IS_ERR(pf)) { 427 pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev); 428 pf->vfs[pdsc->vf_id].vf = NULL; 429 } 430 } else { 431 /* Remove the VFs and their aux_bus connections before other 432 * cleanup so that the clients can use the AdminQ to cleanly 433 * shut themselves down. 434 */ 435 pdsc_sriov_configure(pdev, 0); 436 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); 437 438 timer_shutdown_sync(&pdsc->wdtimer); 439 if (pdsc->wq) 440 destroy_workqueue(pdsc->wq); 441 442 mutex_lock(&pdsc->config_lock); 443 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); 444 445 pdsc_stop(pdsc); 446 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 447 mutex_unlock(&pdsc->config_lock); 448 mutex_destroy(&pdsc->config_lock); 449 mutex_destroy(&pdsc->devcmd_lock); 450 451 pdsc_unmap_bars(pdsc); 452 pci_release_regions(pdev); 453 } 454 455 pci_disable_device(pdev); 456 457 ida_free(&pdsc_ida, pdsc->uid); 458 pdsc_debugfs_del_dev(pdsc); 459 devlink_free(dl); 460 } 461 462 static void pdsc_stop_health_thread(struct pdsc *pdsc) 463 { 464 if (pdsc->pdev->is_virtfn) 465 return; 466 467 timer_shutdown_sync(&pdsc->wdtimer); 468 if (pdsc->health_work.func) 469 cancel_work_sync(&pdsc->health_work); 470 } 471 472 static void pdsc_restart_health_thread(struct pdsc *pdsc) 473 { 474 if (pdsc->pdev->is_virtfn) 475 return; 476 477 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 478 mod_timer(&pdsc->wdtimer, jiffies + 1); 479 } 480 481 static void pdsc_reset_prepare(struct pci_dev *pdev) 482 { 483 struct pdsc *pdsc = pci_get_drvdata(pdev); 484 485 pdsc_stop_health_thread(pdsc); 486 pdsc_fw_down(pdsc); 487 488 if (pdev->is_virtfn) { 489 struct pdsc *pf; 490 491 pf = pdsc_get_pf_struct(pdsc->pdev); 492 if (!IS_ERR(pf)) 493 pdsc_auxbus_dev_del(pdsc, pf, 494 &pf->vfs[pdsc->vf_id].padev); 495 } else { 496 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); 497 } 498 499 pdsc_unmap_bars(pdsc); 500 pci_release_regions(pdev); 501 if (pci_is_enabled(pdev)) 502 pci_disable_device(pdev); 503 } 504 505 static void pdsc_reset_done(struct pci_dev *pdev) 506 { 507 struct pdsc *pdsc = pci_get_drvdata(pdev); 508 struct device *dev = pdsc->dev; 509 int err; 510 511 err = pci_enable_device(pdev); 512 if (err) { 513 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 514 return; 515 } 516 pci_set_master(pdev); 517 518 if (!pdev->is_virtfn) { 519 pcie_print_link_status(pdsc->pdev); 520 521 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 522 if (err) { 523 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 524 ERR_PTR(err)); 525 return; 526 } 527 528 err = pdsc_map_bars(pdsc); 529 if (err) 530 return; 531 } 532 533 pdsc_fw_up(pdsc); 534 pdsc_restart_health_thread(pdsc); 535 536 if (pdev->is_virtfn) { 537 struct pdsc *pf; 538 539 pf = pdsc_get_pf_struct(pdsc->pdev); 540 if (!IS_ERR(pf)) 541 pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA, 542 &pf->vfs[pdsc->vf_id].padev); 543 } else { 544 pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, 545 &pdsc->padev); 546 } 547 } 548 549 static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev, 550 pci_channel_state_t error) 551 { 552 if (error == pci_channel_io_frozen) { 553 pdsc_reset_prepare(pdev); 554 return PCI_ERS_RESULT_NEED_RESET; 555 } 556 557 return PCI_ERS_RESULT_NONE; 558 } 559 560 static void pdsc_pci_error_resume(struct pci_dev *pdev) 561 { 562 struct pdsc *pdsc = pci_get_drvdata(pdev); 563 564 if (test_bit(PDSC_S_FW_DEAD, &pdsc->state)) 565 pci_reset_function_locked(pdev); 566 } 567 568 static const struct pci_error_handlers pdsc_err_handler = { 569 /* FLR handling */ 570 .reset_prepare = pdsc_reset_prepare, 571 .reset_done = pdsc_reset_done, 572 573 /* AER handling */ 574 .error_detected = pdsc_pci_error_detected, 575 .resume = pdsc_pci_error_resume, 576 }; 577 578 static struct pci_driver pdsc_driver = { 579 .name = PDS_CORE_DRV_NAME, 580 .id_table = pdsc_id_table, 581 .probe = pdsc_probe, 582 .remove = pdsc_remove, 583 .sriov_configure = pdsc_sriov_configure, 584 .err_handler = &pdsc_err_handler, 585 }; 586 587 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev) 588 { 589 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver); 590 } 591 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct); 592 593 static int __init pdsc_init_module(void) 594 { 595 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME)) 596 return -EINVAL; 597 598 pdsc_debugfs_create(); 599 return pci_register_driver(&pdsc_driver); 600 } 601 602 static void __exit pdsc_cleanup_module(void) 603 { 604 pci_unregister_driver(&pdsc_driver); 605 pdsc_debugfs_destroy(); 606 } 607 608 module_init(pdsc_init_module); 609 module_exit(pdsc_cleanup_module); 610