1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Endpoint *Function* (EPF) library 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/slab.h> 12 #include <linux/module.h> 13 14 #include <linux/pci-epc.h> 15 #include <linux/pci-epf.h> 16 #include <linux/pci-ep-cfs.h> 17 18 static DEFINE_MUTEX(pci_epf_mutex); 19 20 static const struct bus_type pci_epf_bus_type; 21 static const struct device_type pci_epf_type; 22 23 /** 24 * pci_epf_unbind() - Notify the function driver that the binding between the 25 * EPF device and EPC device has been lost 26 * @epf: the EPF device which has lost the binding with the EPC device 27 * 28 * Invoke to notify the function driver that the binding between the EPF device 29 * and EPC device has been lost. 30 */ 31 void pci_epf_unbind(struct pci_epf *epf) 32 { 33 struct pci_epf *epf_vf; 34 35 if (!epf->driver) { 36 dev_WARN(&epf->dev, "epf device not bound to driver\n"); 37 return; 38 } 39 40 mutex_lock(&epf->lock); 41 list_for_each_entry(epf_vf, &epf->pci_vepf, list) { 42 if (epf_vf->is_bound) 43 epf_vf->driver->ops->unbind(epf_vf); 44 } 45 if (epf->is_bound) 46 epf->driver->ops->unbind(epf); 47 mutex_unlock(&epf->lock); 48 module_put(epf->driver->owner); 49 } 50 EXPORT_SYMBOL_GPL(pci_epf_unbind); 51 52 /** 53 * pci_epf_bind() - Notify the function driver that the EPF device has been 54 * bound to a EPC device 55 * @epf: the EPF device which has been bound to the EPC device 56 * 57 * Invoke to notify the function driver that it has been bound to a EPC device 58 */ 59 int pci_epf_bind(struct pci_epf *epf) 60 { 61 struct device *dev = &epf->dev; 62 struct pci_epf *epf_vf; 63 u8 func_no, vfunc_no; 64 struct pci_epc *epc; 65 int ret; 66 67 if (!epf->driver) { 68 dev_WARN(dev, "epf device not bound to driver\n"); 69 return -EINVAL; 70 } 71 72 if (!try_module_get(epf->driver->owner)) 73 return -EAGAIN; 74 75 mutex_lock(&epf->lock); 76 list_for_each_entry(epf_vf, &epf->pci_vepf, list) { 77 vfunc_no = epf_vf->vfunc_no; 78 79 if (vfunc_no < 1) { 80 dev_err(dev, "Invalid virtual function number\n"); 81 ret = -EINVAL; 82 goto ret; 83 } 84 85 epc = epf->epc; 86 func_no = epf->func_no; 87 if (!IS_ERR_OR_NULL(epc)) { 88 if (!epc->max_vfs) { 89 dev_err(dev, "No support for virt function\n"); 90 ret = -EINVAL; 91 goto ret; 92 } 93 94 if (vfunc_no > epc->max_vfs[func_no]) { 95 dev_err(dev, "PF%d: Exceeds max vfunc number\n", 96 func_no); 97 ret = -EINVAL; 98 goto ret; 99 } 100 } 101 102 epc = epf->sec_epc; 103 func_no = epf->sec_epc_func_no; 104 if (!IS_ERR_OR_NULL(epc)) { 105 if (!epc->max_vfs) { 106 dev_err(dev, "No support for virt function\n"); 107 ret = -EINVAL; 108 goto ret; 109 } 110 111 if (vfunc_no > epc->max_vfs[func_no]) { 112 dev_err(dev, "PF%d: Exceeds max vfunc number\n", 113 func_no); 114 ret = -EINVAL; 115 goto ret; 116 } 117 } 118 119 epf_vf->func_no = epf->func_no; 120 epf_vf->sec_epc_func_no = epf->sec_epc_func_no; 121 epf_vf->epc = epf->epc; 122 epf_vf->sec_epc = epf->sec_epc; 123 ret = epf_vf->driver->ops->bind(epf_vf); 124 if (ret) 125 goto ret; 126 epf_vf->is_bound = true; 127 } 128 129 ret = epf->driver->ops->bind(epf); 130 if (ret) 131 goto ret; 132 epf->is_bound = true; 133 134 mutex_unlock(&epf->lock); 135 return 0; 136 137 ret: 138 mutex_unlock(&epf->lock); 139 pci_epf_unbind(epf); 140 141 return ret; 142 } 143 EXPORT_SYMBOL_GPL(pci_epf_bind); 144 145 /** 146 * pci_epf_add_vepf() - associate virtual EP function to physical EP function 147 * @epf_pf: the physical EP function to which the virtual EP function should be 148 * associated 149 * @epf_vf: the virtual EP function to be added 150 * 151 * A physical endpoint function can be associated with multiple virtual 152 * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint 153 * function to a physical PCI endpoint function. 154 */ 155 int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) 156 { 157 u32 vfunc_no; 158 159 if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf)) 160 return -EINVAL; 161 162 if (epf_pf->epc || epf_vf->epc || epf_vf->epf_pf) 163 return -EBUSY; 164 165 if (epf_pf->sec_epc || epf_vf->sec_epc) 166 return -EBUSY; 167 168 mutex_lock(&epf_pf->lock); 169 vfunc_no = find_first_zero_bit(&epf_pf->vfunction_num_map, 170 BITS_PER_LONG); 171 if (vfunc_no >= BITS_PER_LONG) { 172 mutex_unlock(&epf_pf->lock); 173 return -EINVAL; 174 } 175 176 set_bit(vfunc_no, &epf_pf->vfunction_num_map); 177 epf_vf->vfunc_no = vfunc_no; 178 179 epf_vf->epf_pf = epf_pf; 180 epf_vf->is_vf = true; 181 182 list_add_tail(&epf_vf->list, &epf_pf->pci_vepf); 183 mutex_unlock(&epf_pf->lock); 184 185 return 0; 186 } 187 EXPORT_SYMBOL_GPL(pci_epf_add_vepf); 188 189 /** 190 * pci_epf_remove_vepf() - remove virtual EP function from physical EP function 191 * @epf_pf: the physical EP function from which the virtual EP function should 192 * be removed 193 * @epf_vf: the virtual EP function to be removed 194 * 195 * Invoke to remove a virtual endpoint function from the physical endpoint 196 * function. 197 */ 198 void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) 199 { 200 if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf)) 201 return; 202 203 mutex_lock(&epf_pf->lock); 204 clear_bit(epf_vf->vfunc_no, &epf_pf->vfunction_num_map); 205 epf_vf->epf_pf = NULL; 206 list_del(&epf_vf->list); 207 mutex_unlock(&epf_pf->lock); 208 } 209 EXPORT_SYMBOL_GPL(pci_epf_remove_vepf); 210 211 /** 212 * pci_epf_free_space() - free the allocated PCI EPF register space 213 * @epf: the EPF device from whom to free the memory 214 * @addr: the virtual address of the PCI EPF register space 215 * @bar: the BAR number corresponding to the register space 216 * @type: Identifies if the allocated space is for primary EPC or secondary EPC 217 * 218 * Invoke to free the allocated PCI EPF register space. 219 */ 220 void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, 221 enum pci_epc_interface_type type) 222 { 223 struct device *dev; 224 struct pci_epf_bar *epf_bar; 225 struct pci_epc *epc; 226 227 if (!addr) 228 return; 229 230 if (type == PRIMARY_INTERFACE) { 231 epc = epf->epc; 232 epf_bar = epf->bar; 233 } else { 234 epc = epf->sec_epc; 235 epf_bar = epf->sec_epc_bar; 236 } 237 238 dev = epc->dev.parent; 239 dma_free_coherent(dev, epf_bar[bar].aligned_size, addr, 240 epf_bar[bar].phys_addr); 241 242 epf_bar[bar].phys_addr = 0; 243 epf_bar[bar].addr = NULL; 244 epf_bar[bar].size = 0; 245 epf_bar[bar].aligned_size = 0; 246 epf_bar[bar].barno = 0; 247 epf_bar[bar].flags = 0; 248 } 249 EXPORT_SYMBOL_GPL(pci_epf_free_space); 250 251 /** 252 * pci_epf_alloc_space() - allocate memory for the PCI EPF register space 253 * @epf: the EPF device to whom allocate the memory 254 * @size: the size of the memory that has to be allocated 255 * @bar: the BAR number corresponding to the allocated register space 256 * @epc_features: the features provided by the EPC specific to this EPF 257 * @type: Identifies if the allocation is for primary EPC or secondary EPC 258 * 259 * Invoke to allocate memory for the PCI EPF register space. 260 * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR 261 * can only be a 64-bit BAR, or if the requested size is larger than 2 GB. 262 */ 263 void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, 264 const struct pci_epc_features *epc_features, 265 enum pci_epc_interface_type type) 266 { 267 u64 bar_fixed_size = epc_features->bar[bar].fixed_size; 268 size_t aligned_size, align = epc_features->align; 269 struct pci_epf_bar *epf_bar; 270 dma_addr_t phys_addr; 271 struct pci_epc *epc; 272 struct device *dev; 273 void *space; 274 275 if (size < 128) 276 size = 128; 277 278 /* According to PCIe base spec, min size for a resizable BAR is 1 MB. */ 279 if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M) 280 size = SZ_1M; 281 282 if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) { 283 if (size > bar_fixed_size) { 284 dev_err(&epf->dev, 285 "requested BAR size is larger than fixed size\n"); 286 return NULL; 287 } 288 size = bar_fixed_size; 289 } else { 290 /* BAR size must be power of two */ 291 size = roundup_pow_of_two(size); 292 } 293 294 /* 295 * Allocate enough memory to accommodate the iATU alignment 296 * requirement. In most cases, this will be the same as .size but 297 * it might be different if, for example, the fixed size of a BAR 298 * is smaller than align. 299 */ 300 aligned_size = align ? ALIGN(size, align) : size; 301 302 if (type == PRIMARY_INTERFACE) { 303 epc = epf->epc; 304 epf_bar = epf->bar; 305 } else { 306 epc = epf->sec_epc; 307 epf_bar = epf->sec_epc_bar; 308 } 309 310 dev = epc->dev.parent; 311 space = dma_alloc_coherent(dev, aligned_size, &phys_addr, GFP_KERNEL); 312 if (!space) { 313 dev_err(dev, "failed to allocate mem space\n"); 314 return NULL; 315 } 316 317 epf_bar[bar].phys_addr = phys_addr; 318 epf_bar[bar].addr = space; 319 epf_bar[bar].size = size; 320 epf_bar[bar].aligned_size = aligned_size; 321 epf_bar[bar].barno = bar; 322 if (upper_32_bits(size) || epc_features->bar[bar].only_64bit) 323 epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; 324 else 325 epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32; 326 327 return space; 328 } 329 EXPORT_SYMBOL_GPL(pci_epf_alloc_space); 330 331 static void pci_epf_remove_cfs(struct pci_epf_driver *driver) 332 { 333 struct config_group *group, *tmp; 334 335 if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS)) 336 return; 337 338 mutex_lock(&pci_epf_mutex); 339 list_for_each_entry_safe(group, tmp, &driver->epf_group, group_entry) 340 pci_ep_cfs_remove_epf_group(group); 341 list_del(&driver->epf_group); 342 mutex_unlock(&pci_epf_mutex); 343 } 344 345 /** 346 * pci_epf_unregister_driver() - unregister the PCI EPF driver 347 * @driver: the PCI EPF driver that has to be unregistered 348 * 349 * Invoke to unregister the PCI EPF driver. 350 */ 351 void pci_epf_unregister_driver(struct pci_epf_driver *driver) 352 { 353 pci_epf_remove_cfs(driver); 354 driver_unregister(&driver->driver); 355 } 356 EXPORT_SYMBOL_GPL(pci_epf_unregister_driver); 357 358 static int pci_epf_add_cfs(struct pci_epf_driver *driver) 359 { 360 struct config_group *group; 361 const struct pci_epf_device_id *id; 362 363 if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS)) 364 return 0; 365 366 INIT_LIST_HEAD(&driver->epf_group); 367 368 id = driver->id_table; 369 while (id->name[0]) { 370 group = pci_ep_cfs_add_epf_group(id->name); 371 if (IS_ERR(group)) { 372 pci_epf_remove_cfs(driver); 373 return PTR_ERR(group); 374 } 375 376 mutex_lock(&pci_epf_mutex); 377 list_add_tail(&group->group_entry, &driver->epf_group); 378 mutex_unlock(&pci_epf_mutex); 379 id++; 380 } 381 382 return 0; 383 } 384 385 /** 386 * __pci_epf_register_driver() - register a new PCI EPF driver 387 * @driver: structure representing PCI EPF driver 388 * @owner: the owner of the module that registers the PCI EPF driver 389 * 390 * Invoke to register a new PCI EPF driver. 391 */ 392 int __pci_epf_register_driver(struct pci_epf_driver *driver, 393 struct module *owner) 394 { 395 int ret; 396 397 if (!driver->ops) 398 return -EINVAL; 399 400 if (!driver->ops->bind || !driver->ops->unbind) 401 return -EINVAL; 402 403 driver->driver.bus = &pci_epf_bus_type; 404 driver->driver.owner = owner; 405 406 ret = driver_register(&driver->driver); 407 if (ret) 408 return ret; 409 410 pci_epf_add_cfs(driver); 411 412 return 0; 413 } 414 EXPORT_SYMBOL_GPL(__pci_epf_register_driver); 415 416 /** 417 * pci_epf_destroy() - destroy the created PCI EPF device 418 * @epf: the PCI EPF device that has to be destroyed. 419 * 420 * Invoke to destroy the PCI EPF device created by invoking pci_epf_create(). 421 */ 422 void pci_epf_destroy(struct pci_epf *epf) 423 { 424 device_unregister(&epf->dev); 425 } 426 EXPORT_SYMBOL_GPL(pci_epf_destroy); 427 428 /** 429 * pci_epf_create() - create a new PCI EPF device 430 * @name: the name of the PCI EPF device. This name will be used to bind the 431 * EPF device to a EPF driver 432 * 433 * Invoke to create a new PCI EPF device by providing the name of the function 434 * device. 435 */ 436 struct pci_epf *pci_epf_create(const char *name) 437 { 438 int ret; 439 struct pci_epf *epf; 440 struct device *dev; 441 int len; 442 443 epf = kzalloc(sizeof(*epf), GFP_KERNEL); 444 if (!epf) 445 return ERR_PTR(-ENOMEM); 446 447 len = strchrnul(name, '.') - name; 448 epf->name = kstrndup(name, len, GFP_KERNEL); 449 if (!epf->name) { 450 kfree(epf); 451 return ERR_PTR(-ENOMEM); 452 } 453 454 /* VFs are numbered starting with 1. So set BIT(0) by default */ 455 epf->vfunction_num_map = 1; 456 INIT_LIST_HEAD(&epf->pci_vepf); 457 458 dev = &epf->dev; 459 device_initialize(dev); 460 dev->bus = &pci_epf_bus_type; 461 dev->type = &pci_epf_type; 462 mutex_init(&epf->lock); 463 464 ret = dev_set_name(dev, "%s", name); 465 if (ret) { 466 put_device(dev); 467 return ERR_PTR(ret); 468 } 469 470 ret = device_add(dev); 471 if (ret) { 472 put_device(dev); 473 return ERR_PTR(ret); 474 } 475 476 return epf; 477 } 478 EXPORT_SYMBOL_GPL(pci_epf_create); 479 480 static void pci_epf_dev_release(struct device *dev) 481 { 482 struct pci_epf *epf = to_pci_epf(dev); 483 484 kfree(epf->name); 485 kfree(epf); 486 } 487 488 static const struct device_type pci_epf_type = { 489 .release = pci_epf_dev_release, 490 }; 491 492 static const struct pci_epf_device_id * 493 pci_epf_match_id(const struct pci_epf_device_id *id, const struct pci_epf *epf) 494 { 495 while (id->name[0]) { 496 if (strcmp(epf->name, id->name) == 0) 497 return id; 498 id++; 499 } 500 501 return NULL; 502 } 503 504 static int pci_epf_device_match(struct device *dev, const struct device_driver *drv) 505 { 506 struct pci_epf *epf = to_pci_epf(dev); 507 const struct pci_epf_driver *driver = to_pci_epf_driver(drv); 508 509 if (driver->id_table) 510 return !!pci_epf_match_id(driver->id_table, epf); 511 512 return !strcmp(epf->name, drv->name); 513 } 514 515 static int pci_epf_device_probe(struct device *dev) 516 { 517 struct pci_epf *epf = to_pci_epf(dev); 518 struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver); 519 520 if (!driver->probe) 521 return -ENODEV; 522 523 epf->driver = driver; 524 525 return driver->probe(epf, pci_epf_match_id(driver->id_table, epf)); 526 } 527 528 static void pci_epf_device_remove(struct device *dev) 529 { 530 struct pci_epf *epf = to_pci_epf(dev); 531 struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver); 532 533 if (driver->remove) 534 driver->remove(epf); 535 epf->driver = NULL; 536 } 537 538 static const struct bus_type pci_epf_bus_type = { 539 .name = "pci-epf", 540 .match = pci_epf_device_match, 541 .probe = pci_epf_device_probe, 542 .remove = pci_epf_device_remove, 543 }; 544 545 static int __init pci_epf_init(void) 546 { 547 int ret; 548 549 ret = bus_register(&pci_epf_bus_type); 550 if (ret) { 551 pr_err("failed to register pci epf bus --> %d\n", ret); 552 return ret; 553 } 554 555 return 0; 556 } 557 module_init(pci_epf_init); 558 559 static void __exit pci_epf_exit(void) 560 { 561 bus_unregister(&pci_epf_bus_type); 562 } 563 module_exit(pci_epf_exit); 564 565 MODULE_DESCRIPTION("PCI EPF Library"); 566 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 567