1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Single Root IOV 1.0 5 * Address Translation Service 1.0 6 * 7 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 8 */ 9 10 #include <linux/pci.h> 11 #include <linux/slab.h> 12 #include <linux/export.h> 13 #include <linux/string.h> 14 #include <linux/delay.h> 15 #include "pci.h" 16 17 #define VIRTFN_ID_LEN 17 /* "virtfn%u\0" for 2^32 - 1 */ 18 19 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id) 20 { 21 if (!dev->is_physfn) 22 return -EINVAL; 23 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 24 dev->sriov->stride * vf_id) >> 8); 25 } 26 27 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) 28 { 29 if (!dev->is_physfn) 30 return -EINVAL; 31 return (dev->devfn + dev->sriov->offset + 32 dev->sriov->stride * vf_id) & 0xff; 33 } 34 EXPORT_SYMBOL_GPL(pci_iov_virtfn_devfn); 35 36 int pci_iov_vf_id(struct pci_dev *dev) 37 { 38 struct pci_dev *pf; 39 40 if (!dev->is_virtfn) 41 return -EINVAL; 42 43 pf = pci_physfn(dev); 44 return (pci_dev_id(dev) - (pci_dev_id(pf) + pf->sriov->offset)) / 45 pf->sriov->stride; 46 } 47 EXPORT_SYMBOL_GPL(pci_iov_vf_id); 48 49 /** 50 * pci_iov_get_pf_drvdata - Return the drvdata of a PF 51 * @dev: VF pci_dev 52 * @pf_driver: Device driver required to own the PF 53 * 54 * This must be called from a context that ensures that a VF driver is attached. 55 * The value returned is invalid once the VF driver completes its remove() 56 * callback. 57 * 58 * Locking is achieved by the driver core. A VF driver cannot be probed until 59 * pci_enable_sriov() is called and pci_disable_sriov() does not return until 60 * all VF drivers have completed their remove(). 61 * 62 * The PF driver must call pci_disable_sriov() before it begins to destroy the 63 * drvdata. 64 */ 65 void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver) 66 { 67 struct pci_dev *pf_dev; 68 69 if (!dev->is_virtfn) 70 return ERR_PTR(-EINVAL); 71 pf_dev = dev->physfn; 72 if (pf_dev->driver != pf_driver) 73 return ERR_PTR(-EINVAL); 74 return pci_get_drvdata(pf_dev); 75 } 76 EXPORT_SYMBOL_GPL(pci_iov_get_pf_drvdata); 77 78 /* 79 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may 80 * change when NumVFs changes. 81 * 82 * Update iov->offset and iov->stride when NumVFs is written. 83 */ 84 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn) 85 { 86 struct pci_sriov *iov = dev->sriov; 87 88 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 89 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset); 90 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride); 91 } 92 93 /* 94 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride 95 * determine how many additional bus numbers will be consumed by VFs. 96 * 97 * Iterate over all valid NumVFs, validate offset and stride, and calculate 98 * the maximum number of bus numbers that could ever be required. 99 */ 100 static int compute_max_vf_buses(struct pci_dev *dev) 101 { 102 struct pci_sriov *iov = dev->sriov; 103 int nr_virtfn, busnr, rc = 0; 104 105 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) { 106 pci_iov_set_numvfs(dev, nr_virtfn); 107 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) { 108 rc = -EIO; 109 goto out; 110 } 111 112 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 113 if (busnr > iov->max_VF_buses) 114 iov->max_VF_buses = busnr; 115 } 116 117 out: 118 pci_iov_set_numvfs(dev, 0); 119 return rc; 120 } 121 122 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 123 { 124 struct pci_bus *child; 125 126 if (bus->number == busnr) 127 return bus; 128 129 child = pci_find_bus(pci_domain_nr(bus), busnr); 130 if (child) 131 return child; 132 133 child = pci_add_new_bus(bus, NULL, busnr); 134 if (!child) 135 return NULL; 136 137 pci_bus_insert_busn_res(child, busnr, busnr); 138 139 return child; 140 } 141 142 static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus) 143 { 144 if (physbus != virtbus && list_empty(&virtbus->devices)) 145 pci_remove_bus(virtbus); 146 } 147 148 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) 149 { 150 if (!dev->is_physfn) 151 return 0; 152 153 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES]; 154 } 155 156 static void pci_read_vf_config_common(struct pci_dev *virtfn) 157 { 158 struct pci_dev *physfn = virtfn->physfn; 159 160 /* 161 * Some config registers are the same across all associated VFs. 162 * Read them once from VF0 so we can skip reading them from the 163 * other VFs. 164 * 165 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to 166 * have the same Revision ID and Subsystem ID, but we assume they 167 * do. 168 */ 169 pci_read_config_dword(virtfn, PCI_CLASS_REVISION, 170 &physfn->sriov->class); 171 pci_read_config_byte(virtfn, PCI_HEADER_TYPE, 172 &physfn->sriov->hdr_type); 173 pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID, 174 &physfn->sriov->subsystem_vendor); 175 pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID, 176 &physfn->sriov->subsystem_device); 177 } 178 179 int pci_iov_sysfs_link(struct pci_dev *dev, 180 struct pci_dev *virtfn, int id) 181 { 182 char buf[VIRTFN_ID_LEN]; 183 int rc; 184 185 sprintf(buf, "virtfn%u", id); 186 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 187 if (rc) 188 goto failed; 189 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 190 if (rc) 191 goto failed1; 192 193 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 194 195 return 0; 196 197 failed1: 198 sysfs_remove_link(&dev->dev.kobj, buf); 199 failed: 200 return rc; 201 } 202 203 #ifdef CONFIG_PCI_MSI 204 static ssize_t sriov_vf_total_msix_show(struct device *dev, 205 struct device_attribute *attr, 206 char *buf) 207 { 208 struct pci_dev *pdev = to_pci_dev(dev); 209 u32 vf_total_msix = 0; 210 211 device_lock(dev); 212 if (!pdev->driver || !pdev->driver->sriov_get_vf_total_msix) 213 goto unlock; 214 215 vf_total_msix = pdev->driver->sriov_get_vf_total_msix(pdev); 216 unlock: 217 device_unlock(dev); 218 return sysfs_emit(buf, "%u\n", vf_total_msix); 219 } 220 static DEVICE_ATTR_RO(sriov_vf_total_msix); 221 222 static ssize_t sriov_vf_msix_count_store(struct device *dev, 223 struct device_attribute *attr, 224 const char *buf, size_t count) 225 { 226 struct pci_dev *vf_dev = to_pci_dev(dev); 227 struct pci_dev *pdev = pci_physfn(vf_dev); 228 int val, ret = 0; 229 230 if (kstrtoint(buf, 0, &val) < 0) 231 return -EINVAL; 232 233 if (val < 0) 234 return -EINVAL; 235 236 device_lock(&pdev->dev); 237 if (!pdev->driver || !pdev->driver->sriov_set_msix_vec_count) { 238 ret = -EOPNOTSUPP; 239 goto err_pdev; 240 } 241 242 device_lock(&vf_dev->dev); 243 if (vf_dev->driver) { 244 /* 245 * A driver is already attached to this VF and has configured 246 * itself based on the current MSI-X vector count. Changing 247 * the vector size could mess up the driver, so block it. 248 */ 249 ret = -EBUSY; 250 goto err_dev; 251 } 252 253 ret = pdev->driver->sriov_set_msix_vec_count(vf_dev, val); 254 255 err_dev: 256 device_unlock(&vf_dev->dev); 257 err_pdev: 258 device_unlock(&pdev->dev); 259 return ret ? : count; 260 } 261 static DEVICE_ATTR_WO(sriov_vf_msix_count); 262 #endif 263 264 static struct attribute *sriov_vf_dev_attrs[] = { 265 #ifdef CONFIG_PCI_MSI 266 &dev_attr_sriov_vf_msix_count.attr, 267 #endif 268 NULL, 269 }; 270 271 static umode_t sriov_vf_attrs_are_visible(struct kobject *kobj, 272 struct attribute *a, int n) 273 { 274 struct device *dev = kobj_to_dev(kobj); 275 struct pci_dev *pdev = to_pci_dev(dev); 276 277 if (!pdev->is_virtfn) 278 return 0; 279 280 return a->mode; 281 } 282 283 const struct attribute_group sriov_vf_dev_attr_group = { 284 .attrs = sriov_vf_dev_attrs, 285 .is_visible = sriov_vf_attrs_are_visible, 286 }; 287 288 static struct pci_dev *pci_iov_scan_device(struct pci_dev *dev, int id, 289 struct pci_bus *bus) 290 { 291 struct pci_sriov *iov = dev->sriov; 292 struct pci_dev *virtfn; 293 int rc; 294 295 virtfn = pci_alloc_dev(bus); 296 if (!virtfn) 297 return ERR_PTR(-ENOMEM); 298 299 virtfn->devfn = pci_iov_virtfn_devfn(dev, id); 300 virtfn->vendor = dev->vendor; 301 virtfn->device = iov->vf_device; 302 virtfn->is_virtfn = 1; 303 virtfn->physfn = pci_dev_get(dev); 304 virtfn->no_command_memory = 1; 305 306 if (id == 0) 307 pci_read_vf_config_common(virtfn); 308 309 rc = pci_setup_device(virtfn); 310 if (rc) { 311 pci_dev_put(dev); 312 pci_bus_put(virtfn->bus); 313 kfree(virtfn); 314 return ERR_PTR(rc); 315 } 316 317 return virtfn; 318 } 319 320 int pci_iov_add_virtfn(struct pci_dev *dev, int id) 321 { 322 struct pci_bus *bus; 323 struct pci_dev *virtfn; 324 struct resource *res; 325 int rc, i; 326 u64 size; 327 328 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id)); 329 if (!bus) { 330 rc = -ENOMEM; 331 goto failed; 332 } 333 334 virtfn = pci_iov_scan_device(dev, id, bus); 335 if (IS_ERR(virtfn)) { 336 rc = PTR_ERR(virtfn); 337 goto failed0; 338 } 339 340 virtfn->dev.parent = dev->dev.parent; 341 virtfn->multifunction = 0; 342 343 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 344 res = &dev->resource[i + PCI_IOV_RESOURCES]; 345 if (!res->parent) 346 continue; 347 virtfn->resource[i].name = pci_name(virtfn); 348 virtfn->resource[i].flags = res->flags; 349 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); 350 resource_set_range(&virtfn->resource[i], 351 res->start + size * id, size); 352 rc = request_resource(res, &virtfn->resource[i]); 353 BUG_ON(rc); 354 } 355 356 pci_device_add(virtfn, virtfn->bus); 357 rc = pci_iov_sysfs_link(dev, virtfn, id); 358 if (rc) 359 goto failed1; 360 361 pci_bus_add_device(virtfn); 362 363 return 0; 364 365 failed1: 366 pci_stop_and_remove_bus_device(virtfn); 367 pci_dev_put(dev); 368 failed0: 369 virtfn_remove_bus(dev->bus, bus); 370 failed: 371 372 return rc; 373 } 374 375 void pci_iov_remove_virtfn(struct pci_dev *dev, int id) 376 { 377 char buf[VIRTFN_ID_LEN]; 378 struct pci_dev *virtfn; 379 380 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus), 381 pci_iov_virtfn_bus(dev, id), 382 pci_iov_virtfn_devfn(dev, id)); 383 if (!virtfn) 384 return; 385 386 sprintf(buf, "virtfn%u", id); 387 sysfs_remove_link(&dev->dev.kobj, buf); 388 /* 389 * pci_stop_dev() could have been called for this virtfn already, 390 * so the directory for the virtfn may have been removed before. 391 * Double check to avoid spurious sysfs warnings. 392 */ 393 if (virtfn->dev.kobj.sd) 394 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 395 396 pci_stop_and_remove_bus_device(virtfn); 397 virtfn_remove_bus(dev->bus, virtfn->bus); 398 399 /* balance pci_get_domain_bus_and_slot() */ 400 pci_dev_put(virtfn); 401 pci_dev_put(dev); 402 } 403 404 static ssize_t sriov_totalvfs_show(struct device *dev, 405 struct device_attribute *attr, 406 char *buf) 407 { 408 struct pci_dev *pdev = to_pci_dev(dev); 409 410 return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); 411 } 412 413 static ssize_t sriov_numvfs_show(struct device *dev, 414 struct device_attribute *attr, 415 char *buf) 416 { 417 struct pci_dev *pdev = to_pci_dev(dev); 418 u16 num_vfs; 419 420 /* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */ 421 device_lock(&pdev->dev); 422 num_vfs = pdev->sriov->num_VFs; 423 device_unlock(&pdev->dev); 424 425 return sysfs_emit(buf, "%u\n", num_vfs); 426 } 427 428 /* 429 * num_vfs > 0; number of VFs to enable 430 * num_vfs = 0; disable all VFs 431 * 432 * Note: SRIOV spec does not allow partial VF 433 * disable, so it's all or none. 434 */ 435 static ssize_t sriov_numvfs_store(struct device *dev, 436 struct device_attribute *attr, 437 const char *buf, size_t count) 438 { 439 struct pci_dev *pdev = to_pci_dev(dev); 440 int ret = 0; 441 u16 num_vfs; 442 443 if (kstrtou16(buf, 0, &num_vfs) < 0) 444 return -EINVAL; 445 446 if (num_vfs > pci_sriov_get_totalvfs(pdev)) 447 return -ERANGE; 448 449 device_lock(&pdev->dev); 450 451 if (num_vfs == pdev->sriov->num_VFs) 452 goto exit; 453 454 /* is PF driver loaded */ 455 if (!pdev->driver) { 456 pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n"); 457 ret = -ENOENT; 458 goto exit; 459 } 460 461 /* is PF driver loaded w/callback */ 462 if (!pdev->driver->sriov_configure) { 463 pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n"); 464 ret = -ENOENT; 465 goto exit; 466 } 467 468 if (num_vfs == 0) { 469 /* disable VFs */ 470 ret = pdev->driver->sriov_configure(pdev, 0); 471 goto exit; 472 } 473 474 /* enable VFs */ 475 if (pdev->sriov->num_VFs) { 476 pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n", 477 pdev->sriov->num_VFs, num_vfs); 478 ret = -EBUSY; 479 goto exit; 480 } 481 482 ret = pdev->driver->sriov_configure(pdev, num_vfs); 483 if (ret < 0) 484 goto exit; 485 486 if (ret != num_vfs) 487 pci_warn(pdev, "%d VFs requested; only %d enabled\n", 488 num_vfs, ret); 489 490 exit: 491 device_unlock(&pdev->dev); 492 493 if (ret < 0) 494 return ret; 495 496 return count; 497 } 498 499 static ssize_t sriov_offset_show(struct device *dev, 500 struct device_attribute *attr, 501 char *buf) 502 { 503 struct pci_dev *pdev = to_pci_dev(dev); 504 505 return sysfs_emit(buf, "%u\n", pdev->sriov->offset); 506 } 507 508 static ssize_t sriov_stride_show(struct device *dev, 509 struct device_attribute *attr, 510 char *buf) 511 { 512 struct pci_dev *pdev = to_pci_dev(dev); 513 514 return sysfs_emit(buf, "%u\n", pdev->sriov->stride); 515 } 516 517 static ssize_t sriov_vf_device_show(struct device *dev, 518 struct device_attribute *attr, 519 char *buf) 520 { 521 struct pci_dev *pdev = to_pci_dev(dev); 522 523 return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device); 524 } 525 526 static ssize_t sriov_drivers_autoprobe_show(struct device *dev, 527 struct device_attribute *attr, 528 char *buf) 529 { 530 struct pci_dev *pdev = to_pci_dev(dev); 531 532 return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe); 533 } 534 535 static ssize_t sriov_drivers_autoprobe_store(struct device *dev, 536 struct device_attribute *attr, 537 const char *buf, size_t count) 538 { 539 struct pci_dev *pdev = to_pci_dev(dev); 540 bool drivers_autoprobe; 541 542 if (kstrtobool(buf, &drivers_autoprobe) < 0) 543 return -EINVAL; 544 545 pdev->sriov->drivers_autoprobe = drivers_autoprobe; 546 547 return count; 548 } 549 550 static DEVICE_ATTR_RO(sriov_totalvfs); 551 static DEVICE_ATTR_RW(sriov_numvfs); 552 static DEVICE_ATTR_RO(sriov_offset); 553 static DEVICE_ATTR_RO(sriov_stride); 554 static DEVICE_ATTR_RO(sriov_vf_device); 555 static DEVICE_ATTR_RW(sriov_drivers_autoprobe); 556 557 static struct attribute *sriov_pf_dev_attrs[] = { 558 &dev_attr_sriov_totalvfs.attr, 559 &dev_attr_sriov_numvfs.attr, 560 &dev_attr_sriov_offset.attr, 561 &dev_attr_sriov_stride.attr, 562 &dev_attr_sriov_vf_device.attr, 563 &dev_attr_sriov_drivers_autoprobe.attr, 564 #ifdef CONFIG_PCI_MSI 565 &dev_attr_sriov_vf_total_msix.attr, 566 #endif 567 NULL, 568 }; 569 570 static umode_t sriov_pf_attrs_are_visible(struct kobject *kobj, 571 struct attribute *a, int n) 572 { 573 struct device *dev = kobj_to_dev(kobj); 574 575 if (!dev_is_pf(dev)) 576 return 0; 577 578 return a->mode; 579 } 580 581 const struct attribute_group sriov_pf_dev_attr_group = { 582 .attrs = sriov_pf_dev_attrs, 583 .is_visible = sriov_pf_attrs_are_visible, 584 }; 585 586 int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) 587 { 588 return 0; 589 } 590 591 int __weak pcibios_sriov_disable(struct pci_dev *pdev) 592 { 593 return 0; 594 } 595 596 static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) 597 { 598 unsigned int i; 599 int rc; 600 601 if (dev->no_vf_scan) 602 return 0; 603 604 for (i = 0; i < num_vfs; i++) { 605 rc = pci_iov_add_virtfn(dev, i); 606 if (rc) 607 goto failed; 608 } 609 return 0; 610 failed: 611 while (i--) 612 pci_iov_remove_virtfn(dev, i); 613 614 return rc; 615 } 616 617 static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 618 { 619 int rc; 620 int i; 621 int nres; 622 u16 initial; 623 struct resource *res; 624 struct pci_dev *pdev; 625 struct pci_sriov *iov = dev->sriov; 626 int bars = 0; 627 int bus; 628 629 if (!nr_virtfn) 630 return 0; 631 632 if (iov->num_VFs) 633 return -EINVAL; 634 635 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 636 if (initial > iov->total_VFs || 637 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs))) 638 return -EIO; 639 640 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs || 641 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 642 return -EINVAL; 643 644 nres = 0; 645 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 646 bars |= (1 << (i + PCI_IOV_RESOURCES)); 647 res = &dev->resource[i + PCI_IOV_RESOURCES]; 648 if (res->parent) 649 nres++; 650 } 651 if (nres != iov->nres) { 652 pci_err(dev, "not enough MMIO resources for SR-IOV\n"); 653 return -ENOMEM; 654 } 655 656 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 657 if (bus > dev->bus->busn_res.end) { 658 pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n", 659 nr_virtfn, bus, &dev->bus->busn_res); 660 return -ENOMEM; 661 } 662 663 if (pci_enable_resources(dev, bars)) { 664 pci_err(dev, "SR-IOV: IOV BARS not allocated\n"); 665 return -ENOMEM; 666 } 667 668 if (iov->link != dev->devfn) { 669 pdev = pci_get_slot(dev->bus, iov->link); 670 if (!pdev) 671 return -ENODEV; 672 673 if (!pdev->is_physfn) { 674 pci_dev_put(pdev); 675 return -ENOSYS; 676 } 677 678 rc = sysfs_create_link(&dev->dev.kobj, 679 &pdev->dev.kobj, "dep_link"); 680 pci_dev_put(pdev); 681 if (rc) 682 return rc; 683 } 684 685 iov->initial_VFs = initial; 686 if (nr_virtfn < initial) 687 initial = nr_virtfn; 688 689 rc = pcibios_sriov_enable(dev, initial); 690 if (rc) { 691 pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc); 692 goto err_pcibios; 693 } 694 695 pci_iov_set_numvfs(dev, nr_virtfn); 696 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 697 pci_cfg_access_lock(dev); 698 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 699 msleep(100); 700 pci_cfg_access_unlock(dev); 701 702 rc = sriov_add_vfs(dev, initial); 703 if (rc) 704 goto err_pcibios; 705 706 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 707 iov->num_VFs = nr_virtfn; 708 709 return 0; 710 711 err_pcibios: 712 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 713 pci_cfg_access_lock(dev); 714 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 715 ssleep(1); 716 pci_cfg_access_unlock(dev); 717 718 pcibios_sriov_disable(dev); 719 720 if (iov->link != dev->devfn) 721 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 722 723 pci_iov_set_numvfs(dev, 0); 724 return rc; 725 } 726 727 static void sriov_del_vfs(struct pci_dev *dev) 728 { 729 struct pci_sriov *iov = dev->sriov; 730 int i; 731 732 for (i = 0; i < iov->num_VFs; i++) 733 pci_iov_remove_virtfn(dev, i); 734 } 735 736 static void sriov_disable(struct pci_dev *dev) 737 { 738 struct pci_sriov *iov = dev->sriov; 739 740 if (!iov->num_VFs) 741 return; 742 743 sriov_del_vfs(dev); 744 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 745 pci_cfg_access_lock(dev); 746 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 747 ssleep(1); 748 pci_cfg_access_unlock(dev); 749 750 pcibios_sriov_disable(dev); 751 752 if (iov->link != dev->devfn) 753 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 754 755 iov->num_VFs = 0; 756 pci_iov_set_numvfs(dev, 0); 757 } 758 759 static int sriov_init(struct pci_dev *dev, int pos) 760 { 761 int i, bar64; 762 int rc; 763 int nres; 764 u32 pgsz; 765 u16 ctrl, total; 766 struct pci_sriov *iov; 767 struct resource *res; 768 const char *res_name; 769 struct pci_dev *pdev; 770 u32 sriovbars[PCI_SRIOV_NUM_BARS]; 771 772 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 773 if (ctrl & PCI_SRIOV_CTRL_VFE) { 774 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 775 ssleep(1); 776 } 777 778 ctrl = 0; 779 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 780 if (pdev->is_physfn) 781 goto found; 782 783 pdev = NULL; 784 if (pci_ari_enabled(dev->bus)) 785 ctrl |= PCI_SRIOV_CTRL_ARI; 786 787 found: 788 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 789 790 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 791 if (!total) 792 return 0; 793 794 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 795 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 796 pgsz &= ~((1 << i) - 1); 797 if (!pgsz) 798 return -EIO; 799 800 pgsz &= ~(pgsz - 1); 801 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 802 803 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 804 if (!iov) 805 return -ENOMEM; 806 807 /* Sizing SR-IOV BARs with VF Enable cleared - no decode */ 808 __pci_size_stdbars(dev, PCI_SRIOV_NUM_BARS, 809 pos + PCI_SRIOV_BAR, sriovbars); 810 811 nres = 0; 812 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 813 res = &dev->resource[i + PCI_IOV_RESOURCES]; 814 res_name = pci_resource_name(dev, i + PCI_IOV_RESOURCES); 815 816 /* 817 * If it is already FIXED, don't change it, something 818 * (perhaps EA or header fixups) wants it this way. 819 */ 820 if (res->flags & IORESOURCE_PCI_FIXED) 821 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0; 822 else 823 bar64 = __pci_read_base(dev, pci_bar_unknown, res, 824 pos + PCI_SRIOV_BAR + i * 4, 825 &sriovbars[i]); 826 if (!res->flags) 827 continue; 828 if (resource_size(res) & (PAGE_SIZE - 1)) { 829 rc = -EIO; 830 goto failed; 831 } 832 iov->barsz[i] = resource_size(res); 833 resource_set_size(res, resource_size(res) * total); 834 pci_info(dev, "%s %pR: contains BAR %d for %d VFs\n", 835 res_name, res, i, total); 836 i += bar64; 837 nres++; 838 } 839 840 iov->pos = pos; 841 iov->nres = nres; 842 iov->ctrl = ctrl; 843 iov->total_VFs = total; 844 iov->driver_max_VFs = total; 845 pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); 846 iov->pgsz = pgsz; 847 iov->self = dev; 848 iov->drivers_autoprobe = true; 849 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 850 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 851 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) 852 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 853 854 if (pdev) 855 iov->dev = pci_dev_get(pdev); 856 else 857 iov->dev = dev; 858 859 dev->sriov = iov; 860 dev->is_physfn = 1; 861 rc = compute_max_vf_buses(dev); 862 if (rc) 863 goto fail_max_buses; 864 865 return 0; 866 867 fail_max_buses: 868 dev->sriov = NULL; 869 dev->is_physfn = 0; 870 failed: 871 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 872 res = &dev->resource[i + PCI_IOV_RESOURCES]; 873 res->flags = 0; 874 } 875 876 kfree(iov); 877 return rc; 878 } 879 880 static void sriov_release(struct pci_dev *dev) 881 { 882 BUG_ON(dev->sriov->num_VFs); 883 884 if (dev != dev->sriov->dev) 885 pci_dev_put(dev->sriov->dev); 886 887 kfree(dev->sriov); 888 dev->sriov = NULL; 889 } 890 891 static void sriov_restore_state(struct pci_dev *dev) 892 { 893 int i; 894 u16 ctrl; 895 struct pci_sriov *iov = dev->sriov; 896 897 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 898 if (ctrl & PCI_SRIOV_CTRL_VFE) 899 return; 900 901 /* 902 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because 903 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI. 904 */ 905 ctrl &= ~PCI_SRIOV_CTRL_ARI; 906 ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI; 907 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl); 908 909 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) 910 pci_update_resource(dev, i + PCI_IOV_RESOURCES); 911 912 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 913 pci_iov_set_numvfs(dev, iov->num_VFs); 914 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 915 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 916 msleep(100); 917 } 918 919 /** 920 * pci_iov_init - initialize the IOV capability 921 * @dev: the PCI device 922 * 923 * Returns 0 on success, or negative on failure. 924 */ 925 int pci_iov_init(struct pci_dev *dev) 926 { 927 int pos; 928 929 if (!pci_is_pcie(dev)) 930 return -ENODEV; 931 932 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 933 if (pos) 934 return sriov_init(dev, pos); 935 936 return -ENODEV; 937 } 938 939 /** 940 * pci_iov_release - release resources used by the IOV capability 941 * @dev: the PCI device 942 */ 943 void pci_iov_release(struct pci_dev *dev) 944 { 945 if (dev->is_physfn) 946 sriov_release(dev); 947 } 948 949 /** 950 * pci_iov_remove - clean up SR-IOV state after PF driver is detached 951 * @dev: the PCI device 952 */ 953 void pci_iov_remove(struct pci_dev *dev) 954 { 955 struct pci_sriov *iov = dev->sriov; 956 957 if (!dev->is_physfn) 958 return; 959 960 iov->driver_max_VFs = iov->total_VFs; 961 if (iov->num_VFs) 962 pci_warn(dev, "driver left SR-IOV enabled after remove\n"); 963 } 964 965 /** 966 * pci_iov_update_resource - update a VF BAR 967 * @dev: the PCI device 968 * @resno: the resource number 969 * 970 * Update a VF BAR in the SR-IOV capability of a PF. 971 */ 972 void pci_iov_update_resource(struct pci_dev *dev, int resno) 973 { 974 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; 975 struct resource *res = pci_resource_n(dev, resno); 976 int vf_bar = resno - PCI_IOV_RESOURCES; 977 struct pci_bus_region region; 978 u16 cmd; 979 u32 new; 980 int reg; 981 982 /* 983 * The generic pci_restore_bars() path calls this for all devices, 984 * including VFs and non-SR-IOV devices. If this is not a PF, we 985 * have nothing to do. 986 */ 987 if (!iov) 988 return; 989 990 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); 991 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { 992 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", 993 vf_bar, res); 994 return; 995 } 996 997 /* 998 * Ignore unimplemented BARs, unused resource slots for 64-bit 999 * BARs, and non-movable resources, e.g., those described via 1000 * Enhanced Allocation. 1001 */ 1002 if (!res->flags) 1003 return; 1004 1005 if (res->flags & IORESOURCE_UNSET) 1006 return; 1007 1008 if (res->flags & IORESOURCE_PCI_FIXED) 1009 return; 1010 1011 pcibios_resource_to_bus(dev->bus, ®ion, res); 1012 new = region.start; 1013 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; 1014 1015 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; 1016 pci_write_config_dword(dev, reg, new); 1017 if (res->flags & IORESOURCE_MEM_64) { 1018 new = region.start >> 16 >> 16; 1019 pci_write_config_dword(dev, reg + 4, new); 1020 } 1021 } 1022 1023 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, 1024 int resno) 1025 { 1026 return pci_iov_resource_size(dev, resno); 1027 } 1028 1029 /** 1030 * pci_sriov_resource_alignment - get resource alignment for VF BAR 1031 * @dev: the PCI device 1032 * @resno: the resource number 1033 * 1034 * Returns the alignment of the VF BAR found in the SR-IOV capability. 1035 * This is not the same as the resource size which is defined as 1036 * the VF BAR size multiplied by the number of VFs. The alignment 1037 * is just the VF BAR size. 1038 */ 1039 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno) 1040 { 1041 return pcibios_iov_resource_alignment(dev, resno); 1042 } 1043 1044 /** 1045 * pci_restore_iov_state - restore the state of the IOV capability 1046 * @dev: the PCI device 1047 */ 1048 void pci_restore_iov_state(struct pci_dev *dev) 1049 { 1050 if (dev->is_physfn) 1051 sriov_restore_state(dev); 1052 } 1053 1054 /** 1055 * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs 1056 * @dev: the PCI device 1057 * @auto_probe: set VF drivers auto probe flag 1058 */ 1059 void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe) 1060 { 1061 if (dev->is_physfn) 1062 dev->sriov->drivers_autoprobe = auto_probe; 1063 } 1064 1065 /** 1066 * pci_iov_bus_range - find bus range used by Virtual Function 1067 * @bus: the PCI bus 1068 * 1069 * Returns max number of buses (exclude current one) used by Virtual 1070 * Functions. 1071 */ 1072 int pci_iov_bus_range(struct pci_bus *bus) 1073 { 1074 int max = 0; 1075 struct pci_dev *dev; 1076 1077 list_for_each_entry(dev, &bus->devices, bus_list) { 1078 if (!dev->is_physfn) 1079 continue; 1080 if (dev->sriov->max_VF_buses > max) 1081 max = dev->sriov->max_VF_buses; 1082 } 1083 1084 return max ? max - bus->number : 0; 1085 } 1086 1087 /** 1088 * pci_enable_sriov - enable the SR-IOV capability 1089 * @dev: the PCI device 1090 * @nr_virtfn: number of virtual functions to enable 1091 * 1092 * Returns 0 on success, or negative on failure. 1093 */ 1094 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 1095 { 1096 might_sleep(); 1097 1098 if (!dev->is_physfn) 1099 return -ENOSYS; 1100 1101 return sriov_enable(dev, nr_virtfn); 1102 } 1103 EXPORT_SYMBOL_GPL(pci_enable_sriov); 1104 1105 /** 1106 * pci_disable_sriov - disable the SR-IOV capability 1107 * @dev: the PCI device 1108 */ 1109 void pci_disable_sriov(struct pci_dev *dev) 1110 { 1111 might_sleep(); 1112 1113 if (!dev->is_physfn) 1114 return; 1115 1116 sriov_disable(dev); 1117 } 1118 EXPORT_SYMBOL_GPL(pci_disable_sriov); 1119 1120 /** 1121 * pci_num_vf - return number of VFs associated with a PF device_release_driver 1122 * @dev: the PCI device 1123 * 1124 * Returns number of VFs, or 0 if SR-IOV is not enabled. 1125 */ 1126 int pci_num_vf(struct pci_dev *dev) 1127 { 1128 if (!dev->is_physfn) 1129 return 0; 1130 1131 return dev->sriov->num_VFs; 1132 } 1133 EXPORT_SYMBOL_GPL(pci_num_vf); 1134 1135 /** 1136 * pci_vfs_assigned - returns number of VFs are assigned to a guest 1137 * @dev: the PCI device 1138 * 1139 * Returns number of VFs belonging to this device that are assigned to a guest. 1140 * If device is not a physical function returns 0. 1141 */ 1142 int pci_vfs_assigned(struct pci_dev *dev) 1143 { 1144 struct pci_dev *vfdev; 1145 unsigned int vfs_assigned = 0; 1146 unsigned short dev_id; 1147 1148 /* only search if we are a PF */ 1149 if (!dev->is_physfn) 1150 return 0; 1151 1152 /* 1153 * determine the device ID for the VFs, the vendor ID will be the 1154 * same as the PF so there is no need to check for that one 1155 */ 1156 dev_id = dev->sriov->vf_device; 1157 1158 /* loop through all the VFs to see if we own any that are assigned */ 1159 vfdev = pci_get_device(dev->vendor, dev_id, NULL); 1160 while (vfdev) { 1161 /* 1162 * It is considered assigned if it is a virtual function with 1163 * our dev as the physical function and the assigned bit is set 1164 */ 1165 if (vfdev->is_virtfn && (vfdev->physfn == dev) && 1166 pci_is_dev_assigned(vfdev)) 1167 vfs_assigned++; 1168 1169 vfdev = pci_get_device(dev->vendor, dev_id, vfdev); 1170 } 1171 1172 return vfs_assigned; 1173 } 1174 EXPORT_SYMBOL_GPL(pci_vfs_assigned); 1175 1176 /** 1177 * pci_sriov_set_totalvfs -- reduce the TotalVFs available 1178 * @dev: the PCI PF device 1179 * @numvfs: number that should be used for TotalVFs supported 1180 * 1181 * Should be called from PF driver's probe routine with 1182 * device's mutex held. 1183 * 1184 * Returns 0 if PF is an SRIOV-capable device and 1185 * value of numvfs valid. If not a PF return -ENOSYS; 1186 * if numvfs is invalid return -EINVAL; 1187 * if VFs already enabled, return -EBUSY. 1188 */ 1189 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) 1190 { 1191 if (!dev->is_physfn) 1192 return -ENOSYS; 1193 1194 if (numvfs > dev->sriov->total_VFs) 1195 return -EINVAL; 1196 1197 /* Shouldn't change if VFs already enabled */ 1198 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE) 1199 return -EBUSY; 1200 1201 dev->sriov->driver_max_VFs = numvfs; 1202 return 0; 1203 } 1204 EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); 1205 1206 /** 1207 * pci_sriov_get_totalvfs -- get total VFs supported on this device 1208 * @dev: the PCI PF device 1209 * 1210 * For a PCIe device with SRIOV support, return the PCIe 1211 * SRIOV capability value of TotalVFs or the value of driver_max_VFs 1212 * if the driver reduced it. Otherwise 0. 1213 */ 1214 int pci_sriov_get_totalvfs(struct pci_dev *dev) 1215 { 1216 if (!dev->is_physfn) 1217 return 0; 1218 1219 return dev->sriov->driver_max_VFs; 1220 } 1221 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs); 1222 1223 /** 1224 * pci_sriov_configure_simple - helper to configure SR-IOV 1225 * @dev: the PCI device 1226 * @nr_virtfn: number of virtual functions to enable, 0 to disable 1227 * 1228 * Enable or disable SR-IOV for devices that don't require any PF setup 1229 * before enabling SR-IOV. Return value is negative on error, or number of 1230 * VFs allocated on success. 1231 */ 1232 int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn) 1233 { 1234 int rc; 1235 1236 might_sleep(); 1237 1238 if (!dev->is_physfn) 1239 return -ENODEV; 1240 1241 if (pci_vfs_assigned(dev)) { 1242 pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n"); 1243 return -EPERM; 1244 } 1245 1246 if (nr_virtfn == 0) { 1247 sriov_disable(dev); 1248 return 0; 1249 } 1250 1251 rc = sriov_enable(dev, nr_virtfn); 1252 if (rc < 0) 1253 return rc; 1254 1255 return nr_virtfn; 1256 } 1257 EXPORT_SYMBOL_GPL(pci_sriov_configure_simple); 1258