1 #include "kvm/irq.h" 2 #include "kvm/kvm.h" 3 #include "kvm/kvm-cpu.h" 4 #include "kvm/vfio.h" 5 6 #include <sys/ioctl.h> 7 #include <sys/eventfd.h> 8 #include <sys/resource.h> 9 #include <sys/time.h> 10 11 /* Wrapper around UAPI vfio_irq_set */ 12 union vfio_irq_eventfd { 13 struct vfio_irq_set irq; 14 u8 buffer[sizeof(struct vfio_irq_set) + sizeof(int)]; 15 }; 16 17 static void set_vfio_irq_eventd_payload(union vfio_irq_eventfd *evfd, int fd) 18 { 19 memcpy(&evfd->irq.data, &fd, sizeof(fd)); 20 } 21 22 #define msi_is_enabled(state) ((state) & VFIO_PCI_MSI_STATE_ENABLED) 23 #define msi_is_masked(state) ((state) & VFIO_PCI_MSI_STATE_MASKED) 24 #define msi_is_empty(state) ((state) & VFIO_PCI_MSI_STATE_EMPTY) 25 26 #define msi_update_state(state, val, bit) \ 27 (state) = (val) ? (state) | bit : (state) & ~bit; 28 #define msi_set_enabled(state, val) \ 29 msi_update_state(state, val, VFIO_PCI_MSI_STATE_ENABLED) 30 #define msi_set_masked(state, val) \ 31 msi_update_state(state, val, VFIO_PCI_MSI_STATE_MASKED) 32 #define msi_set_empty(state, val) \ 33 msi_update_state(state, val, VFIO_PCI_MSI_STATE_EMPTY) 34 35 static void vfio_pci_disable_intx(struct kvm *kvm, struct vfio_device *vdev); 36 static int vfio_pci_enable_intx(struct kvm *kvm, struct vfio_device *vdev); 37 38 static int vfio_pci_enable_msis(struct kvm *kvm, struct vfio_device *vdev, 39 bool msix) 40 { 41 size_t i; 42 int ret = 0; 43 int *eventfds; 44 struct vfio_pci_device *pdev = &vdev->pci; 45 struct vfio_pci_msi_common *msis = msix ? &pdev->msix : &pdev->msi; 46 union vfio_irq_eventfd single = { 47 .irq = { 48 .argsz = sizeof(single), 49 .flags = VFIO_IRQ_SET_DATA_EVENTFD | 50 VFIO_IRQ_SET_ACTION_TRIGGER, 51 .index = msis->info.index, 52 .count = 1, 53 }, 54 }; 55 56 if (!msi_is_enabled(msis->virt_state)) 57 return 0; 58 59 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_INTX) 60 /* 61 * PCI (and VFIO) forbids enabling INTx, MSI or MSIX at the same 62 * time. Since INTx has to be enabled from the start (we don't 63 * have a reliable way to know when the guest starts using it), 64 * disable it now. 65 */ 66 vfio_pci_disable_intx(kvm, vdev); 67 68 eventfds = (void *)msis->irq_set + sizeof(struct vfio_irq_set); 69 70 /* 71 * Initial registration of the full range. This enables the physical 72 * MSI/MSI-X capability, which might have desired side effects. For 73 * instance when assigning virtio legacy devices, enabling the MSI 74 * capability modifies the config space layout! 75 * 76 * As an optimization, only update MSIs when guest unmasks the 77 * capability. This greatly reduces the initialization time for Linux 78 * guest with 2048+ MSIs. Linux guest starts by enabling the MSI-X cap 79 * masked, then fills individual vectors, then unmasks the whole 80 * function. So we only do one VFIO ioctl when enabling for the first 81 * time, and then one when unmasking. 82 * 83 * phys_state is empty when it is enabled but no vector has been 84 * registered via SET_IRQS yet. 85 */ 86 if (!msi_is_enabled(msis->phys_state) || 87 (!msi_is_masked(msis->virt_state) && 88 msi_is_empty(msis->phys_state))) { 89 bool empty = true; 90 91 for (i = 0; i < msis->nr_entries; i++) { 92 eventfds[i] = msis->entries[i].gsi >= 0 ? 93 msis->entries[i].eventfd : -1; 94 95 if (eventfds[i] >= 0) 96 empty = false; 97 } 98 99 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, msis->irq_set); 100 if (ret < 0) { 101 perror("VFIO_DEVICE_SET_IRQS(multi)"); 102 return ret; 103 } 104 105 msi_set_enabled(msis->phys_state, true); 106 msi_set_empty(msis->phys_state, empty); 107 108 return 0; 109 } 110 111 if (msi_is_masked(msis->virt_state)) { 112 /* TODO: if phys_state is not empty nor masked, mask all vectors */ 113 return 0; 114 } 115 116 /* Update individual vectors to avoid breaking those in use */ 117 for (i = 0; i < msis->nr_entries; i++) { 118 struct vfio_pci_msi_entry *entry = &msis->entries[i]; 119 int fd = entry->gsi >= 0 ? entry->eventfd : -1; 120 121 if (fd == eventfds[i]) 122 continue; 123 124 single.irq.start = i; 125 set_vfio_irq_eventd_payload(&single, fd); 126 127 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &single); 128 if (ret < 0) { 129 perror("VFIO_DEVICE_SET_IRQS(single)"); 130 break; 131 } 132 133 eventfds[i] = fd; 134 135 if (msi_is_empty(msis->phys_state) && fd >= 0) 136 msi_set_empty(msis->phys_state, false); 137 } 138 139 return ret; 140 } 141 142 static int vfio_pci_disable_msis(struct kvm *kvm, struct vfio_device *vdev, 143 bool msix) 144 { 145 int ret; 146 struct vfio_pci_device *pdev = &vdev->pci; 147 struct vfio_pci_msi_common *msis = msix ? &pdev->msix : &pdev->msi; 148 struct vfio_irq_set irq_set = { 149 .argsz = sizeof(irq_set), 150 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 151 .index = msis->info.index, 152 .start = 0, 153 .count = 0, 154 }; 155 156 if (!msi_is_enabled(msis->phys_state)) 157 return 0; 158 159 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 160 if (ret < 0) { 161 perror("VFIO_DEVICE_SET_IRQS(NONE)"); 162 return ret; 163 } 164 165 msi_set_enabled(msis->phys_state, false); 166 msi_set_empty(msis->phys_state, true); 167 168 /* 169 * When MSI or MSIX is disabled, this might be called when 170 * PCI driver detects the MSI interrupt failure and wants to 171 * rollback to INTx mode. Thus enable INTx if the device 172 * supports INTx mode in this case. 173 */ 174 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_INTX) 175 ret = vfio_pci_enable_intx(kvm, vdev); 176 177 return ret >= 0 ? 0 : ret; 178 } 179 180 static int vfio_pci_update_msi_entry(struct kvm *kvm, struct vfio_device *vdev, 181 struct vfio_pci_msi_entry *entry) 182 { 183 int ret; 184 185 if (entry->eventfd < 0) { 186 entry->eventfd = eventfd(0, 0); 187 if (entry->eventfd < 0) { 188 ret = -errno; 189 vfio_dev_err(vdev, "cannot create eventfd"); 190 return ret; 191 } 192 } 193 194 /* Allocate IRQ if necessary */ 195 if (entry->gsi < 0) { 196 int ret = irq__add_msix_route(kvm, &entry->config.msg, 197 vdev->dev_hdr.dev_num << 3); 198 if (ret < 0) { 199 vfio_dev_err(vdev, "cannot create MSI-X route"); 200 return ret; 201 } 202 entry->gsi = ret; 203 } else { 204 irq__update_msix_route(kvm, entry->gsi, &entry->config.msg); 205 } 206 207 /* 208 * MSI masking is unimplemented in VFIO, so we have to handle it by 209 * disabling/enabling IRQ route instead. We do it on the KVM side rather 210 * than VFIO, because: 211 * - it is 8x faster 212 * - it allows to decouple masking logic from capability state. 213 * - in masked state, after removing irqfd route, we could easily plug 214 * the eventfd in a local handler, in order to serve Pending Bit reads 215 * to the guest. 216 * 217 * So entry->phys_state is masked when there is no active irqfd route. 218 */ 219 if (msi_is_masked(entry->virt_state) == msi_is_masked(entry->phys_state)) 220 return 0; 221 222 if (msi_is_masked(entry->phys_state)) { 223 ret = irq__add_irqfd(kvm, entry->gsi, entry->eventfd, -1); 224 if (ret < 0) { 225 vfio_dev_err(vdev, "cannot setup irqfd"); 226 return ret; 227 } 228 } else { 229 irq__del_irqfd(kvm, entry->gsi, entry->eventfd); 230 } 231 232 msi_set_masked(entry->phys_state, msi_is_masked(entry->virt_state)); 233 234 return 0; 235 } 236 237 static void vfio_pci_msix_pba_access(struct kvm_cpu *vcpu, u64 addr, u8 *data, 238 u32 len, u8 is_write, void *ptr) 239 { 240 struct vfio_pci_device *pdev = ptr; 241 struct vfio_pci_msix_pba *pba = &pdev->msix_pba; 242 u64 offset = addr - pba->guest_phys_addr; 243 struct vfio_device *vdev = container_of(pdev, struct vfio_device, pci); 244 245 if (is_write) 246 return; 247 248 /* 249 * TODO: emulate PBA. Hardware MSI-X is never masked, so reading the PBA 250 * is completely useless here. Note that Linux doesn't use PBA. 251 */ 252 if (pread(vdev->fd, data, len, pba->offset + offset) != (ssize_t)len) 253 vfio_dev_err(vdev, "cannot access MSIX PBA\n"); 254 } 255 256 static void vfio_pci_msix_table_access(struct kvm_cpu *vcpu, u64 addr, u8 *data, 257 u32 len, u8 is_write, void *ptr) 258 { 259 struct kvm *kvm = vcpu->kvm; 260 struct vfio_pci_msi_entry *entry; 261 struct vfio_pci_device *pdev = ptr; 262 struct vfio_device *vdev = container_of(pdev, struct vfio_device, pci); 263 264 u64 offset = addr - pdev->msix_table.guest_phys_addr; 265 266 size_t vector = offset / PCI_MSIX_ENTRY_SIZE; 267 off_t field = offset % PCI_MSIX_ENTRY_SIZE; 268 269 /* 270 * PCI spec says that software must use aligned 4 or 8 bytes accesses 271 * for the MSI-X tables. 272 */ 273 if ((len != 4 && len != 8) || addr & (len - 1)) { 274 vfio_dev_warn(vdev, "invalid MSI-X table access"); 275 return; 276 } 277 278 entry = &pdev->msix.entries[vector]; 279 280 mutex_lock(&pdev->msix.mutex); 281 282 if (!is_write) { 283 memcpy(data, (void *)&entry->config + field, len); 284 goto out_unlock; 285 } 286 287 memcpy((void *)&entry->config + field, data, len); 288 289 /* 290 * Check if access touched the vector control register, which is at the 291 * end of the MSI-X entry. 292 */ 293 if (field + len <= PCI_MSIX_ENTRY_VECTOR_CTRL) 294 goto out_unlock; 295 296 msi_set_masked(entry->virt_state, entry->config.ctrl & 297 PCI_MSIX_ENTRY_CTRL_MASKBIT); 298 299 if (vfio_pci_update_msi_entry(kvm, vdev, entry) < 0) 300 /* Not much we can do here. */ 301 vfio_dev_err(vdev, "failed to configure MSIX vector %zu", vector); 302 303 /* Update the physical capability if necessary */ 304 if (vfio_pci_enable_msis(kvm, vdev, true)) 305 vfio_dev_err(vdev, "cannot enable MSIX"); 306 307 out_unlock: 308 mutex_unlock(&pdev->msix.mutex); 309 } 310 311 static void vfio_pci_msix_cap_write(struct kvm *kvm, 312 struct vfio_device *vdev, u8 off, 313 void *data, int sz) 314 { 315 struct vfio_pci_device *pdev = &vdev->pci; 316 off_t enable_pos = PCI_MSIX_FLAGS + 1; 317 bool enable; 318 u16 flags; 319 320 off -= pdev->msix.pos; 321 322 /* Check if access intersects with the MSI-X Enable bit */ 323 if (off > enable_pos || off + sz <= enable_pos) 324 return; 325 326 /* Read byte that contains the Enable bit */ 327 flags = *(u8 *)(data + enable_pos - off) << 8; 328 329 mutex_lock(&pdev->msix.mutex); 330 331 msi_set_masked(pdev->msix.virt_state, flags & PCI_MSIX_FLAGS_MASKALL); 332 enable = flags & PCI_MSIX_FLAGS_ENABLE; 333 msi_set_enabled(pdev->msix.virt_state, enable); 334 335 if (enable && vfio_pci_enable_msis(kvm, vdev, true)) 336 vfio_dev_err(vdev, "cannot enable MSIX"); 337 else if (!enable && vfio_pci_disable_msis(kvm, vdev, true)) 338 vfio_dev_err(vdev, "cannot disable MSIX"); 339 340 mutex_unlock(&pdev->msix.mutex); 341 } 342 343 static int vfio_pci_msi_vector_write(struct kvm *kvm, struct vfio_device *vdev, 344 u8 off, u8 *data, u32 sz) 345 { 346 size_t i; 347 u32 mask = 0; 348 size_t mask_pos, start, limit; 349 struct vfio_pci_msi_entry *entry; 350 struct vfio_pci_device *pdev = &vdev->pci; 351 struct msi_cap_64 *msi_cap_64 = PCI_CAP(&pdev->hdr, pdev->msi.pos); 352 353 if (!(msi_cap_64->ctrl & PCI_MSI_FLAGS_MASKBIT)) 354 return 0; 355 356 if (msi_cap_64->ctrl & PCI_MSI_FLAGS_64BIT) 357 mask_pos = PCI_MSI_MASK_64; 358 else 359 mask_pos = PCI_MSI_MASK_32; 360 361 if (off >= mask_pos + 4 || off + sz <= mask_pos) 362 return 0; 363 364 /* Set mask to current state */ 365 for (i = 0; i < pdev->msi.nr_entries; i++) { 366 entry = &pdev->msi.entries[i]; 367 mask |= !!msi_is_masked(entry->virt_state) << i; 368 } 369 370 /* Update mask following the intersection of access and register */ 371 start = max_t(size_t, off, mask_pos); 372 limit = min_t(size_t, off + sz, mask_pos + 4); 373 374 memcpy((void *)&mask + start - mask_pos, data + start - off, 375 limit - start); 376 377 /* Update states if necessary */ 378 for (i = 0; i < pdev->msi.nr_entries; i++) { 379 bool masked = mask & (1 << i); 380 381 entry = &pdev->msi.entries[i]; 382 if (masked != msi_is_masked(entry->virt_state)) { 383 msi_set_masked(entry->virt_state, masked); 384 vfio_pci_update_msi_entry(kvm, vdev, entry); 385 } 386 } 387 388 return 1; 389 } 390 391 static void vfio_pci_msi_cap_write(struct kvm *kvm, struct vfio_device *vdev, 392 u8 off, u8 *data, u32 sz) 393 { 394 u8 ctrl; 395 struct msi_msg msg; 396 size_t i, nr_vectors; 397 struct vfio_pci_msi_entry *entry; 398 struct vfio_pci_device *pdev = &vdev->pci; 399 struct msi_cap_64 *msi_cap_64 = PCI_CAP(&pdev->hdr, pdev->msi.pos); 400 401 off -= pdev->msi.pos; 402 403 mutex_lock(&pdev->msi.mutex); 404 405 /* Check if the guest is trying to update mask bits */ 406 if (vfio_pci_msi_vector_write(kvm, vdev, off, data, sz)) 407 goto out_unlock; 408 409 /* Only modify routes when guest pokes the enable bit */ 410 if (off > PCI_MSI_FLAGS || off + sz <= PCI_MSI_FLAGS) 411 goto out_unlock; 412 413 ctrl = *(u8 *)(data + PCI_MSI_FLAGS - off); 414 415 msi_set_enabled(pdev->msi.virt_state, ctrl & PCI_MSI_FLAGS_ENABLE); 416 417 if (!msi_is_enabled(pdev->msi.virt_state)) { 418 vfio_pci_disable_msis(kvm, vdev, false); 419 goto out_unlock; 420 } 421 422 /* Create routes for the requested vectors */ 423 nr_vectors = 1 << ((ctrl & PCI_MSI_FLAGS_QSIZE) >> 4); 424 425 msg.address_lo = msi_cap_64->address_lo; 426 if (msi_cap_64->ctrl & PCI_MSI_FLAGS_64BIT) { 427 msg.address_hi = msi_cap_64->address_hi; 428 msg.data = msi_cap_64->data; 429 } else { 430 struct msi_cap_32 *msi_cap_32 = (void *)msi_cap_64; 431 msg.address_hi = 0; 432 msg.data = msi_cap_32->data; 433 } 434 435 for (i = 0; i < nr_vectors; i++) { 436 entry = &pdev->msi.entries[i]; 437 entry->config.msg = msg; 438 vfio_pci_update_msi_entry(kvm, vdev, entry); 439 } 440 441 /* Update the physical capability if necessary */ 442 if (vfio_pci_enable_msis(kvm, vdev, false)) 443 vfio_dev_err(vdev, "cannot enable MSI"); 444 445 out_unlock: 446 mutex_unlock(&pdev->msi.mutex); 447 } 448 449 static void vfio_pci_cfg_read(struct kvm *kvm, struct pci_device_header *pci_hdr, 450 u8 offset, void *data, int sz) 451 { 452 struct vfio_region_info *info; 453 struct vfio_pci_device *pdev; 454 struct vfio_device *vdev; 455 char base[sz]; 456 457 pdev = container_of(pci_hdr, struct vfio_pci_device, hdr); 458 vdev = container_of(pdev, struct vfio_device, pci); 459 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 460 461 /* Dummy read in case of side-effects */ 462 if (pread(vdev->fd, base, sz, info->offset + offset) != sz) 463 vfio_dev_warn(vdev, "failed to read %d bytes from Configuration Space at 0x%x", 464 sz, offset); 465 } 466 467 static void vfio_pci_cfg_write(struct kvm *kvm, struct pci_device_header *pci_hdr, 468 u8 offset, void *data, int sz) 469 { 470 struct vfio_region_info *info; 471 struct vfio_pci_device *pdev; 472 struct vfio_device *vdev; 473 void *base = pci_hdr; 474 475 pdev = container_of(pci_hdr, struct vfio_pci_device, hdr); 476 vdev = container_of(pdev, struct vfio_device, pci); 477 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 478 479 if (pwrite(vdev->fd, data, sz, info->offset + offset) != sz) 480 vfio_dev_warn(vdev, "Failed to write %d bytes to Configuration Space at 0x%x", 481 sz, offset); 482 483 /* Handle MSI write now, since it might update the hardware capability */ 484 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) 485 vfio_pci_msix_cap_write(kvm, vdev, offset, data, sz); 486 487 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSI) 488 vfio_pci_msi_cap_write(kvm, vdev, offset, data, sz); 489 490 if (pread(vdev->fd, base + offset, sz, info->offset + offset) != sz) 491 vfio_dev_warn(vdev, "Failed to read %d bytes from Configuration Space at 0x%x", 492 sz, offset); 493 } 494 495 static ssize_t vfio_pci_msi_cap_size(struct msi_cap_64 *cap_hdr) 496 { 497 size_t size = 10; 498 499 if (cap_hdr->ctrl & PCI_MSI_FLAGS_64BIT) 500 size += 4; 501 if (cap_hdr->ctrl & PCI_MSI_FLAGS_MASKBIT) 502 size += 10; 503 504 return size; 505 } 506 507 static ssize_t vfio_pci_cap_size(struct pci_cap_hdr *cap_hdr) 508 { 509 switch (cap_hdr->type) { 510 case PCI_CAP_ID_MSIX: 511 return PCI_CAP_MSIX_SIZEOF; 512 case PCI_CAP_ID_MSI: 513 return vfio_pci_msi_cap_size((void *)cap_hdr); 514 default: 515 pr_err("unknown PCI capability 0x%x", cap_hdr->type); 516 return 0; 517 } 518 } 519 520 static int vfio_pci_add_cap(struct vfio_device *vdev, u8 *virt_hdr, 521 struct pci_cap_hdr *cap, off_t pos) 522 { 523 struct pci_cap_hdr *last; 524 struct pci_device_header *hdr = &vdev->pci.hdr; 525 526 cap->next = 0; 527 528 if (!hdr->capabilities) { 529 hdr->capabilities = pos; 530 hdr->status |= PCI_STATUS_CAP_LIST; 531 } else { 532 last = PCI_CAP(virt_hdr, hdr->capabilities); 533 534 while (last->next) 535 last = PCI_CAP(virt_hdr, last->next); 536 537 last->next = pos; 538 } 539 540 memcpy(virt_hdr + pos, cap, vfio_pci_cap_size(cap)); 541 542 return 0; 543 } 544 545 static int vfio_pci_parse_caps(struct vfio_device *vdev) 546 { 547 int ret; 548 size_t size; 549 u8 pos, next; 550 struct pci_cap_hdr *cap; 551 u8 virt_hdr[PCI_DEV_CFG_SIZE]; 552 struct vfio_pci_device *pdev = &vdev->pci; 553 554 if (!(pdev->hdr.status & PCI_STATUS_CAP_LIST)) 555 return 0; 556 557 memset(virt_hdr, 0, PCI_DEV_CFG_SIZE); 558 559 pos = pdev->hdr.capabilities & ~3; 560 561 pdev->hdr.status &= ~PCI_STATUS_CAP_LIST; 562 pdev->hdr.capabilities = 0; 563 564 for (; pos; pos = next) { 565 cap = PCI_CAP(&pdev->hdr, pos); 566 next = cap->next; 567 568 switch (cap->type) { 569 case PCI_CAP_ID_MSIX: 570 ret = vfio_pci_add_cap(vdev, virt_hdr, cap, pos); 571 if (ret) 572 return ret; 573 574 pdev->msix.pos = pos; 575 pdev->irq_modes |= VFIO_PCI_IRQ_MODE_MSIX; 576 break; 577 case PCI_CAP_ID_MSI: 578 ret = vfio_pci_add_cap(vdev, virt_hdr, cap, pos); 579 if (ret) 580 return ret; 581 582 pdev->msi.pos = pos; 583 pdev->irq_modes |= VFIO_PCI_IRQ_MODE_MSI; 584 break; 585 } 586 } 587 588 /* Wipe remaining capabilities */ 589 pos = PCI_STD_HEADER_SIZEOF; 590 size = PCI_DEV_CFG_SIZE - PCI_STD_HEADER_SIZEOF; 591 memcpy((void *)&pdev->hdr + pos, virt_hdr + pos, size); 592 593 return 0; 594 } 595 596 static int vfio_pci_parse_cfg_space(struct vfio_device *vdev) 597 { 598 ssize_t sz = PCI_DEV_CFG_SIZE; 599 struct vfio_region_info *info; 600 struct vfio_pci_device *pdev = &vdev->pci; 601 602 if (vdev->info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX) { 603 vfio_dev_err(vdev, "Config Space not found"); 604 return -ENODEV; 605 } 606 607 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 608 *info = (struct vfio_region_info) { 609 .argsz = sizeof(*info), 610 .index = VFIO_PCI_CONFIG_REGION_INDEX, 611 }; 612 613 ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, info); 614 if (!info->size) { 615 vfio_dev_err(vdev, "Config Space has size zero?!"); 616 return -EINVAL; 617 } 618 619 /* Read standard headers and capabilities */ 620 if (pread(vdev->fd, &pdev->hdr, sz, info->offset) != sz) { 621 vfio_dev_err(vdev, "failed to read %zd bytes of Config Space", sz); 622 return -EIO; 623 } 624 625 /* Strip bit 7, that indicates multifunction */ 626 pdev->hdr.header_type &= 0x7f; 627 628 if (pdev->hdr.header_type != PCI_HEADER_TYPE_NORMAL) { 629 vfio_dev_err(vdev, "unsupported header type %u", 630 pdev->hdr.header_type); 631 return -EOPNOTSUPP; 632 } 633 634 if (pdev->hdr.irq_pin) 635 pdev->irq_modes |= VFIO_PCI_IRQ_MODE_INTX; 636 637 vfio_pci_parse_caps(vdev); 638 639 return 0; 640 } 641 642 static int vfio_pci_fixup_cfg_space(struct vfio_device *vdev) 643 { 644 int i; 645 ssize_t hdr_sz; 646 struct msix_cap *msix; 647 struct vfio_region_info *info; 648 struct vfio_pci_device *pdev = &vdev->pci; 649 650 /* Initialise the BARs */ 651 for (i = VFIO_PCI_BAR0_REGION_INDEX; i <= VFIO_PCI_BAR5_REGION_INDEX; ++i) { 652 u64 base; 653 struct vfio_region *region = &vdev->regions[i]; 654 655 /* Construct a fake reg to match what we've mapped. */ 656 if (region->is_ioport) { 657 base = (region->port_base & PCI_BASE_ADDRESS_IO_MASK) | 658 PCI_BASE_ADDRESS_SPACE_IO; 659 } else { 660 base = (region->guest_phys_addr & 661 PCI_BASE_ADDRESS_MEM_MASK) | 662 PCI_BASE_ADDRESS_SPACE_MEMORY; 663 } 664 665 pdev->hdr.bar[i] = base; 666 667 if (!base) 668 continue; 669 670 pdev->hdr.bar_size[i] = region->info.size; 671 } 672 673 /* I really can't be bothered to support cardbus. */ 674 pdev->hdr.card_bus = 0; 675 676 /* 677 * Nuke the expansion ROM for now. If we want to do this properly, 678 * we need to save its size somewhere and map into the guest. 679 */ 680 pdev->hdr.exp_rom_bar = 0; 681 682 /* Plumb in our fake MSI-X capability, if we have it. */ 683 msix = pci_find_cap(&pdev->hdr, PCI_CAP_ID_MSIX); 684 if (msix) { 685 /* Add a shortcut to the PBA region for the MMIO handler */ 686 int pba_index = VFIO_PCI_BAR0_REGION_INDEX + pdev->msix_pba.bar; 687 pdev->msix_pba.offset = vdev->regions[pba_index].info.offset + 688 (msix->pba_offset & PCI_MSIX_PBA_OFFSET); 689 690 /* Tidy up the capability */ 691 msix->table_offset &= PCI_MSIX_TABLE_BIR; 692 msix->pba_offset &= PCI_MSIX_PBA_BIR; 693 if (pdev->msix_table.bar == pdev->msix_pba.bar) 694 msix->pba_offset |= pdev->msix_table.size & 695 PCI_MSIX_PBA_OFFSET; 696 } 697 698 /* Install our fake Configuration Space */ 699 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 700 hdr_sz = PCI_DEV_CFG_SIZE; 701 if (pwrite(vdev->fd, &pdev->hdr, hdr_sz, info->offset) != hdr_sz) { 702 vfio_dev_err(vdev, "failed to write %zd bytes to Config Space", 703 hdr_sz); 704 return -EIO; 705 } 706 707 /* Register callbacks for cfg accesses */ 708 pdev->hdr.cfg_ops = (struct pci_config_operations) { 709 .read = vfio_pci_cfg_read, 710 .write = vfio_pci_cfg_write, 711 }; 712 713 pdev->hdr.irq_type = IRQ_TYPE_LEVEL_HIGH; 714 715 return 0; 716 } 717 718 static int vfio_pci_create_msix_table(struct kvm *kvm, 719 struct vfio_pci_device *pdev) 720 { 721 int ret; 722 size_t i; 723 size_t mmio_size; 724 size_t nr_entries; 725 struct vfio_pci_msi_entry *entries; 726 struct vfio_pci_msix_pba *pba = &pdev->msix_pba; 727 struct vfio_pci_msix_table *table = &pdev->msix_table; 728 struct msix_cap *msix = PCI_CAP(&pdev->hdr, pdev->msix.pos); 729 730 table->bar = msix->table_offset & PCI_MSIX_TABLE_BIR; 731 pba->bar = msix->pba_offset & PCI_MSIX_TABLE_BIR; 732 733 /* 734 * KVM needs memory regions to be multiple of and aligned on PAGE_SIZE. 735 */ 736 nr_entries = (msix->ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 737 table->size = ALIGN(nr_entries * PCI_MSIX_ENTRY_SIZE, PAGE_SIZE); 738 pba->size = ALIGN(DIV_ROUND_UP(nr_entries, 64), PAGE_SIZE); 739 740 entries = calloc(nr_entries, sizeof(struct vfio_pci_msi_entry)); 741 if (!entries) 742 return -ENOMEM; 743 744 for (i = 0; i < nr_entries; i++) 745 entries[i].config.ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; 746 747 /* 748 * To ease MSI-X cap configuration in case they share the same BAR, 749 * collapse table and pending array. The size of the BAR regions must be 750 * powers of two. 751 */ 752 mmio_size = roundup_pow_of_two(table->size + pba->size); 753 table->guest_phys_addr = pci_get_io_space_block(mmio_size); 754 if (!table->guest_phys_addr) { 755 pr_err("cannot allocate IO space"); 756 ret = -ENOMEM; 757 goto out_free; 758 } 759 pba->guest_phys_addr = table->guest_phys_addr + table->size; 760 761 ret = kvm__register_mmio(kvm, table->guest_phys_addr, table->size, 762 false, vfio_pci_msix_table_access, pdev); 763 if (ret < 0) 764 goto out_free; 765 766 /* 767 * We could map the physical PBA directly into the guest, but it's 768 * likely smaller than a page, and we can only hand full pages to the 769 * guest. Even though the PCI spec disallows sharing a page used for 770 * MSI-X with any other resource, it allows to share the same page 771 * between MSI-X table and PBA. For the sake of isolation, create a 772 * virtual PBA. 773 */ 774 ret = kvm__register_mmio(kvm, pba->guest_phys_addr, pba->size, false, 775 vfio_pci_msix_pba_access, pdev); 776 if (ret < 0) 777 goto out_free; 778 779 pdev->msix.entries = entries; 780 pdev->msix.nr_entries = nr_entries; 781 782 return 0; 783 784 out_free: 785 free(entries); 786 787 return ret; 788 } 789 790 static int vfio_pci_create_msi_cap(struct kvm *kvm, struct vfio_pci_device *pdev) 791 { 792 struct msi_cap_64 *cap = PCI_CAP(&pdev->hdr, pdev->msi.pos); 793 794 pdev->msi.nr_entries = 1 << ((cap->ctrl & PCI_MSI_FLAGS_QMASK) >> 1), 795 pdev->msi.entries = calloc(pdev->msi.nr_entries, 796 sizeof(struct vfio_pci_msi_entry)); 797 if (!pdev->msi.entries) 798 return -ENOMEM; 799 800 return 0; 801 } 802 803 static int vfio_pci_configure_bar(struct kvm *kvm, struct vfio_device *vdev, 804 size_t nr) 805 { 806 int ret; 807 u32 bar; 808 size_t map_size; 809 struct vfio_pci_device *pdev = &vdev->pci; 810 struct vfio_region *region = &vdev->regions[nr]; 811 812 if (nr >= vdev->info.num_regions) 813 return 0; 814 815 bar = pdev->hdr.bar[nr]; 816 817 region->vdev = vdev; 818 region->is_ioport = !!(bar & PCI_BASE_ADDRESS_SPACE_IO); 819 region->info = (struct vfio_region_info) { 820 .argsz = sizeof(region->info), 821 .index = nr, 822 }; 823 824 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, ®ion->info); 825 if (ret) { 826 ret = -errno; 827 vfio_dev_err(vdev, "cannot get info for BAR %zu", nr); 828 return ret; 829 } 830 831 /* Ignore invalid or unimplemented regions */ 832 if (!region->info.size) 833 return 0; 834 835 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) { 836 /* Trap and emulate MSI-X table */ 837 if (nr == pdev->msix_table.bar) { 838 region->guest_phys_addr = pdev->msix_table.guest_phys_addr; 839 return 0; 840 } else if (nr == pdev->msix_pba.bar) { 841 region->guest_phys_addr = pdev->msix_pba.guest_phys_addr; 842 return 0; 843 } 844 } 845 846 if (!region->is_ioport) { 847 /* Grab some MMIO space in the guest */ 848 map_size = ALIGN(region->info.size, PAGE_SIZE); 849 region->guest_phys_addr = pci_get_io_space_block(map_size); 850 } 851 852 /* Map the BARs into the guest or setup a trap region. */ 853 ret = vfio_map_region(kvm, vdev, region); 854 if (ret) 855 return ret; 856 857 return 0; 858 } 859 860 static int vfio_pci_configure_dev_regions(struct kvm *kvm, 861 struct vfio_device *vdev) 862 { 863 int ret; 864 u32 bar; 865 size_t i; 866 bool is_64bit = false; 867 struct vfio_pci_device *pdev = &vdev->pci; 868 869 ret = vfio_pci_parse_cfg_space(vdev); 870 if (ret) 871 return ret; 872 873 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) { 874 ret = vfio_pci_create_msix_table(kvm, pdev); 875 if (ret) 876 return ret; 877 } 878 879 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSI) { 880 ret = vfio_pci_create_msi_cap(kvm, pdev); 881 if (ret) 882 return ret; 883 } 884 885 for (i = VFIO_PCI_BAR0_REGION_INDEX; i <= VFIO_PCI_BAR5_REGION_INDEX; ++i) { 886 /* Ignore top half of 64-bit BAR */ 887 if (i % 2 && is_64bit) 888 continue; 889 890 ret = vfio_pci_configure_bar(kvm, vdev, i); 891 if (ret) 892 return ret; 893 894 bar = pdev->hdr.bar[i]; 895 is_64bit = (bar & PCI_BASE_ADDRESS_SPACE) == 896 PCI_BASE_ADDRESS_SPACE_MEMORY && 897 bar & PCI_BASE_ADDRESS_MEM_TYPE_64; 898 } 899 900 /* We've configured the BARs, fake up a Configuration Space */ 901 return vfio_pci_fixup_cfg_space(vdev); 902 } 903 904 /* 905 * Attempt to update the FD limit, if opening an eventfd for each IRQ vector 906 * would hit the limit. Which is likely to happen when a device uses 2048 MSIs. 907 */ 908 static int vfio_pci_reserve_irq_fds(size_t num) 909 { 910 /* 911 * I counted around 27 fds under normal load. Let's add 100 for good 912 * measure. 913 */ 914 static size_t needed = 128; 915 struct rlimit fd_limit, new_limit; 916 917 needed += num; 918 919 if (getrlimit(RLIMIT_NOFILE, &fd_limit)) { 920 perror("getrlimit(RLIMIT_NOFILE)"); 921 return 0; 922 } 923 924 if (fd_limit.rlim_cur >= needed) 925 return 0; 926 927 new_limit.rlim_cur = needed; 928 929 if (fd_limit.rlim_max < needed) 930 /* Try to bump hard limit (root only) */ 931 new_limit.rlim_max = needed; 932 else 933 new_limit.rlim_max = fd_limit.rlim_max; 934 935 if (setrlimit(RLIMIT_NOFILE, &new_limit)) { 936 perror("setrlimit(RLIMIT_NOFILE)"); 937 pr_warning("not enough FDs for full MSI-X support (estimated need: %zu)", 938 (size_t)(needed - fd_limit.rlim_cur)); 939 } 940 941 return 0; 942 } 943 944 static int vfio_pci_init_msis(struct kvm *kvm, struct vfio_device *vdev, 945 struct vfio_pci_msi_common *msis) 946 { 947 int ret; 948 size_t i; 949 int *eventfds; 950 size_t irq_set_size; 951 struct vfio_pci_msi_entry *entry; 952 size_t nr_entries = msis->nr_entries; 953 954 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &msis->info); 955 if (ret || msis->info.count == 0) { 956 vfio_dev_err(vdev, "no MSI reported by VFIO"); 957 return -ENODEV; 958 } 959 960 if (!(msis->info.flags & VFIO_IRQ_INFO_EVENTFD)) { 961 vfio_dev_err(vdev, "interrupt not EVENTFD capable"); 962 return -EINVAL; 963 } 964 965 if (msis->info.count != nr_entries) { 966 vfio_dev_err(vdev, "invalid number of MSIs reported by VFIO"); 967 return -EINVAL; 968 } 969 970 mutex_init(&msis->mutex); 971 972 vfio_pci_reserve_irq_fds(nr_entries); 973 974 irq_set_size = sizeof(struct vfio_irq_set) + nr_entries * sizeof(int); 975 msis->irq_set = malloc(irq_set_size); 976 if (!msis->irq_set) 977 return -ENOMEM; 978 979 *msis->irq_set = (struct vfio_irq_set) { 980 .argsz = irq_set_size, 981 .flags = VFIO_IRQ_SET_DATA_EVENTFD | 982 VFIO_IRQ_SET_ACTION_TRIGGER, 983 .index = msis->info.index, 984 .start = 0, 985 .count = nr_entries, 986 }; 987 988 eventfds = (void *)msis->irq_set + sizeof(struct vfio_irq_set); 989 990 for (i = 0; i < nr_entries; i++) { 991 entry = &msis->entries[i]; 992 entry->gsi = -1; 993 entry->eventfd = -1; 994 msi_set_masked(entry->virt_state, true); 995 msi_set_masked(entry->phys_state, true); 996 eventfds[i] = -1; 997 } 998 999 return 0; 1000 } 1001 1002 static void vfio_pci_disable_intx(struct kvm *kvm, struct vfio_device *vdev) 1003 { 1004 struct vfio_pci_device *pdev = &vdev->pci; 1005 int gsi = pdev->intx_gsi; 1006 struct vfio_irq_set irq_set = { 1007 .argsz = sizeof(irq_set), 1008 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 1009 .index = VFIO_PCI_INTX_IRQ_INDEX, 1010 }; 1011 1012 if (pdev->intx_fd == -1) 1013 return; 1014 1015 pr_debug("user requested MSI, disabling INTx %d", gsi); 1016 1017 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 1018 irq__del_irqfd(kvm, gsi, pdev->intx_fd); 1019 1020 close(pdev->intx_fd); 1021 close(pdev->unmask_fd); 1022 pdev->intx_fd = -1; 1023 } 1024 1025 static int vfio_pci_enable_intx(struct kvm *kvm, struct vfio_device *vdev) 1026 { 1027 int ret; 1028 int trigger_fd, unmask_fd; 1029 union vfio_irq_eventfd trigger; 1030 union vfio_irq_eventfd unmask; 1031 struct vfio_pci_device *pdev = &vdev->pci; 1032 int gsi = pdev->intx_gsi; 1033 1034 if (pdev->intx_fd != -1) 1035 return 0; 1036 1037 /* 1038 * PCI IRQ is level-triggered, so we use two eventfds. trigger_fd 1039 * signals an interrupt from host to guest, and unmask_fd signals the 1040 * deassertion of the line from guest to host. 1041 */ 1042 trigger_fd = eventfd(0, 0); 1043 if (trigger_fd < 0) { 1044 vfio_dev_err(vdev, "failed to create trigger eventfd"); 1045 return trigger_fd; 1046 } 1047 1048 unmask_fd = eventfd(0, 0); 1049 if (unmask_fd < 0) { 1050 vfio_dev_err(vdev, "failed to create unmask eventfd"); 1051 close(trigger_fd); 1052 return unmask_fd; 1053 } 1054 1055 ret = irq__add_irqfd(kvm, gsi, trigger_fd, unmask_fd); 1056 if (ret) 1057 goto err_close; 1058 1059 trigger.irq = (struct vfio_irq_set) { 1060 .argsz = sizeof(trigger), 1061 .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 1062 .index = VFIO_PCI_INTX_IRQ_INDEX, 1063 .start = 0, 1064 .count = 1, 1065 }; 1066 set_vfio_irq_eventd_payload(&trigger, trigger_fd); 1067 1068 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &trigger); 1069 if (ret < 0) { 1070 vfio_dev_err(vdev, "failed to setup VFIO IRQ"); 1071 goto err_delete_line; 1072 } 1073 1074 unmask.irq = (struct vfio_irq_set) { 1075 .argsz = sizeof(unmask), 1076 .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK, 1077 .index = VFIO_PCI_INTX_IRQ_INDEX, 1078 .start = 0, 1079 .count = 1, 1080 }; 1081 set_vfio_irq_eventd_payload(&unmask, unmask_fd); 1082 1083 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &unmask); 1084 if (ret < 0) { 1085 vfio_dev_err(vdev, "failed to setup unmask IRQ"); 1086 goto err_remove_event; 1087 } 1088 1089 pdev->intx_fd = trigger_fd; 1090 pdev->unmask_fd = unmask_fd; 1091 1092 return 0; 1093 1094 err_remove_event: 1095 /* Remove trigger event */ 1096 trigger.irq.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER; 1097 trigger.irq.count = 0; 1098 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &trigger); 1099 1100 err_delete_line: 1101 irq__del_irqfd(kvm, gsi, trigger_fd); 1102 1103 err_close: 1104 close(trigger_fd); 1105 close(unmask_fd); 1106 return ret; 1107 } 1108 1109 static int vfio_pci_init_intx(struct kvm *kvm, struct vfio_device *vdev) 1110 { 1111 int ret; 1112 struct vfio_pci_device *pdev = &vdev->pci; 1113 struct vfio_irq_info irq_info = { 1114 .argsz = sizeof(irq_info), 1115 .index = VFIO_PCI_INTX_IRQ_INDEX, 1116 }; 1117 1118 vfio_pci_reserve_irq_fds(2); 1119 1120 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); 1121 if (ret || irq_info.count == 0) { 1122 vfio_dev_err(vdev, "no INTx reported by VFIO"); 1123 return -ENODEV; 1124 } 1125 1126 if (!(irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) { 1127 vfio_dev_err(vdev, "interrupt not eventfd capable"); 1128 return -EINVAL; 1129 } 1130 1131 if (!(irq_info.flags & VFIO_IRQ_INFO_AUTOMASKED)) { 1132 vfio_dev_err(vdev, "INTx interrupt not AUTOMASKED"); 1133 return -EINVAL; 1134 } 1135 1136 /* Guest is going to ovewrite our irq_line... */ 1137 pdev->intx_gsi = pdev->hdr.irq_line - KVM_IRQ_OFFSET; 1138 1139 pdev->intx_fd = -1; 1140 1141 return 0; 1142 } 1143 1144 static int vfio_pci_configure_dev_irqs(struct kvm *kvm, struct vfio_device *vdev) 1145 { 1146 int ret = 0; 1147 struct vfio_pci_device *pdev = &vdev->pci; 1148 1149 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) { 1150 pdev->msix.info = (struct vfio_irq_info) { 1151 .argsz = sizeof(pdev->msix.info), 1152 .index = VFIO_PCI_MSIX_IRQ_INDEX, 1153 }; 1154 ret = vfio_pci_init_msis(kvm, vdev, &pdev->msix); 1155 if (ret) 1156 return ret; 1157 } 1158 1159 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSI) { 1160 pdev->msi.info = (struct vfio_irq_info) { 1161 .argsz = sizeof(pdev->msi.info), 1162 .index = VFIO_PCI_MSI_IRQ_INDEX, 1163 }; 1164 ret = vfio_pci_init_msis(kvm, vdev, &pdev->msi); 1165 if (ret) 1166 return ret; 1167 } 1168 1169 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_INTX) { 1170 ret = vfio_pci_init_intx(kvm, vdev); 1171 if (ret) 1172 return ret; 1173 1174 ret = vfio_pci_enable_intx(kvm, vdev); 1175 } 1176 1177 return ret; 1178 } 1179 1180 int vfio_pci_setup_device(struct kvm *kvm, struct vfio_device *vdev) 1181 { 1182 int ret; 1183 1184 ret = vfio_pci_configure_dev_regions(kvm, vdev); 1185 if (ret) { 1186 vfio_dev_err(vdev, "failed to configure regions"); 1187 return ret; 1188 } 1189 1190 vdev->dev_hdr = (struct device_header) { 1191 .bus_type = DEVICE_BUS_PCI, 1192 .data = &vdev->pci.hdr, 1193 }; 1194 1195 ret = device__register(&vdev->dev_hdr); 1196 if (ret) { 1197 vfio_dev_err(vdev, "failed to register VFIO device"); 1198 return ret; 1199 } 1200 1201 ret = vfio_pci_configure_dev_irqs(kvm, vdev); 1202 if (ret) { 1203 vfio_dev_err(vdev, "failed to configure IRQs"); 1204 return ret; 1205 } 1206 1207 return 0; 1208 } 1209 1210 void vfio_pci_teardown_device(struct kvm *kvm, struct vfio_device *vdev) 1211 { 1212 size_t i; 1213 struct vfio_pci_device *pdev = &vdev->pci; 1214 1215 for (i = 0; i < vdev->info.num_regions; i++) 1216 vfio_unmap_region(kvm, &vdev->regions[i]); 1217 1218 device__unregister(&vdev->dev_hdr); 1219 1220 free(pdev->msix.irq_set); 1221 free(pdev->msix.entries); 1222 free(pdev->msi.irq_set); 1223 free(pdev->msi.entries); 1224 } 1225