1 #include "kvm/irq.h" 2 #include "kvm/kvm.h" 3 #include "kvm/kvm-cpu.h" 4 #include "kvm/vfio.h" 5 6 #include <sys/ioctl.h> 7 #include <sys/eventfd.h> 8 #include <sys/resource.h> 9 #include <sys/time.h> 10 11 /* Wrapper around UAPI vfio_irq_set */ 12 union vfio_irq_eventfd { 13 struct vfio_irq_set irq; 14 u8 buffer[sizeof(struct vfio_irq_set) + sizeof(int)]; 15 }; 16 17 static void set_vfio_irq_eventd_payload(union vfio_irq_eventfd *evfd, int fd) 18 { 19 memcpy(&evfd->irq.data, &fd, sizeof(fd)); 20 } 21 22 #define msi_is_enabled(state) ((state) & VFIO_PCI_MSI_STATE_ENABLED) 23 #define msi_is_masked(state) ((state) & VFIO_PCI_MSI_STATE_MASKED) 24 #define msi_is_empty(state) ((state) & VFIO_PCI_MSI_STATE_EMPTY) 25 26 #define msi_update_state(state, val, bit) \ 27 (state) = (val) ? (state) | bit : (state) & ~bit; 28 #define msi_set_enabled(state, val) \ 29 msi_update_state(state, val, VFIO_PCI_MSI_STATE_ENABLED) 30 #define msi_set_masked(state, val) \ 31 msi_update_state(state, val, VFIO_PCI_MSI_STATE_MASKED) 32 #define msi_set_empty(state, val) \ 33 msi_update_state(state, val, VFIO_PCI_MSI_STATE_EMPTY) 34 35 static void vfio_pci_disable_intx(struct kvm *kvm, struct vfio_device *vdev); 36 static int vfio_pci_enable_intx(struct kvm *kvm, struct vfio_device *vdev); 37 38 static int vfio_pci_enable_msis(struct kvm *kvm, struct vfio_device *vdev, 39 bool msix) 40 { 41 size_t i; 42 int ret = 0; 43 int *eventfds; 44 struct vfio_pci_device *pdev = &vdev->pci; 45 struct vfio_pci_msi_common *msis = msix ? &pdev->msix : &pdev->msi; 46 union vfio_irq_eventfd single = { 47 .irq = { 48 .argsz = sizeof(single), 49 .flags = VFIO_IRQ_SET_DATA_EVENTFD | 50 VFIO_IRQ_SET_ACTION_TRIGGER, 51 .index = msis->info.index, 52 .count = 1, 53 }, 54 }; 55 56 if (!msi_is_enabled(msis->virt_state)) 57 return 0; 58 59 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_INTX) 60 /* 61 * PCI (and VFIO) forbids enabling INTx, MSI or MSIX at the same 62 * time. Since INTx has to be enabled from the start (we don't 63 * have a reliable way to know when the guest starts using it), 64 * disable it now. 65 */ 66 vfio_pci_disable_intx(kvm, vdev); 67 68 eventfds = (void *)msis->irq_set + sizeof(struct vfio_irq_set); 69 70 /* 71 * Initial registration of the full range. This enables the physical 72 * MSI/MSI-X capability, which might have desired side effects. For 73 * instance when assigning virtio legacy devices, enabling the MSI 74 * capability modifies the config space layout! 75 * 76 * As an optimization, only update MSIs when guest unmasks the 77 * capability. This greatly reduces the initialization time for Linux 78 * guest with 2048+ MSIs. Linux guest starts by enabling the MSI-X cap 79 * masked, then fills individual vectors, then unmasks the whole 80 * function. So we only do one VFIO ioctl when enabling for the first 81 * time, and then one when unmasking. 82 * 83 * phys_state is empty when it is enabled but no vector has been 84 * registered via SET_IRQS yet. 85 */ 86 if (!msi_is_enabled(msis->phys_state) || 87 (!msi_is_masked(msis->virt_state) && 88 msi_is_empty(msis->phys_state))) { 89 bool empty = true; 90 91 for (i = 0; i < msis->nr_entries; i++) { 92 eventfds[i] = msis->entries[i].gsi >= 0 ? 93 msis->entries[i].eventfd : -1; 94 95 if (eventfds[i] >= 0) 96 empty = false; 97 } 98 99 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, msis->irq_set); 100 if (ret < 0) { 101 perror("VFIO_DEVICE_SET_IRQS(multi)"); 102 return ret; 103 } 104 105 msi_set_enabled(msis->phys_state, true); 106 msi_set_empty(msis->phys_state, empty); 107 108 return 0; 109 } 110 111 if (msi_is_masked(msis->virt_state)) { 112 /* TODO: if phys_state is not empty nor masked, mask all vectors */ 113 return 0; 114 } 115 116 /* Update individual vectors to avoid breaking those in use */ 117 for (i = 0; i < msis->nr_entries; i++) { 118 struct vfio_pci_msi_entry *entry = &msis->entries[i]; 119 int fd = entry->gsi >= 0 ? entry->eventfd : -1; 120 121 if (fd == eventfds[i]) 122 continue; 123 124 single.irq.start = i; 125 set_vfio_irq_eventd_payload(&single, fd); 126 127 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &single); 128 if (ret < 0) { 129 perror("VFIO_DEVICE_SET_IRQS(single)"); 130 break; 131 } 132 133 eventfds[i] = fd; 134 135 if (msi_is_empty(msis->phys_state) && fd >= 0) 136 msi_set_empty(msis->phys_state, false); 137 } 138 139 return ret; 140 } 141 142 static int vfio_pci_disable_msis(struct kvm *kvm, struct vfio_device *vdev, 143 bool msix) 144 { 145 int ret; 146 struct vfio_pci_device *pdev = &vdev->pci; 147 struct vfio_pci_msi_common *msis = msix ? &pdev->msix : &pdev->msi; 148 struct vfio_irq_set irq_set = { 149 .argsz = sizeof(irq_set), 150 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 151 .index = msis->info.index, 152 .start = 0, 153 .count = 0, 154 }; 155 156 if (!msi_is_enabled(msis->phys_state)) 157 return 0; 158 159 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 160 if (ret < 0) { 161 perror("VFIO_DEVICE_SET_IRQS(NONE)"); 162 return ret; 163 } 164 165 msi_set_enabled(msis->phys_state, false); 166 msi_set_empty(msis->phys_state, true); 167 168 /* 169 * When MSI or MSIX is disabled, this might be called when 170 * PCI driver detects the MSI interrupt failure and wants to 171 * rollback to INTx mode. Thus enable INTx if the device 172 * supports INTx mode in this case. 173 */ 174 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_INTX) 175 ret = vfio_pci_enable_intx(kvm, vdev); 176 177 return ret >= 0 ? 0 : ret; 178 } 179 180 static int vfio_pci_update_msi_entry(struct kvm *kvm, struct vfio_device *vdev, 181 struct vfio_pci_msi_entry *entry) 182 { 183 int ret; 184 185 if (entry->eventfd < 0) { 186 entry->eventfd = eventfd(0, 0); 187 if (entry->eventfd < 0) { 188 ret = -errno; 189 vfio_dev_err(vdev, "cannot create eventfd"); 190 return ret; 191 } 192 } 193 194 /* Allocate IRQ if necessary */ 195 if (entry->gsi < 0) { 196 int ret = irq__add_msix_route(kvm, &entry->config.msg, 197 vdev->dev_hdr.dev_num << 3); 198 if (ret < 0) { 199 vfio_dev_err(vdev, "cannot create MSI-X route"); 200 return ret; 201 } 202 entry->gsi = ret; 203 } else { 204 irq__update_msix_route(kvm, entry->gsi, &entry->config.msg); 205 } 206 207 /* 208 * MSI masking is unimplemented in VFIO, so we have to handle it by 209 * disabling/enabling IRQ route instead. We do it on the KVM side rather 210 * than VFIO, because: 211 * - it is 8x faster 212 * - it allows to decouple masking logic from capability state. 213 * - in masked state, after removing irqfd route, we could easily plug 214 * the eventfd in a local handler, in order to serve Pending Bit reads 215 * to the guest. 216 * 217 * So entry->phys_state is masked when there is no active irqfd route. 218 */ 219 if (msi_is_masked(entry->virt_state) == msi_is_masked(entry->phys_state)) 220 return 0; 221 222 if (msi_is_masked(entry->phys_state)) { 223 ret = irq__add_irqfd(kvm, entry->gsi, entry->eventfd, -1); 224 if (ret < 0) { 225 vfio_dev_err(vdev, "cannot setup irqfd"); 226 return ret; 227 } 228 } else { 229 irq__del_irqfd(kvm, entry->gsi, entry->eventfd); 230 } 231 232 msi_set_masked(entry->phys_state, msi_is_masked(entry->virt_state)); 233 234 return 0; 235 } 236 237 static void vfio_pci_msix_pba_access(struct kvm_cpu *vcpu, u64 addr, u8 *data, 238 u32 len, u8 is_write, void *ptr) 239 { 240 struct vfio_pci_device *pdev = ptr; 241 struct vfio_pci_msix_pba *pba = &pdev->msix_pba; 242 u64 offset = addr - pba->guest_phys_addr; 243 struct vfio_device *vdev = container_of(pdev, struct vfio_device, pci); 244 245 if (is_write) 246 return; 247 248 /* 249 * TODO: emulate PBA. Hardware MSI-X is never masked, so reading the PBA 250 * is completely useless here. Note that Linux doesn't use PBA. 251 */ 252 if (pread(vdev->fd, data, len, pba->offset + offset) != (ssize_t)len) 253 vfio_dev_err(vdev, "cannot access MSIX PBA\n"); 254 } 255 256 static void vfio_pci_msix_table_access(struct kvm_cpu *vcpu, u64 addr, u8 *data, 257 u32 len, u8 is_write, void *ptr) 258 { 259 struct kvm *kvm = vcpu->kvm; 260 struct vfio_pci_msi_entry *entry; 261 struct vfio_pci_device *pdev = ptr; 262 struct vfio_device *vdev = container_of(pdev, struct vfio_device, pci); 263 264 u64 offset = addr - pdev->msix_table.guest_phys_addr; 265 266 size_t vector = offset / PCI_MSIX_ENTRY_SIZE; 267 off_t field = offset % PCI_MSIX_ENTRY_SIZE; 268 269 /* 270 * PCI spec says that software must use aligned 4 or 8 bytes accesses 271 * for the MSI-X tables. 272 */ 273 if ((len != 4 && len != 8) || addr & (len - 1)) { 274 vfio_dev_warn(vdev, "invalid MSI-X table access"); 275 return; 276 } 277 278 entry = &pdev->msix.entries[vector]; 279 280 mutex_lock(&pdev->msix.mutex); 281 282 if (!is_write) { 283 memcpy(data, (void *)&entry->config + field, len); 284 goto out_unlock; 285 } 286 287 memcpy((void *)&entry->config + field, data, len); 288 289 /* 290 * Check if access touched the vector control register, which is at the 291 * end of the MSI-X entry. 292 */ 293 if (field + len <= PCI_MSIX_ENTRY_VECTOR_CTRL) 294 goto out_unlock; 295 296 msi_set_masked(entry->virt_state, entry->config.ctrl & 297 PCI_MSIX_ENTRY_CTRL_MASKBIT); 298 299 if (vfio_pci_update_msi_entry(kvm, vdev, entry) < 0) 300 /* Not much we can do here. */ 301 vfio_dev_err(vdev, "failed to configure MSIX vector %zu", vector); 302 303 /* Update the physical capability if necessary */ 304 if (vfio_pci_enable_msis(kvm, vdev, true)) 305 vfio_dev_err(vdev, "cannot enable MSIX"); 306 307 out_unlock: 308 mutex_unlock(&pdev->msix.mutex); 309 } 310 311 static void vfio_pci_msix_cap_write(struct kvm *kvm, 312 struct vfio_device *vdev, u8 off, 313 void *data, int sz) 314 { 315 struct vfio_pci_device *pdev = &vdev->pci; 316 off_t enable_pos = PCI_MSIX_FLAGS + 1; 317 bool enable; 318 u16 flags; 319 320 off -= pdev->msix.pos; 321 322 /* Check if access intersects with the MSI-X Enable bit */ 323 if (off > enable_pos || off + sz <= enable_pos) 324 return; 325 326 /* Read byte that contains the Enable bit */ 327 flags = *(u8 *)(data + enable_pos - off) << 8; 328 329 mutex_lock(&pdev->msix.mutex); 330 331 msi_set_masked(pdev->msix.virt_state, flags & PCI_MSIX_FLAGS_MASKALL); 332 enable = flags & PCI_MSIX_FLAGS_ENABLE; 333 msi_set_enabled(pdev->msix.virt_state, enable); 334 335 if (enable && vfio_pci_enable_msis(kvm, vdev, true)) 336 vfio_dev_err(vdev, "cannot enable MSIX"); 337 else if (!enable && vfio_pci_disable_msis(kvm, vdev, true)) 338 vfio_dev_err(vdev, "cannot disable MSIX"); 339 340 mutex_unlock(&pdev->msix.mutex); 341 } 342 343 static int vfio_pci_msi_vector_write(struct kvm *kvm, struct vfio_device *vdev, 344 u8 off, u8 *data, u32 sz) 345 { 346 size_t i; 347 u32 mask = 0; 348 size_t mask_pos, start, limit; 349 struct vfio_pci_msi_entry *entry; 350 struct vfio_pci_device *pdev = &vdev->pci; 351 struct msi_cap_64 *msi_cap_64 = PCI_CAP(&pdev->hdr, pdev->msi.pos); 352 353 if (!(msi_cap_64->ctrl & PCI_MSI_FLAGS_MASKBIT)) 354 return 0; 355 356 if (msi_cap_64->ctrl & PCI_MSI_FLAGS_64BIT) 357 mask_pos = PCI_MSI_MASK_64; 358 else 359 mask_pos = PCI_MSI_MASK_32; 360 361 if (off >= mask_pos + 4 || off + sz <= mask_pos) 362 return 0; 363 364 /* Set mask to current state */ 365 for (i = 0; i < pdev->msi.nr_entries; i++) { 366 entry = &pdev->msi.entries[i]; 367 mask |= !!msi_is_masked(entry->virt_state) << i; 368 } 369 370 /* Update mask following the intersection of access and register */ 371 start = max_t(size_t, off, mask_pos); 372 limit = min_t(size_t, off + sz, mask_pos + 4); 373 374 memcpy((void *)&mask + start - mask_pos, data + start - off, 375 limit - start); 376 377 /* Update states if necessary */ 378 for (i = 0; i < pdev->msi.nr_entries; i++) { 379 bool masked = mask & (1 << i); 380 381 entry = &pdev->msi.entries[i]; 382 if (masked != msi_is_masked(entry->virt_state)) { 383 msi_set_masked(entry->virt_state, masked); 384 vfio_pci_update_msi_entry(kvm, vdev, entry); 385 } 386 } 387 388 return 1; 389 } 390 391 static void vfio_pci_msi_cap_write(struct kvm *kvm, struct vfio_device *vdev, 392 u8 off, u8 *data, u32 sz) 393 { 394 u8 ctrl; 395 struct msi_msg msg; 396 size_t i, nr_vectors; 397 struct vfio_pci_msi_entry *entry; 398 struct vfio_pci_device *pdev = &vdev->pci; 399 struct msi_cap_64 *msi_cap_64 = PCI_CAP(&pdev->hdr, pdev->msi.pos); 400 401 off -= pdev->msi.pos; 402 403 mutex_lock(&pdev->msi.mutex); 404 405 /* Check if the guest is trying to update mask bits */ 406 if (vfio_pci_msi_vector_write(kvm, vdev, off, data, sz)) 407 goto out_unlock; 408 409 /* Only modify routes when guest pokes the enable bit */ 410 if (off > PCI_MSI_FLAGS || off + sz <= PCI_MSI_FLAGS) 411 goto out_unlock; 412 413 ctrl = *(u8 *)(data + PCI_MSI_FLAGS - off); 414 415 msi_set_enabled(pdev->msi.virt_state, ctrl & PCI_MSI_FLAGS_ENABLE); 416 417 if (!msi_is_enabled(pdev->msi.virt_state)) { 418 vfio_pci_disable_msis(kvm, vdev, false); 419 goto out_unlock; 420 } 421 422 /* Create routes for the requested vectors */ 423 nr_vectors = 1 << ((ctrl & PCI_MSI_FLAGS_QSIZE) >> 4); 424 425 msg.address_lo = msi_cap_64->address_lo; 426 if (msi_cap_64->ctrl & PCI_MSI_FLAGS_64BIT) { 427 msg.address_hi = msi_cap_64->address_hi; 428 msg.data = msi_cap_64->data; 429 } else { 430 struct msi_cap_32 *msi_cap_32 = (void *)msi_cap_64; 431 msg.address_hi = 0; 432 msg.data = msi_cap_32->data; 433 } 434 435 for (i = 0; i < nr_vectors; i++) { 436 entry = &pdev->msi.entries[i]; 437 entry->config.msg = msg; 438 vfio_pci_update_msi_entry(kvm, vdev, entry); 439 } 440 441 /* Update the physical capability if necessary */ 442 if (vfio_pci_enable_msis(kvm, vdev, false)) 443 vfio_dev_err(vdev, "cannot enable MSI"); 444 445 out_unlock: 446 mutex_unlock(&pdev->msi.mutex); 447 } 448 449 static void vfio_pci_cfg_read(struct kvm *kvm, struct pci_device_header *pci_hdr, 450 u8 offset, void *data, int sz) 451 { 452 struct vfio_region_info *info; 453 struct vfio_pci_device *pdev; 454 struct vfio_device *vdev; 455 char base[sz]; 456 457 pdev = container_of(pci_hdr, struct vfio_pci_device, hdr); 458 vdev = container_of(pdev, struct vfio_device, pci); 459 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 460 461 /* Dummy read in case of side-effects */ 462 if (pread(vdev->fd, base, sz, info->offset + offset) != sz) 463 vfio_dev_warn(vdev, "failed to read %d bytes from Configuration Space at 0x%x", 464 sz, offset); 465 } 466 467 static void vfio_pci_cfg_write(struct kvm *kvm, struct pci_device_header *pci_hdr, 468 u8 offset, void *data, int sz) 469 { 470 struct vfio_region_info *info; 471 struct vfio_pci_device *pdev; 472 struct vfio_device *vdev; 473 void *base = pci_hdr; 474 475 pdev = container_of(pci_hdr, struct vfio_pci_device, hdr); 476 vdev = container_of(pdev, struct vfio_device, pci); 477 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 478 479 if (pwrite(vdev->fd, data, sz, info->offset + offset) != sz) 480 vfio_dev_warn(vdev, "Failed to write %d bytes to Configuration Space at 0x%x", 481 sz, offset); 482 483 /* Handle MSI write now, since it might update the hardware capability */ 484 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) 485 vfio_pci_msix_cap_write(kvm, vdev, offset, data, sz); 486 487 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSI) 488 vfio_pci_msi_cap_write(kvm, vdev, offset, data, sz); 489 490 if (pread(vdev->fd, base + offset, sz, info->offset + offset) != sz) 491 vfio_dev_warn(vdev, "Failed to read %d bytes from Configuration Space at 0x%x", 492 sz, offset); 493 } 494 495 static ssize_t vfio_pci_msi_cap_size(struct msi_cap_64 *cap_hdr) 496 { 497 size_t size = 10; 498 499 if (cap_hdr->ctrl & PCI_MSI_FLAGS_64BIT) 500 size += 4; 501 if (cap_hdr->ctrl & PCI_MSI_FLAGS_MASKBIT) 502 size += 10; 503 504 return size; 505 } 506 507 static ssize_t vfio_pci_cap_size(struct pci_cap_hdr *cap_hdr) 508 { 509 switch (cap_hdr->type) { 510 case PCI_CAP_ID_MSIX: 511 return PCI_CAP_MSIX_SIZEOF; 512 case PCI_CAP_ID_MSI: 513 return vfio_pci_msi_cap_size((void *)cap_hdr); 514 default: 515 pr_err("unknown PCI capability 0x%x", cap_hdr->type); 516 return 0; 517 } 518 } 519 520 static int vfio_pci_add_cap(struct vfio_device *vdev, u8 *virt_hdr, 521 struct pci_cap_hdr *cap, off_t pos) 522 { 523 struct pci_cap_hdr *last; 524 struct pci_device_header *hdr = &vdev->pci.hdr; 525 526 cap->next = 0; 527 528 if (!hdr->capabilities) { 529 hdr->capabilities = pos; 530 hdr->status |= PCI_STATUS_CAP_LIST; 531 } else { 532 last = PCI_CAP(virt_hdr, hdr->capabilities); 533 534 while (last->next) 535 last = PCI_CAP(virt_hdr, last->next); 536 537 last->next = pos; 538 } 539 540 memcpy(virt_hdr + pos, cap, vfio_pci_cap_size(cap)); 541 542 return 0; 543 } 544 545 static int vfio_pci_parse_caps(struct vfio_device *vdev) 546 { 547 int ret; 548 size_t size; 549 u8 pos, next; 550 struct pci_cap_hdr *cap; 551 u8 virt_hdr[PCI_DEV_CFG_SIZE]; 552 struct vfio_pci_device *pdev = &vdev->pci; 553 554 if (!(pdev->hdr.status & PCI_STATUS_CAP_LIST)) 555 return 0; 556 557 memset(virt_hdr, 0, PCI_DEV_CFG_SIZE); 558 559 pos = pdev->hdr.capabilities & ~3; 560 561 pdev->hdr.status &= ~PCI_STATUS_CAP_LIST; 562 pdev->hdr.capabilities = 0; 563 564 for (; pos; pos = next) { 565 cap = PCI_CAP(&pdev->hdr, pos); 566 next = cap->next; 567 568 switch (cap->type) { 569 case PCI_CAP_ID_MSIX: 570 ret = vfio_pci_add_cap(vdev, virt_hdr, cap, pos); 571 if (ret) 572 return ret; 573 574 pdev->msix.pos = pos; 575 pdev->irq_modes |= VFIO_PCI_IRQ_MODE_MSIX; 576 break; 577 case PCI_CAP_ID_MSI: 578 ret = vfio_pci_add_cap(vdev, virt_hdr, cap, pos); 579 if (ret) 580 return ret; 581 582 pdev->msi.pos = pos; 583 pdev->irq_modes |= VFIO_PCI_IRQ_MODE_MSI; 584 break; 585 } 586 } 587 588 /* Wipe remaining capabilities */ 589 pos = PCI_STD_HEADER_SIZEOF; 590 size = PCI_DEV_CFG_SIZE - PCI_STD_HEADER_SIZEOF; 591 memcpy((void *)&pdev->hdr + pos, virt_hdr + pos, size); 592 593 return 0; 594 } 595 596 static int vfio_pci_parse_cfg_space(struct vfio_device *vdev) 597 { 598 ssize_t sz = PCI_DEV_CFG_SIZE; 599 struct vfio_region_info *info; 600 struct vfio_pci_device *pdev = &vdev->pci; 601 602 if (vdev->info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX) { 603 vfio_dev_err(vdev, "Config Space not found"); 604 return -ENODEV; 605 } 606 607 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 608 *info = (struct vfio_region_info) { 609 .argsz = sizeof(*info), 610 .index = VFIO_PCI_CONFIG_REGION_INDEX, 611 }; 612 613 ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, info); 614 if (!info->size) { 615 vfio_dev_err(vdev, "Config Space has size zero?!"); 616 return -EINVAL; 617 } 618 619 /* Read standard headers and capabilities */ 620 if (pread(vdev->fd, &pdev->hdr, sz, info->offset) != sz) { 621 vfio_dev_err(vdev, "failed to read %zd bytes of Config Space", sz); 622 return -EIO; 623 } 624 625 /* Strip bit 7, that indicates multifunction */ 626 pdev->hdr.header_type &= 0x7f; 627 628 if (pdev->hdr.header_type != PCI_HEADER_TYPE_NORMAL) { 629 vfio_dev_err(vdev, "unsupported header type %u", 630 pdev->hdr.header_type); 631 return -EOPNOTSUPP; 632 } 633 634 if (pdev->hdr.irq_pin) 635 pdev->irq_modes |= VFIO_PCI_IRQ_MODE_INTX; 636 637 vfio_pci_parse_caps(vdev); 638 639 return 0; 640 } 641 642 static int vfio_pci_fixup_cfg_space(struct vfio_device *vdev) 643 { 644 int i; 645 ssize_t hdr_sz; 646 struct msix_cap *msix; 647 struct vfio_region_info *info; 648 struct vfio_pci_device *pdev = &vdev->pci; 649 650 /* Initialise the BARs */ 651 for (i = VFIO_PCI_BAR0_REGION_INDEX; i <= VFIO_PCI_BAR5_REGION_INDEX; ++i) { 652 u64 base; 653 struct vfio_region *region = &vdev->regions[i]; 654 655 /* Construct a fake reg to match what we've mapped. */ 656 if (region->is_ioport) { 657 base = (region->port_base & PCI_BASE_ADDRESS_IO_MASK) | 658 PCI_BASE_ADDRESS_SPACE_IO; 659 } else { 660 base = (region->guest_phys_addr & 661 PCI_BASE_ADDRESS_MEM_MASK) | 662 PCI_BASE_ADDRESS_SPACE_MEMORY; 663 } 664 665 pdev->hdr.bar[i] = base; 666 667 if (!base) 668 continue; 669 670 pdev->hdr.bar_size[i] = region->info.size; 671 } 672 673 /* I really can't be bothered to support cardbus. */ 674 pdev->hdr.card_bus = 0; 675 676 /* 677 * Nuke the expansion ROM for now. If we want to do this properly, 678 * we need to save its size somewhere and map into the guest. 679 */ 680 pdev->hdr.exp_rom_bar = 0; 681 682 /* Plumb in our fake MSI-X capability, if we have it. */ 683 msix = pci_find_cap(&pdev->hdr, PCI_CAP_ID_MSIX); 684 if (msix) { 685 /* Add a shortcut to the PBA region for the MMIO handler */ 686 int pba_index = VFIO_PCI_BAR0_REGION_INDEX + pdev->msix_pba.bar; 687 pdev->msix_pba.offset = vdev->regions[pba_index].info.offset + 688 (msix->pba_offset & PCI_MSIX_PBA_OFFSET); 689 690 /* Tidy up the capability */ 691 msix->table_offset &= PCI_MSIX_TABLE_BIR; 692 msix->pba_offset &= PCI_MSIX_PBA_BIR; 693 if (pdev->msix_table.bar == pdev->msix_pba.bar) 694 msix->pba_offset |= pdev->msix_table.size & 695 PCI_MSIX_PBA_OFFSET; 696 } 697 698 /* Install our fake Configuration Space */ 699 info = &vdev->regions[VFIO_PCI_CONFIG_REGION_INDEX].info; 700 hdr_sz = PCI_DEV_CFG_SIZE; 701 if (pwrite(vdev->fd, &pdev->hdr, hdr_sz, info->offset) != hdr_sz) { 702 vfio_dev_err(vdev, "failed to write %zd bytes to Config Space", 703 hdr_sz); 704 return -EIO; 705 } 706 707 /* Register callbacks for cfg accesses */ 708 pdev->hdr.cfg_ops = (struct pci_config_operations) { 709 .read = vfio_pci_cfg_read, 710 .write = vfio_pci_cfg_write, 711 }; 712 713 pdev->hdr.irq_type = IRQ_TYPE_LEVEL_HIGH; 714 715 return 0; 716 } 717 718 static int vfio_pci_get_region_info(struct vfio_device *vdev, u32 index, 719 struct vfio_region_info *info) 720 { 721 int ret; 722 723 *info = (struct vfio_region_info) { 724 .argsz = sizeof(*info), 725 .index = index, 726 }; 727 728 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, info); 729 if (ret) { 730 ret = -errno; 731 vfio_dev_err(vdev, "cannot get info for BAR %u", index); 732 return ret; 733 } 734 735 if (info->size && !is_power_of_two(info->size)) { 736 vfio_dev_err(vdev, "region is not power of two: 0x%llx", 737 info->size); 738 return -EINVAL; 739 } 740 741 return 0; 742 } 743 744 static int vfio_pci_create_msix_table(struct kvm *kvm, struct vfio_device *vdev) 745 { 746 int ret; 747 size_t i; 748 size_t map_size; 749 size_t nr_entries; 750 struct vfio_pci_msi_entry *entries; 751 struct vfio_pci_device *pdev = &vdev->pci; 752 struct vfio_pci_msix_pba *pba = &pdev->msix_pba; 753 struct vfio_pci_msix_table *table = &pdev->msix_table; 754 struct msix_cap *msix = PCI_CAP(&pdev->hdr, pdev->msix.pos); 755 struct vfio_region_info info; 756 757 table->bar = msix->table_offset & PCI_MSIX_TABLE_BIR; 758 pba->bar = msix->pba_offset & PCI_MSIX_TABLE_BIR; 759 760 /* 761 * KVM needs memory regions to be multiple of and aligned on PAGE_SIZE. 762 */ 763 nr_entries = (msix->ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 764 table->size = ALIGN(nr_entries * PCI_MSIX_ENTRY_SIZE, PAGE_SIZE); 765 pba->size = ALIGN(DIV_ROUND_UP(nr_entries, 64), PAGE_SIZE); 766 767 entries = calloc(nr_entries, sizeof(struct vfio_pci_msi_entry)); 768 if (!entries) 769 return -ENOMEM; 770 771 for (i = 0; i < nr_entries; i++) 772 entries[i].config.ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; 773 774 ret = vfio_pci_get_region_info(vdev, table->bar, &info); 775 if (ret) 776 return ret; 777 if (!info.size) 778 return -EINVAL; 779 map_size = info.size; 780 781 if (table->bar != pba->bar) { 782 ret = vfio_pci_get_region_info(vdev, pba->bar, &info); 783 if (ret) 784 return ret; 785 if (!info.size) 786 return -EINVAL; 787 map_size += info.size; 788 } 789 790 /* 791 * To ease MSI-X cap configuration in case they share the same BAR, 792 * collapse table and pending array. The size of the BAR regions must be 793 * powers of two. 794 */ 795 map_size = ALIGN(map_size, PAGE_SIZE); 796 table->guest_phys_addr = pci_get_mmio_block(map_size); 797 if (!table->guest_phys_addr) { 798 pr_err("cannot allocate MMIO space"); 799 ret = -ENOMEM; 800 goto out_free; 801 } 802 pba->guest_phys_addr = table->guest_phys_addr + table->size; 803 804 ret = kvm__register_mmio(kvm, table->guest_phys_addr, table->size, 805 false, vfio_pci_msix_table_access, pdev); 806 if (ret < 0) 807 goto out_free; 808 809 /* 810 * We could map the physical PBA directly into the guest, but it's 811 * likely smaller than a page, and we can only hand full pages to the 812 * guest. Even though the PCI spec disallows sharing a page used for 813 * MSI-X with any other resource, it allows to share the same page 814 * between MSI-X table and PBA. For the sake of isolation, create a 815 * virtual PBA. 816 */ 817 ret = kvm__register_mmio(kvm, pba->guest_phys_addr, pba->size, false, 818 vfio_pci_msix_pba_access, pdev); 819 if (ret < 0) 820 goto out_free; 821 822 pdev->msix.entries = entries; 823 pdev->msix.nr_entries = nr_entries; 824 825 return 0; 826 827 out_free: 828 free(entries); 829 830 return ret; 831 } 832 833 static int vfio_pci_create_msi_cap(struct kvm *kvm, struct vfio_pci_device *pdev) 834 { 835 struct msi_cap_64 *cap = PCI_CAP(&pdev->hdr, pdev->msi.pos); 836 837 pdev->msi.nr_entries = 1 << ((cap->ctrl & PCI_MSI_FLAGS_QMASK) >> 1), 838 pdev->msi.entries = calloc(pdev->msi.nr_entries, 839 sizeof(struct vfio_pci_msi_entry)); 840 if (!pdev->msi.entries) 841 return -ENOMEM; 842 843 return 0; 844 } 845 846 static int vfio_pci_configure_bar(struct kvm *kvm, struct vfio_device *vdev, 847 size_t nr) 848 { 849 int ret; 850 u32 bar; 851 size_t map_size; 852 struct vfio_pci_device *pdev = &vdev->pci; 853 struct vfio_region *region = &vdev->regions[nr]; 854 855 if (nr >= vdev->info.num_regions) 856 return 0; 857 858 bar = pdev->hdr.bar[nr]; 859 860 region->vdev = vdev; 861 region->is_ioport = !!(bar & PCI_BASE_ADDRESS_SPACE_IO); 862 863 ret = vfio_pci_get_region_info(vdev, nr, ®ion->info); 864 if (ret) 865 return ret; 866 867 /* Ignore invalid or unimplemented regions */ 868 if (!region->info.size) 869 return 0; 870 871 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) { 872 /* Trap and emulate MSI-X table */ 873 if (nr == pdev->msix_table.bar) { 874 region->guest_phys_addr = pdev->msix_table.guest_phys_addr; 875 return 0; 876 } else if (nr == pdev->msix_pba.bar) { 877 region->guest_phys_addr = pdev->msix_pba.guest_phys_addr; 878 return 0; 879 } 880 } 881 882 if (!region->is_ioport) { 883 /* Grab some MMIO space in the guest */ 884 map_size = ALIGN(region->info.size, PAGE_SIZE); 885 region->guest_phys_addr = pci_get_mmio_block(map_size); 886 } 887 888 /* Map the BARs into the guest or setup a trap region. */ 889 ret = vfio_map_region(kvm, vdev, region); 890 if (ret) 891 return ret; 892 893 return 0; 894 } 895 896 static int vfio_pci_configure_dev_regions(struct kvm *kvm, 897 struct vfio_device *vdev) 898 { 899 int ret; 900 u32 bar; 901 size_t i; 902 bool is_64bit = false; 903 struct vfio_pci_device *pdev = &vdev->pci; 904 905 ret = vfio_pci_parse_cfg_space(vdev); 906 if (ret) 907 return ret; 908 909 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) { 910 ret = vfio_pci_create_msix_table(kvm, vdev); 911 if (ret) 912 return ret; 913 } 914 915 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSI) { 916 ret = vfio_pci_create_msi_cap(kvm, pdev); 917 if (ret) 918 return ret; 919 } 920 921 for (i = VFIO_PCI_BAR0_REGION_INDEX; i <= VFIO_PCI_BAR5_REGION_INDEX; ++i) { 922 /* Ignore top half of 64-bit BAR */ 923 if (is_64bit) { 924 is_64bit = false; 925 continue; 926 } 927 928 ret = vfio_pci_configure_bar(kvm, vdev, i); 929 if (ret) 930 return ret; 931 932 bar = pdev->hdr.bar[i]; 933 is_64bit = (bar & PCI_BASE_ADDRESS_SPACE) == 934 PCI_BASE_ADDRESS_SPACE_MEMORY && 935 bar & PCI_BASE_ADDRESS_MEM_TYPE_64; 936 } 937 938 /* We've configured the BARs, fake up a Configuration Space */ 939 return vfio_pci_fixup_cfg_space(vdev); 940 } 941 942 /* 943 * Attempt to update the FD limit, if opening an eventfd for each IRQ vector 944 * would hit the limit. Which is likely to happen when a device uses 2048 MSIs. 945 */ 946 static int vfio_pci_reserve_irq_fds(size_t num) 947 { 948 /* 949 * I counted around 27 fds under normal load. Let's add 100 for good 950 * measure. 951 */ 952 static size_t needed = 128; 953 struct rlimit fd_limit, new_limit; 954 955 needed += num; 956 957 if (getrlimit(RLIMIT_NOFILE, &fd_limit)) { 958 perror("getrlimit(RLIMIT_NOFILE)"); 959 return 0; 960 } 961 962 if (fd_limit.rlim_cur >= needed) 963 return 0; 964 965 new_limit.rlim_cur = needed; 966 967 if (fd_limit.rlim_max < needed) 968 /* Try to bump hard limit (root only) */ 969 new_limit.rlim_max = needed; 970 else 971 new_limit.rlim_max = fd_limit.rlim_max; 972 973 if (setrlimit(RLIMIT_NOFILE, &new_limit)) { 974 perror("setrlimit(RLIMIT_NOFILE)"); 975 pr_warning("not enough FDs for full MSI-X support (estimated need: %zu)", 976 (size_t)(needed - fd_limit.rlim_cur)); 977 } 978 979 return 0; 980 } 981 982 static int vfio_pci_init_msis(struct kvm *kvm, struct vfio_device *vdev, 983 struct vfio_pci_msi_common *msis) 984 { 985 int ret; 986 size_t i; 987 int *eventfds; 988 size_t irq_set_size; 989 struct vfio_pci_msi_entry *entry; 990 size_t nr_entries = msis->nr_entries; 991 992 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &msis->info); 993 if (ret || msis->info.count == 0) { 994 vfio_dev_err(vdev, "no MSI reported by VFIO"); 995 return -ENODEV; 996 } 997 998 if (!(msis->info.flags & VFIO_IRQ_INFO_EVENTFD)) { 999 vfio_dev_err(vdev, "interrupt not EVENTFD capable"); 1000 return -EINVAL; 1001 } 1002 1003 if (msis->info.count != nr_entries) { 1004 vfio_dev_err(vdev, "invalid number of MSIs reported by VFIO"); 1005 return -EINVAL; 1006 } 1007 1008 mutex_init(&msis->mutex); 1009 1010 vfio_pci_reserve_irq_fds(nr_entries); 1011 1012 irq_set_size = sizeof(struct vfio_irq_set) + nr_entries * sizeof(int); 1013 msis->irq_set = malloc(irq_set_size); 1014 if (!msis->irq_set) 1015 return -ENOMEM; 1016 1017 *msis->irq_set = (struct vfio_irq_set) { 1018 .argsz = irq_set_size, 1019 .flags = VFIO_IRQ_SET_DATA_EVENTFD | 1020 VFIO_IRQ_SET_ACTION_TRIGGER, 1021 .index = msis->info.index, 1022 .start = 0, 1023 .count = nr_entries, 1024 }; 1025 1026 eventfds = (void *)msis->irq_set + sizeof(struct vfio_irq_set); 1027 1028 for (i = 0; i < nr_entries; i++) { 1029 entry = &msis->entries[i]; 1030 entry->gsi = -1; 1031 entry->eventfd = -1; 1032 msi_set_masked(entry->virt_state, true); 1033 msi_set_masked(entry->phys_state, true); 1034 eventfds[i] = -1; 1035 } 1036 1037 return 0; 1038 } 1039 1040 static void vfio_pci_disable_intx(struct kvm *kvm, struct vfio_device *vdev) 1041 { 1042 struct vfio_pci_device *pdev = &vdev->pci; 1043 int gsi = pdev->intx_gsi; 1044 struct vfio_irq_set irq_set = { 1045 .argsz = sizeof(irq_set), 1046 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 1047 .index = VFIO_PCI_INTX_IRQ_INDEX, 1048 }; 1049 1050 if (pdev->intx_fd == -1) 1051 return; 1052 1053 pr_debug("user requested MSI, disabling INTx %d", gsi); 1054 1055 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 1056 irq__del_irqfd(kvm, gsi, pdev->intx_fd); 1057 1058 close(pdev->intx_fd); 1059 close(pdev->unmask_fd); 1060 pdev->intx_fd = -1; 1061 } 1062 1063 static int vfio_pci_enable_intx(struct kvm *kvm, struct vfio_device *vdev) 1064 { 1065 int ret; 1066 int trigger_fd, unmask_fd; 1067 union vfio_irq_eventfd trigger; 1068 union vfio_irq_eventfd unmask; 1069 struct vfio_pci_device *pdev = &vdev->pci; 1070 int gsi = pdev->intx_gsi; 1071 1072 if (pdev->intx_fd != -1) 1073 return 0; 1074 1075 /* 1076 * PCI IRQ is level-triggered, so we use two eventfds. trigger_fd 1077 * signals an interrupt from host to guest, and unmask_fd signals the 1078 * deassertion of the line from guest to host. 1079 */ 1080 trigger_fd = eventfd(0, 0); 1081 if (trigger_fd < 0) { 1082 vfio_dev_err(vdev, "failed to create trigger eventfd"); 1083 return trigger_fd; 1084 } 1085 1086 unmask_fd = eventfd(0, 0); 1087 if (unmask_fd < 0) { 1088 vfio_dev_err(vdev, "failed to create unmask eventfd"); 1089 close(trigger_fd); 1090 return unmask_fd; 1091 } 1092 1093 ret = irq__add_irqfd(kvm, gsi, trigger_fd, unmask_fd); 1094 if (ret) 1095 goto err_close; 1096 1097 trigger.irq = (struct vfio_irq_set) { 1098 .argsz = sizeof(trigger), 1099 .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 1100 .index = VFIO_PCI_INTX_IRQ_INDEX, 1101 .start = 0, 1102 .count = 1, 1103 }; 1104 set_vfio_irq_eventd_payload(&trigger, trigger_fd); 1105 1106 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &trigger); 1107 if (ret < 0) { 1108 vfio_dev_err(vdev, "failed to setup VFIO IRQ"); 1109 goto err_delete_line; 1110 } 1111 1112 unmask.irq = (struct vfio_irq_set) { 1113 .argsz = sizeof(unmask), 1114 .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK, 1115 .index = VFIO_PCI_INTX_IRQ_INDEX, 1116 .start = 0, 1117 .count = 1, 1118 }; 1119 set_vfio_irq_eventd_payload(&unmask, unmask_fd); 1120 1121 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &unmask); 1122 if (ret < 0) { 1123 vfio_dev_err(vdev, "failed to setup unmask IRQ"); 1124 goto err_remove_event; 1125 } 1126 1127 pdev->intx_fd = trigger_fd; 1128 pdev->unmask_fd = unmask_fd; 1129 1130 return 0; 1131 1132 err_remove_event: 1133 /* Remove trigger event */ 1134 trigger.irq.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER; 1135 trigger.irq.count = 0; 1136 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &trigger); 1137 1138 err_delete_line: 1139 irq__del_irqfd(kvm, gsi, trigger_fd); 1140 1141 err_close: 1142 close(trigger_fd); 1143 close(unmask_fd); 1144 return ret; 1145 } 1146 1147 static int vfio_pci_init_intx(struct kvm *kvm, struct vfio_device *vdev) 1148 { 1149 int ret; 1150 struct vfio_pci_device *pdev = &vdev->pci; 1151 struct vfio_irq_info irq_info = { 1152 .argsz = sizeof(irq_info), 1153 .index = VFIO_PCI_INTX_IRQ_INDEX, 1154 }; 1155 1156 vfio_pci_reserve_irq_fds(2); 1157 1158 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); 1159 if (ret || irq_info.count == 0) { 1160 vfio_dev_err(vdev, "no INTx reported by VFIO"); 1161 return -ENODEV; 1162 } 1163 1164 if (!(irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) { 1165 vfio_dev_err(vdev, "interrupt not eventfd capable"); 1166 return -EINVAL; 1167 } 1168 1169 if (!(irq_info.flags & VFIO_IRQ_INFO_AUTOMASKED)) { 1170 vfio_dev_err(vdev, "INTx interrupt not AUTOMASKED"); 1171 return -EINVAL; 1172 } 1173 1174 /* Guest is going to ovewrite our irq_line... */ 1175 pdev->intx_gsi = pdev->hdr.irq_line - KVM_IRQ_OFFSET; 1176 1177 pdev->intx_fd = -1; 1178 1179 return 0; 1180 } 1181 1182 static int vfio_pci_configure_dev_irqs(struct kvm *kvm, struct vfio_device *vdev) 1183 { 1184 int ret = 0; 1185 struct vfio_pci_device *pdev = &vdev->pci; 1186 1187 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSIX) { 1188 pdev->msix.info = (struct vfio_irq_info) { 1189 .argsz = sizeof(pdev->msix.info), 1190 .index = VFIO_PCI_MSIX_IRQ_INDEX, 1191 }; 1192 ret = vfio_pci_init_msis(kvm, vdev, &pdev->msix); 1193 if (ret) 1194 return ret; 1195 } 1196 1197 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_MSI) { 1198 pdev->msi.info = (struct vfio_irq_info) { 1199 .argsz = sizeof(pdev->msi.info), 1200 .index = VFIO_PCI_MSI_IRQ_INDEX, 1201 }; 1202 ret = vfio_pci_init_msis(kvm, vdev, &pdev->msi); 1203 if (ret) 1204 return ret; 1205 } 1206 1207 if (pdev->irq_modes & VFIO_PCI_IRQ_MODE_INTX) { 1208 ret = vfio_pci_init_intx(kvm, vdev); 1209 if (ret) 1210 return ret; 1211 1212 ret = vfio_pci_enable_intx(kvm, vdev); 1213 } 1214 1215 return ret; 1216 } 1217 1218 int vfio_pci_setup_device(struct kvm *kvm, struct vfio_device *vdev) 1219 { 1220 int ret; 1221 1222 ret = vfio_pci_configure_dev_regions(kvm, vdev); 1223 if (ret) { 1224 vfio_dev_err(vdev, "failed to configure regions"); 1225 return ret; 1226 } 1227 1228 vdev->dev_hdr = (struct device_header) { 1229 .bus_type = DEVICE_BUS_PCI, 1230 .data = &vdev->pci.hdr, 1231 }; 1232 1233 ret = device__register(&vdev->dev_hdr); 1234 if (ret) { 1235 vfio_dev_err(vdev, "failed to register VFIO device"); 1236 return ret; 1237 } 1238 1239 ret = vfio_pci_configure_dev_irqs(kvm, vdev); 1240 if (ret) { 1241 vfio_dev_err(vdev, "failed to configure IRQs"); 1242 return ret; 1243 } 1244 1245 return 0; 1246 } 1247 1248 void vfio_pci_teardown_device(struct kvm *kvm, struct vfio_device *vdev) 1249 { 1250 size_t i; 1251 struct vfio_pci_device *pdev = &vdev->pci; 1252 1253 for (i = 0; i < vdev->info.num_regions; i++) 1254 vfio_unmap_region(kvm, &vdev->regions[i]); 1255 1256 device__unregister(&vdev->dev_hdr); 1257 1258 free(pdev->msix.irq_set); 1259 free(pdev->msix.entries); 1260 free(pdev->msi.irq_set); 1261 free(pdev->msi.entries); 1262 } 1263