1 /* 2 * vhost-vdpa 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <linux/vhost.h> 14 #include <linux/vfio.h> 15 #include <sys/eventfd.h> 16 #include <sys/ioctl.h> 17 #include "hw/virtio/vhost.h" 18 #include "hw/virtio/vhost-backend.h" 19 #include "hw/virtio/virtio-net.h" 20 #include "hw/virtio/vhost-shadow-virtqueue.h" 21 #include "hw/virtio/vhost-vdpa.h" 22 #include "exec/address-spaces.h" 23 #include "migration/blocker.h" 24 #include "qemu/cutils.h" 25 #include "qemu/main-loop.h" 26 #include "cpu.h" 27 #include "trace.h" 28 #include "qapi/error.h" 29 30 /* 31 * Return one past the end of the end of section. Be careful with uint64_t 32 * conversions! 33 */ 34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section) 35 { 36 Int128 llend = int128_make64(section->offset_within_address_space); 37 llend = int128_add(llend, section->size); 38 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); 39 40 return llend; 41 } 42 43 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section, 44 uint64_t iova_min, 45 uint64_t iova_max, 46 int page_mask) 47 { 48 Int128 llend; 49 50 if ((!memory_region_is_ram(section->mr) && 51 !memory_region_is_iommu(section->mr)) || 52 memory_region_is_protected(section->mr) || 53 /* vhost-vDPA doesn't allow MMIO to be mapped */ 54 memory_region_is_ram_device(section->mr)) { 55 return true; 56 } 57 58 if (section->offset_within_address_space < iova_min) { 59 error_report("RAM section out of device range (min=0x%" PRIx64 60 ", addr=0x%" HWADDR_PRIx ")", 61 iova_min, section->offset_within_address_space); 62 return true; 63 } 64 /* 65 * While using vIOMMU, sometimes the section will be larger than iova_max, 66 * but the memory that actually maps is smaller, so move the check to 67 * function vhost_vdpa_iommu_map_notify(). That function will use the actual 68 * size that maps to the kernel 69 */ 70 71 if (!memory_region_is_iommu(section->mr)) { 72 llend = vhost_vdpa_section_end(section); 73 if (int128_gt(llend, int128_make64(iova_max))) { 74 error_report("RAM section out of device range (max=0x%" PRIx64 75 ", end addr=0x%" PRIx64 ")", 76 iova_max, int128_get64(llend)); 77 return true; 78 } 79 } 80 81 return false; 82 } 83 84 /* 85 * The caller must set asid = 0 if the device does not support asid. 86 * This is not an ABI break since it is set to 0 by the initializer anyway. 87 */ 88 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 89 hwaddr size, void *vaddr, bool readonly) 90 { 91 struct vhost_msg_v2 msg = {}; 92 int fd = v->device_fd; 93 int ret = 0; 94 95 msg.type = v->msg_type; 96 msg.asid = asid; 97 msg.iotlb.iova = iova; 98 msg.iotlb.size = size; 99 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 100 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 101 msg.iotlb.type = VHOST_IOTLB_UPDATE; 102 103 trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.asid, msg.iotlb.iova, 104 msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm, 105 msg.iotlb.type); 106 107 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 108 error_report("failed to write, fd=%d, errno=%d (%s)", 109 fd, errno, strerror(errno)); 110 return -EIO ; 111 } 112 113 return ret; 114 } 115 116 /* 117 * The caller must set asid = 0 if the device does not support asid. 118 * This is not an ABI break since it is set to 0 by the initializer anyway. 119 */ 120 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 121 hwaddr size) 122 { 123 struct vhost_msg_v2 msg = {}; 124 int fd = v->device_fd; 125 int ret = 0; 126 127 msg.type = v->msg_type; 128 msg.asid = asid; 129 msg.iotlb.iova = iova; 130 msg.iotlb.size = size; 131 msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 132 133 trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.asid, msg.iotlb.iova, 134 msg.iotlb.size, msg.iotlb.type); 135 136 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 137 error_report("failed to write, fd=%d, errno=%d (%s)", 138 fd, errno, strerror(errno)); 139 return -EIO ; 140 } 141 142 return ret; 143 } 144 145 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v) 146 { 147 int fd = v->device_fd; 148 struct vhost_msg_v2 msg = { 149 .type = v->msg_type, 150 .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, 151 }; 152 153 trace_vhost_vdpa_listener_begin_batch(v, fd, msg.type, msg.iotlb.type); 154 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 155 error_report("failed to write, fd=%d, errno=%d (%s)", 156 fd, errno, strerror(errno)); 157 } 158 } 159 160 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v) 161 { 162 if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && 163 !v->iotlb_batch_begin_sent) { 164 vhost_vdpa_listener_begin_batch(v); 165 } 166 167 v->iotlb_batch_begin_sent = true; 168 } 169 170 static void vhost_vdpa_listener_commit(MemoryListener *listener) 171 { 172 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 173 struct vhost_dev *dev = v->dev; 174 struct vhost_msg_v2 msg = {}; 175 int fd = v->device_fd; 176 177 if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { 178 return; 179 } 180 181 if (!v->iotlb_batch_begin_sent) { 182 return; 183 } 184 185 msg.type = v->msg_type; 186 msg.iotlb.type = VHOST_IOTLB_BATCH_END; 187 188 trace_vhost_vdpa_listener_commit(v, fd, msg.type, msg.iotlb.type); 189 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 190 error_report("failed to write, fd=%d, errno=%d (%s)", 191 fd, errno, strerror(errno)); 192 } 193 194 v->iotlb_batch_begin_sent = false; 195 } 196 197 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 198 { 199 struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n); 200 201 hwaddr iova = iotlb->iova + iommu->iommu_offset; 202 struct vhost_vdpa *v = iommu->dev; 203 void *vaddr; 204 int ret; 205 Int128 llend; 206 207 if (iotlb->target_as != &address_space_memory) { 208 error_report("Wrong target AS \"%s\", only system memory is allowed", 209 iotlb->target_as->name ? iotlb->target_as->name : "none"); 210 return; 211 } 212 RCU_READ_LOCK_GUARD(); 213 /* check if RAM section out of device range */ 214 llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova)); 215 if (int128_gt(llend, int128_make64(v->iova_range.last))) { 216 error_report("RAM section out of device range (max=0x%" PRIx64 217 ", end addr=0x%" PRIx64 ")", 218 v->iova_range.last, int128_get64(llend)); 219 return; 220 } 221 222 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 223 bool read_only; 224 225 if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) { 226 return; 227 } 228 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 229 iotlb->addr_mask + 1, vaddr, read_only); 230 if (ret) { 231 error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", " 232 "0x%" HWADDR_PRIx ", %p) = %d (%m)", 233 v, iova, iotlb->addr_mask + 1, vaddr, ret); 234 } 235 } else { 236 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 237 iotlb->addr_mask + 1); 238 if (ret) { 239 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 240 "0x%" HWADDR_PRIx ") = %d (%m)", 241 v, iova, iotlb->addr_mask + 1, ret); 242 } 243 } 244 } 245 246 static void vhost_vdpa_iommu_region_add(MemoryListener *listener, 247 MemoryRegionSection *section) 248 { 249 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 250 251 struct vdpa_iommu *iommu; 252 Int128 end; 253 int iommu_idx; 254 IOMMUMemoryRegion *iommu_mr; 255 int ret; 256 257 iommu_mr = IOMMU_MEMORY_REGION(section->mr); 258 259 iommu = g_malloc0(sizeof(*iommu)); 260 end = int128_add(int128_make64(section->offset_within_region), 261 section->size); 262 end = int128_sub(end, int128_one()); 263 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 264 MEMTXATTRS_UNSPECIFIED); 265 iommu->iommu_mr = iommu_mr; 266 iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify, 267 IOMMU_NOTIFIER_IOTLB_EVENTS, 268 section->offset_within_region, 269 int128_get64(end), 270 iommu_idx); 271 iommu->iommu_offset = section->offset_within_address_space - 272 section->offset_within_region; 273 iommu->dev = v; 274 275 ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 276 if (ret) { 277 g_free(iommu); 278 return; 279 } 280 281 QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next); 282 memory_region_iommu_replay(iommu->iommu_mr, &iommu->n); 283 284 return; 285 } 286 287 static void vhost_vdpa_iommu_region_del(MemoryListener *listener, 288 MemoryRegionSection *section) 289 { 290 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 291 292 struct vdpa_iommu *iommu; 293 294 QLIST_FOREACH(iommu, &v->iommu_list, iommu_next) 295 { 296 if (MEMORY_REGION(iommu->iommu_mr) == section->mr && 297 iommu->n.start == section->offset_within_region) { 298 memory_region_unregister_iommu_notifier(section->mr, &iommu->n); 299 QLIST_REMOVE(iommu, iommu_next); 300 g_free(iommu); 301 break; 302 } 303 } 304 } 305 306 static void vhost_vdpa_listener_region_add(MemoryListener *listener, 307 MemoryRegionSection *section) 308 { 309 DMAMap mem_region = {}; 310 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 311 hwaddr iova; 312 Int128 llend, llsize; 313 void *vaddr; 314 int ret; 315 316 if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first, 317 v->iova_range.last, TARGET_PAGE_MASK)) { 318 return; 319 } 320 if (memory_region_is_iommu(section->mr)) { 321 vhost_vdpa_iommu_region_add(listener, section); 322 return; 323 } 324 325 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != 326 (section->offset_within_region & ~TARGET_PAGE_MASK))) { 327 trace_vhost_vdpa_listener_region_add_unaligned(v, section->mr->name, 328 section->offset_within_address_space & ~TARGET_PAGE_MASK, 329 section->offset_within_region & ~TARGET_PAGE_MASK); 330 return; 331 } 332 333 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); 334 llend = vhost_vdpa_section_end(section); 335 if (int128_ge(int128_make64(iova), llend)) { 336 return; 337 } 338 339 memory_region_ref(section->mr); 340 341 /* Here we assume that memory_region_is_ram(section->mr)==true */ 342 343 vaddr = memory_region_get_ram_ptr(section->mr) + 344 section->offset_within_region + 345 (iova - section->offset_within_address_space); 346 347 trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend), 348 vaddr, section->readonly); 349 350 llsize = int128_sub(llend, int128_make64(iova)); 351 if (v->shadow_data) { 352 int r; 353 354 mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr, 355 mem_region.size = int128_get64(llsize) - 1, 356 mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly), 357 358 r = vhost_iova_tree_map_alloc(v->iova_tree, &mem_region); 359 if (unlikely(r != IOVA_OK)) { 360 error_report("Can't allocate a mapping (%d)", r); 361 goto fail; 362 } 363 364 iova = mem_region.iova; 365 } 366 367 vhost_vdpa_iotlb_batch_begin_once(v); 368 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 369 int128_get64(llsize), vaddr, section->readonly); 370 if (ret) { 371 error_report("vhost vdpa map fail!"); 372 goto fail_map; 373 } 374 375 return; 376 377 fail_map: 378 if (v->shadow_data) { 379 vhost_iova_tree_remove(v->iova_tree, mem_region); 380 } 381 382 fail: 383 /* 384 * On the initfn path, store the first error in the container so we 385 * can gracefully fail. Runtime, there's not much we can do other 386 * than throw a hardware error. 387 */ 388 error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 389 return; 390 391 } 392 393 static void vhost_vdpa_listener_region_del(MemoryListener *listener, 394 MemoryRegionSection *section) 395 { 396 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 397 hwaddr iova; 398 Int128 llend, llsize; 399 int ret; 400 401 if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first, 402 v->iova_range.last, TARGET_PAGE_MASK)) { 403 return; 404 } 405 if (memory_region_is_iommu(section->mr)) { 406 vhost_vdpa_iommu_region_del(listener, section); 407 } 408 409 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != 410 (section->offset_within_region & ~TARGET_PAGE_MASK))) { 411 trace_vhost_vdpa_listener_region_del_unaligned(v, section->mr->name, 412 section->offset_within_address_space & ~TARGET_PAGE_MASK, 413 section->offset_within_region & ~TARGET_PAGE_MASK); 414 return; 415 } 416 417 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); 418 llend = vhost_vdpa_section_end(section); 419 420 trace_vhost_vdpa_listener_region_del(v, iova, 421 int128_get64(int128_sub(llend, int128_one()))); 422 423 if (int128_ge(int128_make64(iova), llend)) { 424 return; 425 } 426 427 llsize = int128_sub(llend, int128_make64(iova)); 428 429 if (v->shadow_data) { 430 const DMAMap *result; 431 const void *vaddr = memory_region_get_ram_ptr(section->mr) + 432 section->offset_within_region + 433 (iova - section->offset_within_address_space); 434 DMAMap mem_region = { 435 .translated_addr = (hwaddr)(uintptr_t)vaddr, 436 .size = int128_get64(llsize) - 1, 437 }; 438 439 result = vhost_iova_tree_find_iova(v->iova_tree, &mem_region); 440 if (!result) { 441 /* The memory listener map wasn't mapped */ 442 return; 443 } 444 iova = result->iova; 445 vhost_iova_tree_remove(v->iova_tree, *result); 446 } 447 vhost_vdpa_iotlb_batch_begin_once(v); 448 /* 449 * The unmap ioctl doesn't accept a full 64-bit. need to check it 450 */ 451 if (int128_eq(llsize, int128_2_64())) { 452 llsize = int128_rshift(llsize, 1); 453 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 454 int128_get64(llsize)); 455 456 if (ret) { 457 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 458 "0x%" HWADDR_PRIx ") = %d (%m)", 459 v, iova, int128_get64(llsize), ret); 460 } 461 iova += int128_get64(llsize); 462 } 463 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 464 int128_get64(llsize)); 465 466 if (ret) { 467 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 468 "0x%" HWADDR_PRIx ") = %d (%m)", 469 v, iova, int128_get64(llsize), ret); 470 } 471 472 memory_region_unref(section->mr); 473 } 474 /* 475 * IOTLB API is used by vhost-vdpa which requires incremental updating 476 * of the mapping. So we can not use generic vhost memory listener which 477 * depends on the addnop(). 478 */ 479 static const MemoryListener vhost_vdpa_memory_listener = { 480 .name = "vhost-vdpa", 481 .commit = vhost_vdpa_listener_commit, 482 .region_add = vhost_vdpa_listener_region_add, 483 .region_del = vhost_vdpa_listener_region_del, 484 }; 485 486 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 487 void *arg) 488 { 489 struct vhost_vdpa *v = dev->opaque; 490 int fd = v->device_fd; 491 int ret; 492 493 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 494 495 ret = ioctl(fd, request, arg); 496 return ret < 0 ? -errno : ret; 497 } 498 499 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 500 { 501 uint8_t s; 502 int ret; 503 504 trace_vhost_vdpa_add_status(dev, status); 505 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 506 if (ret < 0) { 507 return ret; 508 } 509 510 s |= status; 511 512 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 513 if (ret < 0) { 514 return ret; 515 } 516 517 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 518 if (ret < 0) { 519 return ret; 520 } 521 522 if (!(s & status)) { 523 return -EIO; 524 } 525 526 return 0; 527 } 528 529 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range) 530 { 531 int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range); 532 533 return ret < 0 ? -errno : 0; 534 } 535 536 /* 537 * The use of this function is for requests that only need to be 538 * applied once. Typically such request occurs at the beginning 539 * of operation, and before setting up queues. It should not be 540 * used for request that performs operation until all queues are 541 * set, which would need to check dev->vq_index_end instead. 542 */ 543 static bool vhost_vdpa_first_dev(struct vhost_dev *dev) 544 { 545 struct vhost_vdpa *v = dev->opaque; 546 547 return v->index == 0; 548 } 549 550 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev, 551 uint64_t *features) 552 { 553 int ret; 554 555 ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 556 trace_vhost_vdpa_get_features(dev, *features); 557 return ret; 558 } 559 560 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v) 561 { 562 g_autoptr(GPtrArray) shadow_vqs = NULL; 563 564 shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free); 565 for (unsigned n = 0; n < hdev->nvqs; ++n) { 566 VhostShadowVirtqueue *svq; 567 568 svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque); 569 g_ptr_array_add(shadow_vqs, svq); 570 } 571 572 v->shadow_vqs = g_steal_pointer(&shadow_vqs); 573 } 574 575 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) 576 { 577 struct vhost_vdpa *v; 578 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 579 trace_vhost_vdpa_init(dev, opaque); 580 int ret; 581 582 v = opaque; 583 v->dev = dev; 584 dev->opaque = opaque ; 585 v->listener = vhost_vdpa_memory_listener; 586 v->msg_type = VHOST_IOTLB_MSG_V2; 587 vhost_vdpa_init_svq(dev, v); 588 589 error_propagate(&dev->migration_blocker, v->migration_blocker); 590 if (!vhost_vdpa_first_dev(dev)) { 591 return 0; 592 } 593 594 /* 595 * If dev->shadow_vqs_enabled at initialization that means the device has 596 * been started with x-svq=on, so don't block migration 597 */ 598 if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { 599 /* We don't have dev->features yet */ 600 uint64_t features; 601 ret = vhost_vdpa_get_dev_features(dev, &features); 602 if (unlikely(ret)) { 603 error_setg_errno(errp, -ret, "Could not get device features"); 604 return ret; 605 } 606 vhost_svq_valid_features(features, &dev->migration_blocker); 607 } 608 609 /* 610 * Similar to VFIO, we end up pinning all guest memory and have to 611 * disable discarding of RAM. 612 */ 613 ret = ram_block_discard_disable(true); 614 if (ret) { 615 error_report("Cannot set discarding of RAM broken"); 616 return ret; 617 } 618 619 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 620 VIRTIO_CONFIG_S_DRIVER); 621 622 return 0; 623 } 624 625 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, 626 int queue_index) 627 { 628 size_t page_size = qemu_real_host_page_size(); 629 struct vhost_vdpa *v = dev->opaque; 630 VirtIODevice *vdev = dev->vdev; 631 VhostVDPAHostNotifier *n; 632 633 n = &v->notifier[queue_index]; 634 635 if (n->addr) { 636 virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); 637 object_unparent(OBJECT(&n->mr)); 638 munmap(n->addr, page_size); 639 n->addr = NULL; 640 } 641 } 642 643 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) 644 { 645 size_t page_size = qemu_real_host_page_size(); 646 struct vhost_vdpa *v = dev->opaque; 647 VirtIODevice *vdev = dev->vdev; 648 VhostVDPAHostNotifier *n; 649 int fd = v->device_fd; 650 void *addr; 651 char *name; 652 653 vhost_vdpa_host_notifier_uninit(dev, queue_index); 654 655 n = &v->notifier[queue_index]; 656 657 addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, 658 queue_index * page_size); 659 if (addr == MAP_FAILED) { 660 goto err; 661 } 662 663 name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", 664 v, queue_index); 665 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 666 page_size, addr); 667 g_free(name); 668 669 if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { 670 object_unparent(OBJECT(&n->mr)); 671 munmap(addr, page_size); 672 goto err; 673 } 674 n->addr = addr; 675 676 return 0; 677 678 err: 679 return -1; 680 } 681 682 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) 683 { 684 int i; 685 686 /* 687 * Pack all the changes to the memory regions in a single 688 * transaction to avoid a few updating of the address space 689 * topology. 690 */ 691 memory_region_transaction_begin(); 692 693 for (i = dev->vq_index; i < dev->vq_index + n; i++) { 694 vhost_vdpa_host_notifier_uninit(dev, i); 695 } 696 697 memory_region_transaction_commit(); 698 } 699 700 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) 701 { 702 struct vhost_vdpa *v = dev->opaque; 703 int i; 704 705 if (v->shadow_vqs_enabled) { 706 /* FIXME SVQ is not compatible with host notifiers mr */ 707 return; 708 } 709 710 /* 711 * Pack all the changes to the memory regions in a single 712 * transaction to avoid a few updating of the address space 713 * topology. 714 */ 715 memory_region_transaction_begin(); 716 717 for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { 718 if (vhost_vdpa_host_notifier_init(dev, i)) { 719 vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index); 720 break; 721 } 722 } 723 724 memory_region_transaction_commit(); 725 } 726 727 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev) 728 { 729 struct vhost_vdpa *v = dev->opaque; 730 size_t idx; 731 732 for (idx = 0; idx < v->shadow_vqs->len; ++idx) { 733 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx)); 734 } 735 g_ptr_array_free(v->shadow_vqs, true); 736 } 737 738 static int vhost_vdpa_cleanup(struct vhost_dev *dev) 739 { 740 struct vhost_vdpa *v; 741 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 742 v = dev->opaque; 743 trace_vhost_vdpa_cleanup(dev, v); 744 if (vhost_vdpa_first_dev(dev)) { 745 ram_block_discard_disable(false); 746 } 747 748 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 749 memory_listener_unregister(&v->listener); 750 vhost_vdpa_svq_cleanup(dev); 751 752 dev->opaque = NULL; 753 754 return 0; 755 } 756 757 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 758 { 759 trace_vhost_vdpa_memslots_limit(dev, INT_MAX); 760 return INT_MAX; 761 } 762 763 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 764 struct vhost_memory *mem) 765 { 766 if (!vhost_vdpa_first_dev(dev)) { 767 return 0; 768 } 769 770 trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding); 771 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) && 772 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) { 773 int i; 774 for (i = 0; i < mem->nregions; i++) { 775 trace_vhost_vdpa_dump_regions(dev, i, 776 mem->regions[i].guest_phys_addr, 777 mem->regions[i].memory_size, 778 mem->regions[i].userspace_addr, 779 mem->regions[i].flags_padding); 780 } 781 } 782 if (mem->padding) { 783 return -EINVAL; 784 } 785 786 return 0; 787 } 788 789 static int vhost_vdpa_set_features(struct vhost_dev *dev, 790 uint64_t features) 791 { 792 struct vhost_vdpa *v = dev->opaque; 793 int ret; 794 795 if (!vhost_vdpa_first_dev(dev)) { 796 return 0; 797 } 798 799 if (v->shadow_vqs_enabled) { 800 if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) { 801 /* 802 * QEMU is just trying to enable or disable logging. SVQ handles 803 * this sepparately, so no need to forward this. 804 */ 805 v->acked_features = features; 806 return 0; 807 } 808 809 v->acked_features = features; 810 811 /* We must not ack _F_LOG if SVQ is enabled */ 812 features &= ~BIT_ULL(VHOST_F_LOG_ALL); 813 } 814 815 trace_vhost_vdpa_set_features(dev, features); 816 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 817 if (ret) { 818 return ret; 819 } 820 821 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 822 } 823 824 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) 825 { 826 uint64_t features; 827 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 828 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | 829 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | 830 0x1ULL << VHOST_BACKEND_F_SUSPEND; 831 int r; 832 833 if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { 834 return -EFAULT; 835 } 836 837 features &= f; 838 839 if (vhost_vdpa_first_dev(dev)) { 840 r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); 841 if (r) { 842 return -EFAULT; 843 } 844 } 845 846 dev->backend_cap = features; 847 848 return 0; 849 } 850 851 static int vhost_vdpa_get_device_id(struct vhost_dev *dev, 852 uint32_t *device_id) 853 { 854 int ret; 855 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 856 trace_vhost_vdpa_get_device_id(dev, *device_id); 857 return ret; 858 } 859 860 static int vhost_vdpa_reset_device(struct vhost_dev *dev) 861 { 862 struct vhost_vdpa *v = dev->opaque; 863 int ret; 864 uint8_t status = 0; 865 866 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 867 trace_vhost_vdpa_reset_device(dev); 868 v->suspended = false; 869 return ret; 870 } 871 872 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 873 { 874 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 875 876 trace_vhost_vdpa_get_vq_index(dev, idx, idx); 877 return idx; 878 } 879 880 static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev) 881 { 882 int i; 883 trace_vhost_vdpa_set_vring_ready(dev); 884 for (i = 0; i < dev->nvqs; ++i) { 885 struct vhost_vring_state state = { 886 .index = dev->vq_index + i, 887 .num = 1, 888 }; 889 vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 890 } 891 return 0; 892 } 893 894 static int vhost_vdpa_set_config_call(struct vhost_dev *dev, 895 int fd) 896 { 897 trace_vhost_vdpa_set_config_call(dev, fd); 898 return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd); 899 } 900 901 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config, 902 uint32_t config_len) 903 { 904 int b, len; 905 char line[QEMU_HEXDUMP_LINE_LEN]; 906 907 for (b = 0; b < config_len; b += 16) { 908 len = config_len - b; 909 qemu_hexdump_line(line, b, config, len, false); 910 trace_vhost_vdpa_dump_config(dev, line); 911 } 912 } 913 914 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 915 uint32_t offset, uint32_t size, 916 uint32_t flags) 917 { 918 struct vhost_vdpa_config *config; 919 int ret; 920 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 921 922 trace_vhost_vdpa_set_config(dev, offset, size, flags); 923 config = g_malloc(size + config_size); 924 config->off = offset; 925 config->len = size; 926 memcpy(config->buf, data, size); 927 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) && 928 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 929 vhost_vdpa_dump_config(dev, data, size); 930 } 931 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 932 g_free(config); 933 return ret; 934 } 935 936 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 937 uint32_t config_len, Error **errp) 938 { 939 struct vhost_vdpa_config *v_config; 940 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 941 int ret; 942 943 trace_vhost_vdpa_get_config(dev, config, config_len); 944 v_config = g_malloc(config_len + config_size); 945 v_config->len = config_len; 946 v_config->off = 0; 947 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 948 memcpy(config, v_config->buf, config_len); 949 g_free(v_config); 950 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) && 951 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 952 vhost_vdpa_dump_config(dev, config, config_len); 953 } 954 return ret; 955 } 956 957 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev, 958 struct vhost_vring_state *ring) 959 { 960 trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num); 961 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 962 } 963 964 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev, 965 struct vhost_vring_file *file) 966 { 967 trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd); 968 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 969 } 970 971 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev, 972 struct vhost_vring_file *file) 973 { 974 trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd); 975 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 976 } 977 978 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev, 979 struct vhost_vring_addr *addr) 980 { 981 trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags, 982 addr->desc_user_addr, addr->used_user_addr, 983 addr->avail_user_addr, 984 addr->log_guest_addr); 985 986 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 987 988 } 989 990 /** 991 * Set the shadow virtqueue descriptors to the device 992 * 993 * @dev: The vhost device model 994 * @svq: The shadow virtqueue 995 * @idx: The index of the virtqueue in the vhost device 996 * @errp: Error 997 * 998 * Note that this function does not rewind kick file descriptor if cannot set 999 * call one. 1000 */ 1001 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev, 1002 VhostShadowVirtqueue *svq, unsigned idx, 1003 Error **errp) 1004 { 1005 struct vhost_vring_file file = { 1006 .index = dev->vq_index + idx, 1007 }; 1008 const EventNotifier *event_notifier = &svq->hdev_kick; 1009 int r; 1010 1011 r = event_notifier_init(&svq->hdev_kick, 0); 1012 if (r != 0) { 1013 error_setg_errno(errp, -r, "Couldn't create kick event notifier"); 1014 goto err_init_hdev_kick; 1015 } 1016 1017 r = event_notifier_init(&svq->hdev_call, 0); 1018 if (r != 0) { 1019 error_setg_errno(errp, -r, "Couldn't create call event notifier"); 1020 goto err_init_hdev_call; 1021 } 1022 1023 file.fd = event_notifier_get_fd(event_notifier); 1024 r = vhost_vdpa_set_vring_dev_kick(dev, &file); 1025 if (unlikely(r != 0)) { 1026 error_setg_errno(errp, -r, "Can't set device kick fd"); 1027 goto err_init_set_dev_fd; 1028 } 1029 1030 event_notifier = &svq->hdev_call; 1031 file.fd = event_notifier_get_fd(event_notifier); 1032 r = vhost_vdpa_set_vring_dev_call(dev, &file); 1033 if (unlikely(r != 0)) { 1034 error_setg_errno(errp, -r, "Can't set device call fd"); 1035 goto err_init_set_dev_fd; 1036 } 1037 1038 return 0; 1039 1040 err_init_set_dev_fd: 1041 event_notifier_set_handler(&svq->hdev_call, NULL); 1042 1043 err_init_hdev_call: 1044 event_notifier_cleanup(&svq->hdev_kick); 1045 1046 err_init_hdev_kick: 1047 return r; 1048 } 1049 1050 /** 1051 * Unmap a SVQ area in the device 1052 */ 1053 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr) 1054 { 1055 const DMAMap needle = { 1056 .translated_addr = addr, 1057 }; 1058 const DMAMap *result = vhost_iova_tree_find_iova(v->iova_tree, &needle); 1059 hwaddr size; 1060 int r; 1061 1062 if (unlikely(!result)) { 1063 error_report("Unable to find SVQ address to unmap"); 1064 return; 1065 } 1066 1067 size = ROUND_UP(result->size, qemu_real_host_page_size()); 1068 r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size); 1069 if (unlikely(r < 0)) { 1070 error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r); 1071 return; 1072 } 1073 1074 vhost_iova_tree_remove(v->iova_tree, *result); 1075 } 1076 1077 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev, 1078 const VhostShadowVirtqueue *svq) 1079 { 1080 struct vhost_vdpa *v = dev->opaque; 1081 struct vhost_vring_addr svq_addr; 1082 1083 vhost_svq_get_vring_addr(svq, &svq_addr); 1084 1085 vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr); 1086 1087 vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr); 1088 } 1089 1090 /** 1091 * Map the SVQ area in the device 1092 * 1093 * @v: Vhost-vdpa device 1094 * @needle: The area to search iova 1095 * @errorp: Error pointer 1096 */ 1097 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle, 1098 Error **errp) 1099 { 1100 int r; 1101 1102 r = vhost_iova_tree_map_alloc(v->iova_tree, needle); 1103 if (unlikely(r != IOVA_OK)) { 1104 error_setg(errp, "Cannot allocate iova (%d)", r); 1105 return false; 1106 } 1107 1108 r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova, 1109 needle->size + 1, 1110 (void *)(uintptr_t)needle->translated_addr, 1111 needle->perm == IOMMU_RO); 1112 if (unlikely(r != 0)) { 1113 error_setg_errno(errp, -r, "Cannot map region to device"); 1114 vhost_iova_tree_remove(v->iova_tree, *needle); 1115 } 1116 1117 return r == 0; 1118 } 1119 1120 /** 1121 * Map the shadow virtqueue rings in the device 1122 * 1123 * @dev: The vhost device 1124 * @svq: The shadow virtqueue 1125 * @addr: Assigned IOVA addresses 1126 * @errp: Error pointer 1127 */ 1128 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev, 1129 const VhostShadowVirtqueue *svq, 1130 struct vhost_vring_addr *addr, 1131 Error **errp) 1132 { 1133 ERRP_GUARD(); 1134 DMAMap device_region, driver_region; 1135 struct vhost_vring_addr svq_addr; 1136 struct vhost_vdpa *v = dev->opaque; 1137 size_t device_size = vhost_svq_device_area_size(svq); 1138 size_t driver_size = vhost_svq_driver_area_size(svq); 1139 size_t avail_offset; 1140 bool ok; 1141 1142 vhost_svq_get_vring_addr(svq, &svq_addr); 1143 1144 driver_region = (DMAMap) { 1145 .translated_addr = svq_addr.desc_user_addr, 1146 .size = driver_size - 1, 1147 .perm = IOMMU_RO, 1148 }; 1149 ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp); 1150 if (unlikely(!ok)) { 1151 error_prepend(errp, "Cannot create vq driver region: "); 1152 return false; 1153 } 1154 addr->desc_user_addr = driver_region.iova; 1155 avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr; 1156 addr->avail_user_addr = driver_region.iova + avail_offset; 1157 1158 device_region = (DMAMap) { 1159 .translated_addr = svq_addr.used_user_addr, 1160 .size = device_size - 1, 1161 .perm = IOMMU_RW, 1162 }; 1163 ok = vhost_vdpa_svq_map_ring(v, &device_region, errp); 1164 if (unlikely(!ok)) { 1165 error_prepend(errp, "Cannot create vq device region: "); 1166 vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr); 1167 } 1168 addr->used_user_addr = device_region.iova; 1169 1170 return ok; 1171 } 1172 1173 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev, 1174 VhostShadowVirtqueue *svq, unsigned idx, 1175 Error **errp) 1176 { 1177 uint16_t vq_index = dev->vq_index + idx; 1178 struct vhost_vring_state s = { 1179 .index = vq_index, 1180 }; 1181 int r; 1182 1183 r = vhost_vdpa_set_dev_vring_base(dev, &s); 1184 if (unlikely(r)) { 1185 error_setg_errno(errp, -r, "Cannot set vring base"); 1186 return false; 1187 } 1188 1189 r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp); 1190 return r == 0; 1191 } 1192 1193 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev) 1194 { 1195 struct vhost_vdpa *v = dev->opaque; 1196 Error *err = NULL; 1197 unsigned i; 1198 1199 if (!v->shadow_vqs_enabled) { 1200 return true; 1201 } 1202 1203 for (i = 0; i < v->shadow_vqs->len; ++i) { 1204 VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i); 1205 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1206 struct vhost_vring_addr addr = { 1207 .index = dev->vq_index + i, 1208 }; 1209 int r; 1210 bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err); 1211 if (unlikely(!ok)) { 1212 goto err; 1213 } 1214 1215 vhost_svq_start(svq, dev->vdev, vq, v->iova_tree); 1216 ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err); 1217 if (unlikely(!ok)) { 1218 goto err_map; 1219 } 1220 1221 /* Override vring GPA set by vhost subsystem */ 1222 r = vhost_vdpa_set_vring_dev_addr(dev, &addr); 1223 if (unlikely(r != 0)) { 1224 error_setg_errno(&err, -r, "Cannot set device address"); 1225 goto err_set_addr; 1226 } 1227 } 1228 1229 return true; 1230 1231 err_set_addr: 1232 vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i)); 1233 1234 err_map: 1235 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i)); 1236 1237 err: 1238 error_reportf_err(err, "Cannot setup SVQ %u: ", i); 1239 for (unsigned j = 0; j < i; ++j) { 1240 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j); 1241 vhost_vdpa_svq_unmap_rings(dev, svq); 1242 vhost_svq_stop(svq); 1243 } 1244 1245 return false; 1246 } 1247 1248 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) 1249 { 1250 struct vhost_vdpa *v = dev->opaque; 1251 1252 if (!v->shadow_vqs_enabled) { 1253 return; 1254 } 1255 1256 for (unsigned i = 0; i < v->shadow_vqs->len; ++i) { 1257 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1258 1259 vhost_svq_stop(svq); 1260 vhost_vdpa_svq_unmap_rings(dev, svq); 1261 1262 event_notifier_cleanup(&svq->hdev_kick); 1263 event_notifier_cleanup(&svq->hdev_call); 1264 } 1265 } 1266 1267 static void vhost_vdpa_suspend(struct vhost_dev *dev) 1268 { 1269 struct vhost_vdpa *v = dev->opaque; 1270 int r; 1271 1272 if (!vhost_vdpa_first_dev(dev)) { 1273 return; 1274 } 1275 1276 if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { 1277 trace_vhost_vdpa_suspend(dev); 1278 r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND); 1279 if (unlikely(r)) { 1280 error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); 1281 } else { 1282 v->suspended = true; 1283 return; 1284 } 1285 } 1286 1287 vhost_vdpa_reset_device(dev); 1288 } 1289 1290 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 1291 { 1292 struct vhost_vdpa *v = dev->opaque; 1293 bool ok; 1294 trace_vhost_vdpa_dev_start(dev, started); 1295 1296 if (started) { 1297 vhost_vdpa_host_notifiers_init(dev); 1298 ok = vhost_vdpa_svqs_start(dev); 1299 if (unlikely(!ok)) { 1300 return -1; 1301 } 1302 vhost_vdpa_set_vring_ready(dev); 1303 } else { 1304 vhost_vdpa_suspend(dev); 1305 vhost_vdpa_svqs_stop(dev); 1306 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 1307 } 1308 1309 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1310 return 0; 1311 } 1312 1313 if (started) { 1314 if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) { 1315 error_report("SVQ can not work while IOMMU enable, please disable" 1316 "IOMMU and try again"); 1317 return -1; 1318 } 1319 memory_listener_register(&v->listener, dev->vdev->dma_as); 1320 1321 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 1322 } 1323 1324 return 0; 1325 } 1326 1327 static void vhost_vdpa_reset_status(struct vhost_dev *dev) 1328 { 1329 struct vhost_vdpa *v = dev->opaque; 1330 1331 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1332 return; 1333 } 1334 1335 vhost_vdpa_reset_device(dev); 1336 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 1337 VIRTIO_CONFIG_S_DRIVER); 1338 memory_listener_unregister(&v->listener); 1339 } 1340 1341 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 1342 struct vhost_log *log) 1343 { 1344 struct vhost_vdpa *v = dev->opaque; 1345 if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) { 1346 return 0; 1347 } 1348 1349 trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd, 1350 log->log); 1351 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 1352 } 1353 1354 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 1355 struct vhost_vring_addr *addr) 1356 { 1357 struct vhost_vdpa *v = dev->opaque; 1358 1359 if (v->shadow_vqs_enabled) { 1360 /* 1361 * Device vring addr was set at device start. SVQ base is handled by 1362 * VirtQueue code. 1363 */ 1364 return 0; 1365 } 1366 1367 return vhost_vdpa_set_vring_dev_addr(dev, addr); 1368 } 1369 1370 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 1371 struct vhost_vring_state *ring) 1372 { 1373 trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num); 1374 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 1375 } 1376 1377 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 1378 struct vhost_vring_state *ring) 1379 { 1380 struct vhost_vdpa *v = dev->opaque; 1381 1382 if (v->shadow_vqs_enabled) { 1383 /* 1384 * Device vring base was set at device start. SVQ base is handled by 1385 * VirtQueue code. 1386 */ 1387 return 0; 1388 } 1389 1390 return vhost_vdpa_set_dev_vring_base(dev, ring); 1391 } 1392 1393 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 1394 struct vhost_vring_state *ring) 1395 { 1396 struct vhost_vdpa *v = dev->opaque; 1397 int ret; 1398 1399 if (v->shadow_vqs_enabled) { 1400 ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index); 1401 return 0; 1402 } 1403 1404 if (!v->suspended) { 1405 /* 1406 * Cannot trust in value returned by device, let vhost recover used 1407 * idx from guest. 1408 */ 1409 return -1; 1410 } 1411 1412 ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 1413 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); 1414 return ret; 1415 } 1416 1417 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 1418 struct vhost_vring_file *file) 1419 { 1420 struct vhost_vdpa *v = dev->opaque; 1421 int vdpa_idx = file->index - dev->vq_index; 1422 1423 if (v->shadow_vqs_enabled) { 1424 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1425 vhost_svq_set_svq_kick_fd(svq, file->fd); 1426 return 0; 1427 } else { 1428 return vhost_vdpa_set_vring_dev_kick(dev, file); 1429 } 1430 } 1431 1432 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 1433 struct vhost_vring_file *file) 1434 { 1435 struct vhost_vdpa *v = dev->opaque; 1436 int vdpa_idx = file->index - dev->vq_index; 1437 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1438 1439 /* Remember last call fd because we can switch to SVQ anytime. */ 1440 vhost_svq_set_svq_call_fd(svq, file->fd); 1441 if (v->shadow_vqs_enabled) { 1442 return 0; 1443 } 1444 1445 return vhost_vdpa_set_vring_dev_call(dev, file); 1446 } 1447 1448 static int vhost_vdpa_get_features(struct vhost_dev *dev, 1449 uint64_t *features) 1450 { 1451 int ret = vhost_vdpa_get_dev_features(dev, features); 1452 1453 if (ret == 0) { 1454 /* Add SVQ logging capabilities */ 1455 *features |= BIT_ULL(VHOST_F_LOG_ALL); 1456 } 1457 1458 return ret; 1459 } 1460 1461 static int vhost_vdpa_set_owner(struct vhost_dev *dev) 1462 { 1463 if (!vhost_vdpa_first_dev(dev)) { 1464 return 0; 1465 } 1466 1467 trace_vhost_vdpa_set_owner(dev); 1468 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 1469 } 1470 1471 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 1472 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 1473 { 1474 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 1475 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 1476 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 1477 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 1478 trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, 1479 addr->avail_user_addr, addr->used_user_addr); 1480 return 0; 1481 } 1482 1483 static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 1484 { 1485 return true; 1486 } 1487 1488 const VhostOps vdpa_ops = { 1489 .backend_type = VHOST_BACKEND_TYPE_VDPA, 1490 .vhost_backend_init = vhost_vdpa_init, 1491 .vhost_backend_cleanup = vhost_vdpa_cleanup, 1492 .vhost_set_log_base = vhost_vdpa_set_log_base, 1493 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 1494 .vhost_set_vring_num = vhost_vdpa_set_vring_num, 1495 .vhost_set_vring_base = vhost_vdpa_set_vring_base, 1496 .vhost_get_vring_base = vhost_vdpa_get_vring_base, 1497 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 1498 .vhost_set_vring_call = vhost_vdpa_set_vring_call, 1499 .vhost_get_features = vhost_vdpa_get_features, 1500 .vhost_set_backend_cap = vhost_vdpa_set_backend_cap, 1501 .vhost_set_owner = vhost_vdpa_set_owner, 1502 .vhost_set_vring_endian = NULL, 1503 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 1504 .vhost_set_mem_table = vhost_vdpa_set_mem_table, 1505 .vhost_set_features = vhost_vdpa_set_features, 1506 .vhost_reset_device = vhost_vdpa_reset_device, 1507 .vhost_get_vq_index = vhost_vdpa_get_vq_index, 1508 .vhost_get_config = vhost_vdpa_get_config, 1509 .vhost_set_config = vhost_vdpa_set_config, 1510 .vhost_requires_shm_log = NULL, 1511 .vhost_migration_done = NULL, 1512 .vhost_backend_can_merge = NULL, 1513 .vhost_net_set_mtu = NULL, 1514 .vhost_set_iotlb_callback = NULL, 1515 .vhost_send_device_iotlb_msg = NULL, 1516 .vhost_dev_start = vhost_vdpa_dev_start, 1517 .vhost_get_device_id = vhost_vdpa_get_device_id, 1518 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 1519 .vhost_force_iommu = vhost_vdpa_force_iommu, 1520 .vhost_set_config_call = vhost_vdpa_set_config_call, 1521 .vhost_reset_status = vhost_vdpa_reset_status, 1522 }; 1523