1 /* 2 * vhost-vdpa 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <linux/vhost.h> 14 #include <linux/vfio.h> 15 #include <sys/eventfd.h> 16 #include <sys/ioctl.h> 17 #include "exec/target_page.h" 18 #include "hw/virtio/vhost.h" 19 #include "hw/virtio/vhost-backend.h" 20 #include "hw/virtio/virtio-net.h" 21 #include "hw/virtio/vhost-shadow-virtqueue.h" 22 #include "hw/virtio/vhost-vdpa.h" 23 #include "exec/address-spaces.h" 24 #include "migration/blocker.h" 25 #include "qemu/cutils.h" 26 #include "qemu/main-loop.h" 27 #include "trace.h" 28 #include "qapi/error.h" 29 30 /* 31 * Return one past the end of the end of section. Be careful with uint64_t 32 * conversions! 33 */ 34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section, 35 int page_mask) 36 { 37 Int128 llend = int128_make64(section->offset_within_address_space); 38 llend = int128_add(llend, section->size); 39 llend = int128_and(llend, int128_exts64(page_mask)); 40 41 return llend; 42 } 43 44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section, 45 uint64_t iova_min, 46 uint64_t iova_max, 47 int page_mask) 48 { 49 Int128 llend; 50 bool is_ram = memory_region_is_ram(section->mr); 51 bool is_iommu = memory_region_is_iommu(section->mr); 52 bool is_protected = memory_region_is_protected(section->mr); 53 54 /* vhost-vDPA doesn't allow MMIO to be mapped */ 55 bool is_ram_device = memory_region_is_ram_device(section->mr); 56 57 if ((!is_ram && !is_iommu) || is_protected || is_ram_device) { 58 return true; 59 } 60 61 if (section->offset_within_address_space < iova_min) { 62 error_report("RAM section out of device range (min=0x%" PRIx64 63 ", addr=0x%" HWADDR_PRIx ")", 64 iova_min, section->offset_within_address_space); 65 return true; 66 } 67 /* 68 * While using vIOMMU, sometimes the section will be larger than iova_max, 69 * but the memory that actually maps is smaller, so move the check to 70 * function vhost_vdpa_iommu_map_notify(). That function will use the actual 71 * size that maps to the kernel 72 */ 73 74 if (!is_iommu) { 75 llend = vhost_vdpa_section_end(section, page_mask); 76 if (int128_gt(llend, int128_make64(iova_max))) { 77 error_report("RAM section out of device range (max=0x%" PRIx64 78 ", end addr=0x%" PRIx64 ")", 79 iova_max, int128_get64(llend)); 80 return true; 81 } 82 } 83 84 return false; 85 } 86 87 /* 88 * The caller must set asid = 0 if the device does not support asid. 89 * This is not an ABI break since it is set to 0 by the initializer anyway. 90 */ 91 int vhost_vdpa_dma_map(VhostVDPAShared *s, uint32_t asid, hwaddr iova, 92 hwaddr size, void *vaddr, bool readonly) 93 { 94 struct vhost_msg_v2 msg = {}; 95 int fd = s->device_fd; 96 int ret = 0; 97 98 msg.type = VHOST_IOTLB_MSG_V2; 99 msg.asid = asid; 100 msg.iotlb.iova = iova; 101 msg.iotlb.size = size; 102 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 103 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 104 msg.iotlb.type = VHOST_IOTLB_UPDATE; 105 106 trace_vhost_vdpa_dma_map(s, fd, msg.type, msg.asid, msg.iotlb.iova, 107 msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm, 108 msg.iotlb.type); 109 110 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 111 error_report("failed to write, fd=%d, errno=%d (%s)", 112 fd, errno, strerror(errno)); 113 return -EIO ; 114 } 115 116 return ret; 117 } 118 119 /* 120 * The caller must set asid = 0 if the device does not support asid. 121 * This is not an ABI break since it is set to 0 by the initializer anyway. 122 */ 123 int vhost_vdpa_dma_unmap(VhostVDPAShared *s, uint32_t asid, hwaddr iova, 124 hwaddr size) 125 { 126 struct vhost_msg_v2 msg = {}; 127 int fd = s->device_fd; 128 int ret = 0; 129 130 msg.type = VHOST_IOTLB_MSG_V2; 131 msg.asid = asid; 132 msg.iotlb.iova = iova; 133 msg.iotlb.size = size; 134 msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 135 136 trace_vhost_vdpa_dma_unmap(s, fd, msg.type, msg.asid, msg.iotlb.iova, 137 msg.iotlb.size, msg.iotlb.type); 138 139 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 140 error_report("failed to write, fd=%d, errno=%d (%s)", 141 fd, errno, strerror(errno)); 142 return -EIO ; 143 } 144 145 return ret; 146 } 147 148 static void vhost_vdpa_listener_begin_batch(VhostVDPAShared *s) 149 { 150 int fd = s->device_fd; 151 struct vhost_msg_v2 msg = { 152 .type = VHOST_IOTLB_MSG_V2, 153 .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, 154 }; 155 156 trace_vhost_vdpa_listener_begin_batch(s, fd, msg.type, msg.iotlb.type); 157 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 158 error_report("failed to write, fd=%d, errno=%d (%s)", 159 fd, errno, strerror(errno)); 160 } 161 } 162 163 static void vhost_vdpa_iotlb_batch_begin_once(VhostVDPAShared *s) 164 { 165 if (s->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && 166 !s->iotlb_batch_begin_sent) { 167 vhost_vdpa_listener_begin_batch(s); 168 } 169 170 s->iotlb_batch_begin_sent = true; 171 } 172 173 static void vhost_vdpa_listener_commit(MemoryListener *listener) 174 { 175 VhostVDPAShared *s = container_of(listener, VhostVDPAShared, listener); 176 struct vhost_msg_v2 msg = {}; 177 int fd = s->device_fd; 178 179 if (!(s->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { 180 return; 181 } 182 183 if (!s->iotlb_batch_begin_sent) { 184 return; 185 } 186 187 msg.type = VHOST_IOTLB_MSG_V2; 188 msg.iotlb.type = VHOST_IOTLB_BATCH_END; 189 190 trace_vhost_vdpa_listener_commit(s, fd, msg.type, msg.iotlb.type); 191 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 192 error_report("failed to write, fd=%d, errno=%d (%s)", 193 fd, errno, strerror(errno)); 194 } 195 196 s->iotlb_batch_begin_sent = false; 197 } 198 199 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 200 { 201 struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n); 202 203 hwaddr iova = iotlb->iova + iommu->iommu_offset; 204 VhostVDPAShared *s = iommu->dev_shared; 205 void *vaddr; 206 int ret; 207 Int128 llend; 208 209 if (iotlb->target_as != &address_space_memory) { 210 error_report("Wrong target AS \"%s\", only system memory is allowed", 211 iotlb->target_as->name ? iotlb->target_as->name : "none"); 212 return; 213 } 214 RCU_READ_LOCK_GUARD(); 215 /* check if RAM section out of device range */ 216 llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova)); 217 if (int128_gt(llend, int128_make64(s->iova_range.last))) { 218 error_report("RAM section out of device range (max=0x%" PRIx64 219 ", end addr=0x%" PRIx64 ")", 220 s->iova_range.last, int128_get64(llend)); 221 return; 222 } 223 224 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 225 bool read_only; 226 227 if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) { 228 return; 229 } 230 ret = vhost_vdpa_dma_map(s, VHOST_VDPA_GUEST_PA_ASID, iova, 231 iotlb->addr_mask + 1, vaddr, read_only); 232 if (ret) { 233 error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", " 234 "0x%" HWADDR_PRIx ", %p) = %d (%m)", 235 s, iova, iotlb->addr_mask + 1, vaddr, ret); 236 } 237 } else { 238 ret = vhost_vdpa_dma_unmap(s, VHOST_VDPA_GUEST_PA_ASID, iova, 239 iotlb->addr_mask + 1); 240 if (ret) { 241 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 242 "0x%" HWADDR_PRIx ") = %d (%m)", 243 s, iova, iotlb->addr_mask + 1, ret); 244 } 245 } 246 } 247 248 static void vhost_vdpa_iommu_region_add(MemoryListener *listener, 249 MemoryRegionSection *section) 250 { 251 VhostVDPAShared *s = container_of(listener, VhostVDPAShared, listener); 252 253 struct vdpa_iommu *iommu; 254 Int128 end; 255 int iommu_idx; 256 IOMMUMemoryRegion *iommu_mr; 257 int ret; 258 259 iommu_mr = IOMMU_MEMORY_REGION(section->mr); 260 261 iommu = g_malloc0(sizeof(*iommu)); 262 end = int128_add(int128_make64(section->offset_within_region), 263 section->size); 264 end = int128_sub(end, int128_one()); 265 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 266 MEMTXATTRS_UNSPECIFIED); 267 iommu->iommu_mr = iommu_mr; 268 iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify, 269 IOMMU_NOTIFIER_IOTLB_EVENTS, 270 section->offset_within_region, 271 int128_get64(end), 272 iommu_idx); 273 iommu->iommu_offset = section->offset_within_address_space - 274 section->offset_within_region; 275 iommu->dev_shared = s; 276 277 ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 278 if (ret) { 279 g_free(iommu); 280 return; 281 } 282 283 QLIST_INSERT_HEAD(&s->iommu_list, iommu, iommu_next); 284 memory_region_iommu_replay(iommu->iommu_mr, &iommu->n); 285 286 return; 287 } 288 289 static void vhost_vdpa_iommu_region_del(MemoryListener *listener, 290 MemoryRegionSection *section) 291 { 292 VhostVDPAShared *s = container_of(listener, VhostVDPAShared, listener); 293 294 struct vdpa_iommu *iommu; 295 296 QLIST_FOREACH(iommu, &s->iommu_list, iommu_next) 297 { 298 if (MEMORY_REGION(iommu->iommu_mr) == section->mr && 299 iommu->n.start == section->offset_within_region) { 300 memory_region_unregister_iommu_notifier(section->mr, &iommu->n); 301 QLIST_REMOVE(iommu, iommu_next); 302 g_free(iommu); 303 break; 304 } 305 } 306 } 307 308 static void vhost_vdpa_listener_region_add(MemoryListener *listener, 309 MemoryRegionSection *section) 310 { 311 DMAMap mem_region = {}; 312 VhostVDPAShared *s = container_of(listener, VhostVDPAShared, listener); 313 hwaddr iova; 314 Int128 llend, llsize; 315 void *vaddr; 316 int ret; 317 int page_size = qemu_target_page_size(); 318 int page_mask = -page_size; 319 320 if (vhost_vdpa_listener_skipped_section(section, s->iova_range.first, 321 s->iova_range.last, page_mask)) { 322 return; 323 } 324 if (memory_region_is_iommu(section->mr)) { 325 vhost_vdpa_iommu_region_add(listener, section); 326 return; 327 } 328 329 if (unlikely((section->offset_within_address_space & ~page_mask) != 330 (section->offset_within_region & ~page_mask))) { 331 trace_vhost_vdpa_listener_region_add_unaligned(s, section->mr->name, 332 section->offset_within_address_space & ~page_mask, 333 section->offset_within_region & ~page_mask); 334 return; 335 } 336 337 iova = ROUND_UP(section->offset_within_address_space, page_size); 338 llend = vhost_vdpa_section_end(section, page_mask); 339 if (int128_ge(int128_make64(iova), llend)) { 340 return; 341 } 342 343 memory_region_ref(section->mr); 344 345 /* Here we assume that memory_region_is_ram(section->mr)==true */ 346 347 vaddr = memory_region_get_ram_ptr(section->mr) + 348 section->offset_within_region + 349 (iova - section->offset_within_address_space); 350 351 trace_vhost_vdpa_listener_region_add(s, iova, int128_get64(llend), 352 vaddr, section->readonly); 353 354 llsize = int128_sub(llend, int128_make64(iova)); 355 if (s->shadow_data) { 356 int r; 357 358 mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr, 359 mem_region.size = int128_get64(llsize) - 1, 360 mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly), 361 362 r = vhost_iova_tree_map_alloc(s->iova_tree, &mem_region); 363 if (unlikely(r != IOVA_OK)) { 364 error_report("Can't allocate a mapping (%d)", r); 365 goto fail; 366 } 367 368 iova = mem_region.iova; 369 } 370 371 vhost_vdpa_iotlb_batch_begin_once(s); 372 ret = vhost_vdpa_dma_map(s, VHOST_VDPA_GUEST_PA_ASID, iova, 373 int128_get64(llsize), vaddr, section->readonly); 374 if (ret) { 375 error_report("vhost vdpa map fail!"); 376 goto fail_map; 377 } 378 379 return; 380 381 fail_map: 382 if (s->shadow_data) { 383 vhost_iova_tree_remove(s->iova_tree, mem_region); 384 } 385 386 fail: 387 /* 388 * On the initfn path, store the first error in the container so we 389 * can gracefully fail. Runtime, there's not much we can do other 390 * than throw a hardware error. 391 */ 392 error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 393 return; 394 395 } 396 397 static void vhost_vdpa_listener_region_del(MemoryListener *listener, 398 MemoryRegionSection *section) 399 { 400 VhostVDPAShared *s = container_of(listener, VhostVDPAShared, listener); 401 hwaddr iova; 402 Int128 llend, llsize; 403 int ret; 404 int page_size = qemu_target_page_size(); 405 int page_mask = -page_size; 406 407 if (vhost_vdpa_listener_skipped_section(section, s->iova_range.first, 408 s->iova_range.last, page_mask)) { 409 return; 410 } 411 if (memory_region_is_iommu(section->mr)) { 412 vhost_vdpa_iommu_region_del(listener, section); 413 } 414 415 if (unlikely((section->offset_within_address_space & ~page_mask) != 416 (section->offset_within_region & ~page_mask))) { 417 trace_vhost_vdpa_listener_region_del_unaligned(s, section->mr->name, 418 section->offset_within_address_space & ~page_mask, 419 section->offset_within_region & ~page_mask); 420 return; 421 } 422 423 iova = ROUND_UP(section->offset_within_address_space, page_size); 424 llend = vhost_vdpa_section_end(section, page_mask); 425 426 trace_vhost_vdpa_listener_region_del(s, iova, 427 int128_get64(int128_sub(llend, int128_one()))); 428 429 if (int128_ge(int128_make64(iova), llend)) { 430 return; 431 } 432 433 llsize = int128_sub(llend, int128_make64(iova)); 434 435 if (s->shadow_data) { 436 const DMAMap *result; 437 const void *vaddr = memory_region_get_ram_ptr(section->mr) + 438 section->offset_within_region + 439 (iova - section->offset_within_address_space); 440 DMAMap mem_region = { 441 .translated_addr = (hwaddr)(uintptr_t)vaddr, 442 .size = int128_get64(llsize) - 1, 443 }; 444 445 result = vhost_iova_tree_find_iova(s->iova_tree, &mem_region); 446 if (!result) { 447 /* The memory listener map wasn't mapped */ 448 return; 449 } 450 iova = result->iova; 451 vhost_iova_tree_remove(s->iova_tree, *result); 452 } 453 vhost_vdpa_iotlb_batch_begin_once(s); 454 /* 455 * The unmap ioctl doesn't accept a full 64-bit. need to check it 456 */ 457 if (int128_eq(llsize, int128_2_64())) { 458 llsize = int128_rshift(llsize, 1); 459 ret = vhost_vdpa_dma_unmap(s, VHOST_VDPA_GUEST_PA_ASID, iova, 460 int128_get64(llsize)); 461 462 if (ret) { 463 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 464 "0x%" HWADDR_PRIx ") = %d (%m)", 465 s, iova, int128_get64(llsize), ret); 466 } 467 iova += int128_get64(llsize); 468 } 469 ret = vhost_vdpa_dma_unmap(s, VHOST_VDPA_GUEST_PA_ASID, iova, 470 int128_get64(llsize)); 471 472 if (ret) { 473 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 474 "0x%" HWADDR_PRIx ") = %d (%m)", 475 s, iova, int128_get64(llsize), ret); 476 } 477 478 memory_region_unref(section->mr); 479 } 480 /* 481 * IOTLB API is used by vhost-vdpa which requires incremental updating 482 * of the mapping. So we can not use generic vhost memory listener which 483 * depends on the addnop(). 484 */ 485 static const MemoryListener vhost_vdpa_memory_listener = { 486 .name = "vhost-vdpa", 487 .commit = vhost_vdpa_listener_commit, 488 .region_add = vhost_vdpa_listener_region_add, 489 .region_del = vhost_vdpa_listener_region_del, 490 }; 491 492 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 493 void *arg) 494 { 495 struct vhost_vdpa *v = dev->opaque; 496 int fd = v->shared->device_fd; 497 int ret; 498 499 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 500 501 ret = ioctl(fd, request, arg); 502 return ret < 0 ? -errno : ret; 503 } 504 505 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 506 { 507 uint8_t s; 508 int ret; 509 510 trace_vhost_vdpa_add_status(dev, status); 511 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 512 if (ret < 0) { 513 return ret; 514 } 515 if ((s & status) == status) { 516 /* Don't set bits already set */ 517 return 0; 518 } 519 520 s |= status; 521 522 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 523 if (ret < 0) { 524 return ret; 525 } 526 527 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 528 if (ret < 0) { 529 return ret; 530 } 531 532 if (!(s & status)) { 533 return -EIO; 534 } 535 536 return 0; 537 } 538 539 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range) 540 { 541 int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range); 542 543 return ret < 0 ? -errno : 0; 544 } 545 546 /* 547 * The use of this function is for requests that only need to be 548 * applied once. Typically such request occurs at the beginning 549 * of operation, and before setting up queues. It should not be 550 * used for request that performs operation until all queues are 551 * set, which would need to check dev->vq_index_end instead. 552 */ 553 static bool vhost_vdpa_first_dev(struct vhost_dev *dev) 554 { 555 struct vhost_vdpa *v = dev->opaque; 556 557 return v->index == 0; 558 } 559 560 static bool vhost_vdpa_last_dev(struct vhost_dev *dev) 561 { 562 return dev->vq_index + dev->nvqs == dev->vq_index_end; 563 } 564 565 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev, 566 uint64_t *features) 567 { 568 int ret; 569 570 ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 571 trace_vhost_vdpa_get_features(dev, *features); 572 return ret; 573 } 574 575 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v) 576 { 577 g_autoptr(GPtrArray) shadow_vqs = NULL; 578 579 shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free); 580 for (unsigned n = 0; n < hdev->nvqs; ++n) { 581 VhostShadowVirtqueue *svq; 582 583 svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque); 584 g_ptr_array_add(shadow_vqs, svq); 585 } 586 587 v->shadow_vqs = g_steal_pointer(&shadow_vqs); 588 } 589 590 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) 591 { 592 struct vhost_vdpa *v = opaque; 593 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 594 trace_vhost_vdpa_init(dev, v->shared, opaque); 595 int ret; 596 597 v->dev = dev; 598 dev->opaque = opaque ; 599 v->shared->listener = vhost_vdpa_memory_listener; 600 vhost_vdpa_init_svq(dev, v); 601 602 error_propagate(&dev->migration_blocker, v->migration_blocker); 603 if (!vhost_vdpa_first_dev(dev)) { 604 return 0; 605 } 606 607 /* 608 * If dev->shadow_vqs_enabled at initialization that means the device has 609 * been started with x-svq=on, so don't block migration 610 */ 611 if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { 612 /* We don't have dev->features yet */ 613 uint64_t features; 614 ret = vhost_vdpa_get_dev_features(dev, &features); 615 if (unlikely(ret)) { 616 error_setg_errno(errp, -ret, "Could not get device features"); 617 return ret; 618 } 619 vhost_svq_valid_features(features, &dev->migration_blocker); 620 } 621 622 /* 623 * Similar to VFIO, we end up pinning all guest memory and have to 624 * disable discarding of RAM. 625 */ 626 ret = ram_block_discard_disable(true); 627 if (ret) { 628 error_report("Cannot set discarding of RAM broken"); 629 return ret; 630 } 631 632 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 633 VIRTIO_CONFIG_S_DRIVER); 634 635 return 0; 636 } 637 638 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, 639 int queue_index) 640 { 641 size_t page_size = qemu_real_host_page_size(); 642 struct vhost_vdpa *v = dev->opaque; 643 VirtIODevice *vdev = dev->vdev; 644 VhostVDPAHostNotifier *n; 645 646 n = &v->notifier[queue_index]; 647 648 if (n->addr) { 649 virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); 650 object_unparent(OBJECT(&n->mr)); 651 munmap(n->addr, page_size); 652 n->addr = NULL; 653 } 654 } 655 656 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) 657 { 658 size_t page_size = qemu_real_host_page_size(); 659 struct vhost_vdpa *v = dev->opaque; 660 VirtIODevice *vdev = dev->vdev; 661 VhostVDPAHostNotifier *n; 662 int fd = v->shared->device_fd; 663 void *addr; 664 char *name; 665 666 vhost_vdpa_host_notifier_uninit(dev, queue_index); 667 668 n = &v->notifier[queue_index]; 669 670 addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, 671 queue_index * page_size); 672 if (addr == MAP_FAILED) { 673 goto err; 674 } 675 676 name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", 677 v, queue_index); 678 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 679 page_size, addr); 680 g_free(name); 681 682 if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { 683 object_unparent(OBJECT(&n->mr)); 684 munmap(addr, page_size); 685 goto err; 686 } 687 n->addr = addr; 688 689 return 0; 690 691 err: 692 return -1; 693 } 694 695 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) 696 { 697 int i; 698 699 /* 700 * Pack all the changes to the memory regions in a single 701 * transaction to avoid a few updating of the address space 702 * topology. 703 */ 704 memory_region_transaction_begin(); 705 706 for (i = dev->vq_index; i < dev->vq_index + n; i++) { 707 vhost_vdpa_host_notifier_uninit(dev, i); 708 } 709 710 memory_region_transaction_commit(); 711 } 712 713 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) 714 { 715 struct vhost_vdpa *v = dev->opaque; 716 int i; 717 718 if (v->shadow_vqs_enabled) { 719 /* FIXME SVQ is not compatible with host notifiers mr */ 720 return; 721 } 722 723 /* 724 * Pack all the changes to the memory regions in a single 725 * transaction to avoid a few updating of the address space 726 * topology. 727 */ 728 memory_region_transaction_begin(); 729 730 for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { 731 if (vhost_vdpa_host_notifier_init(dev, i)) { 732 vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index); 733 break; 734 } 735 } 736 737 memory_region_transaction_commit(); 738 } 739 740 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev) 741 { 742 struct vhost_vdpa *v = dev->opaque; 743 size_t idx; 744 745 for (idx = 0; idx < v->shadow_vqs->len; ++idx) { 746 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx)); 747 } 748 g_ptr_array_free(v->shadow_vqs, true); 749 } 750 751 static int vhost_vdpa_cleanup(struct vhost_dev *dev) 752 { 753 struct vhost_vdpa *v; 754 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 755 v = dev->opaque; 756 trace_vhost_vdpa_cleanup(dev, v); 757 if (vhost_vdpa_first_dev(dev)) { 758 ram_block_discard_disable(false); 759 memory_listener_unregister(&v->shared->listener); 760 } 761 762 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 763 vhost_vdpa_svq_cleanup(dev); 764 765 dev->opaque = NULL; 766 767 return 0; 768 } 769 770 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 771 { 772 trace_vhost_vdpa_memslots_limit(dev, INT_MAX); 773 return INT_MAX; 774 } 775 776 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 777 struct vhost_memory *mem) 778 { 779 if (!vhost_vdpa_first_dev(dev)) { 780 return 0; 781 } 782 783 trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding); 784 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) && 785 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) { 786 int i; 787 for (i = 0; i < mem->nregions; i++) { 788 trace_vhost_vdpa_dump_regions(dev, i, 789 mem->regions[i].guest_phys_addr, 790 mem->regions[i].memory_size, 791 mem->regions[i].userspace_addr, 792 mem->regions[i].flags_padding); 793 } 794 } 795 if (mem->padding) { 796 return -EINVAL; 797 } 798 799 return 0; 800 } 801 802 static int vhost_vdpa_set_features(struct vhost_dev *dev, 803 uint64_t features) 804 { 805 struct vhost_vdpa *v = dev->opaque; 806 int ret; 807 808 if (!vhost_vdpa_first_dev(dev)) { 809 return 0; 810 } 811 812 if (v->shadow_vqs_enabled) { 813 if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) { 814 /* 815 * QEMU is just trying to enable or disable logging. SVQ handles 816 * this sepparately, so no need to forward this. 817 */ 818 v->acked_features = features; 819 return 0; 820 } 821 822 v->acked_features = features; 823 824 /* We must not ack _F_LOG if SVQ is enabled */ 825 features &= ~BIT_ULL(VHOST_F_LOG_ALL); 826 } 827 828 trace_vhost_vdpa_set_features(dev, features); 829 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 830 if (ret) { 831 return ret; 832 } 833 834 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 835 } 836 837 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) 838 { 839 struct vhost_vdpa *v = dev->opaque; 840 841 uint64_t features; 842 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 843 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | 844 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | 845 0x1ULL << VHOST_BACKEND_F_SUSPEND; 846 int r; 847 848 if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { 849 return -EFAULT; 850 } 851 852 features &= f; 853 854 if (vhost_vdpa_first_dev(dev)) { 855 r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); 856 if (r) { 857 return -EFAULT; 858 } 859 } 860 861 dev->backend_cap = features; 862 v->shared->backend_cap = features; 863 864 return 0; 865 } 866 867 static int vhost_vdpa_get_device_id(struct vhost_dev *dev, 868 uint32_t *device_id) 869 { 870 int ret; 871 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 872 trace_vhost_vdpa_get_device_id(dev, *device_id); 873 return ret; 874 } 875 876 static int vhost_vdpa_reset_device(struct vhost_dev *dev) 877 { 878 struct vhost_vdpa *v = dev->opaque; 879 int ret; 880 uint8_t status = 0; 881 882 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 883 trace_vhost_vdpa_reset_device(dev); 884 v->suspended = false; 885 return ret; 886 } 887 888 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 889 { 890 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 891 892 trace_vhost_vdpa_get_vq_index(dev, idx, idx); 893 return idx; 894 } 895 896 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx) 897 { 898 struct vhost_dev *dev = v->dev; 899 struct vhost_vring_state state = { 900 .index = idx, 901 .num = 1, 902 }; 903 int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 904 905 trace_vhost_vdpa_set_vring_ready(dev, idx, r); 906 return r; 907 } 908 909 static int vhost_vdpa_set_config_call(struct vhost_dev *dev, 910 int fd) 911 { 912 trace_vhost_vdpa_set_config_call(dev, fd); 913 return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd); 914 } 915 916 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config, 917 uint32_t config_len) 918 { 919 int b, len; 920 char line[QEMU_HEXDUMP_LINE_LEN]; 921 922 for (b = 0; b < config_len; b += 16) { 923 len = config_len - b; 924 qemu_hexdump_line(line, b, config, len, false); 925 trace_vhost_vdpa_dump_config(dev, line); 926 } 927 } 928 929 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 930 uint32_t offset, uint32_t size, 931 uint32_t flags) 932 { 933 struct vhost_vdpa_config *config; 934 int ret; 935 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 936 937 trace_vhost_vdpa_set_config(dev, offset, size, flags); 938 config = g_malloc(size + config_size); 939 config->off = offset; 940 config->len = size; 941 memcpy(config->buf, data, size); 942 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) && 943 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 944 vhost_vdpa_dump_config(dev, data, size); 945 } 946 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 947 g_free(config); 948 return ret; 949 } 950 951 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 952 uint32_t config_len, Error **errp) 953 { 954 struct vhost_vdpa_config *v_config; 955 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 956 int ret; 957 958 trace_vhost_vdpa_get_config(dev, config, config_len); 959 v_config = g_malloc(config_len + config_size); 960 v_config->len = config_len; 961 v_config->off = 0; 962 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 963 memcpy(config, v_config->buf, config_len); 964 g_free(v_config); 965 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) && 966 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 967 vhost_vdpa_dump_config(dev, config, config_len); 968 } 969 return ret; 970 } 971 972 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev, 973 struct vhost_vring_state *ring) 974 { 975 struct vhost_vdpa *v = dev->opaque; 976 977 trace_vhost_vdpa_set_dev_vring_base(dev, ring->index, ring->num, 978 v->shadow_vqs_enabled); 979 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 980 } 981 982 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev, 983 struct vhost_vring_file *file) 984 { 985 trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd); 986 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 987 } 988 989 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev, 990 struct vhost_vring_file *file) 991 { 992 trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd); 993 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 994 } 995 996 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev, 997 struct vhost_vring_addr *addr) 998 { 999 trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags, 1000 addr->desc_user_addr, addr->used_user_addr, 1001 addr->avail_user_addr, 1002 addr->log_guest_addr); 1003 1004 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 1005 1006 } 1007 1008 /** 1009 * Set the shadow virtqueue descriptors to the device 1010 * 1011 * @dev: The vhost device model 1012 * @svq: The shadow virtqueue 1013 * @idx: The index of the virtqueue in the vhost device 1014 * @errp: Error 1015 * 1016 * Note that this function does not rewind kick file descriptor if cannot set 1017 * call one. 1018 */ 1019 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev, 1020 VhostShadowVirtqueue *svq, unsigned idx, 1021 Error **errp) 1022 { 1023 struct vhost_vring_file file = { 1024 .index = dev->vq_index + idx, 1025 }; 1026 const EventNotifier *event_notifier = &svq->hdev_kick; 1027 int r; 1028 1029 r = event_notifier_init(&svq->hdev_kick, 0); 1030 if (r != 0) { 1031 error_setg_errno(errp, -r, "Couldn't create kick event notifier"); 1032 goto err_init_hdev_kick; 1033 } 1034 1035 r = event_notifier_init(&svq->hdev_call, 0); 1036 if (r != 0) { 1037 error_setg_errno(errp, -r, "Couldn't create call event notifier"); 1038 goto err_init_hdev_call; 1039 } 1040 1041 file.fd = event_notifier_get_fd(event_notifier); 1042 r = vhost_vdpa_set_vring_dev_kick(dev, &file); 1043 if (unlikely(r != 0)) { 1044 error_setg_errno(errp, -r, "Can't set device kick fd"); 1045 goto err_init_set_dev_fd; 1046 } 1047 1048 event_notifier = &svq->hdev_call; 1049 file.fd = event_notifier_get_fd(event_notifier); 1050 r = vhost_vdpa_set_vring_dev_call(dev, &file); 1051 if (unlikely(r != 0)) { 1052 error_setg_errno(errp, -r, "Can't set device call fd"); 1053 goto err_init_set_dev_fd; 1054 } 1055 1056 return 0; 1057 1058 err_init_set_dev_fd: 1059 event_notifier_set_handler(&svq->hdev_call, NULL); 1060 1061 err_init_hdev_call: 1062 event_notifier_cleanup(&svq->hdev_kick); 1063 1064 err_init_hdev_kick: 1065 return r; 1066 } 1067 1068 /** 1069 * Unmap a SVQ area in the device 1070 */ 1071 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr) 1072 { 1073 const DMAMap needle = { 1074 .translated_addr = addr, 1075 }; 1076 const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree, 1077 &needle); 1078 hwaddr size; 1079 int r; 1080 1081 if (unlikely(!result)) { 1082 error_report("Unable to find SVQ address to unmap"); 1083 return; 1084 } 1085 1086 size = ROUND_UP(result->size, qemu_real_host_page_size()); 1087 r = vhost_vdpa_dma_unmap(v->shared, v->address_space_id, result->iova, 1088 size); 1089 if (unlikely(r < 0)) { 1090 error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r); 1091 return; 1092 } 1093 1094 vhost_iova_tree_remove(v->shared->iova_tree, *result); 1095 } 1096 1097 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev, 1098 const VhostShadowVirtqueue *svq) 1099 { 1100 struct vhost_vdpa *v = dev->opaque; 1101 struct vhost_vring_addr svq_addr; 1102 1103 vhost_svq_get_vring_addr(svq, &svq_addr); 1104 1105 vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr); 1106 1107 vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr); 1108 } 1109 1110 /** 1111 * Map the SVQ area in the device 1112 * 1113 * @v: Vhost-vdpa device 1114 * @needle: The area to search iova 1115 * @errorp: Error pointer 1116 */ 1117 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle, 1118 Error **errp) 1119 { 1120 int r; 1121 1122 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle); 1123 if (unlikely(r != IOVA_OK)) { 1124 error_setg(errp, "Cannot allocate iova (%d)", r); 1125 return false; 1126 } 1127 1128 r = vhost_vdpa_dma_map(v->shared, v->address_space_id, needle->iova, 1129 needle->size + 1, 1130 (void *)(uintptr_t)needle->translated_addr, 1131 needle->perm == IOMMU_RO); 1132 if (unlikely(r != 0)) { 1133 error_setg_errno(errp, -r, "Cannot map region to device"); 1134 vhost_iova_tree_remove(v->shared->iova_tree, *needle); 1135 } 1136 1137 return r == 0; 1138 } 1139 1140 /** 1141 * Map the shadow virtqueue rings in the device 1142 * 1143 * @dev: The vhost device 1144 * @svq: The shadow virtqueue 1145 * @addr: Assigned IOVA addresses 1146 * @errp: Error pointer 1147 */ 1148 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev, 1149 const VhostShadowVirtqueue *svq, 1150 struct vhost_vring_addr *addr, 1151 Error **errp) 1152 { 1153 ERRP_GUARD(); 1154 DMAMap device_region, driver_region; 1155 struct vhost_vring_addr svq_addr; 1156 struct vhost_vdpa *v = dev->opaque; 1157 size_t device_size = vhost_svq_device_area_size(svq); 1158 size_t driver_size = vhost_svq_driver_area_size(svq); 1159 size_t avail_offset; 1160 bool ok; 1161 1162 vhost_svq_get_vring_addr(svq, &svq_addr); 1163 1164 driver_region = (DMAMap) { 1165 .translated_addr = svq_addr.desc_user_addr, 1166 .size = driver_size - 1, 1167 .perm = IOMMU_RO, 1168 }; 1169 ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp); 1170 if (unlikely(!ok)) { 1171 error_prepend(errp, "Cannot create vq driver region: "); 1172 return false; 1173 } 1174 addr->desc_user_addr = driver_region.iova; 1175 avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr; 1176 addr->avail_user_addr = driver_region.iova + avail_offset; 1177 1178 device_region = (DMAMap) { 1179 .translated_addr = svq_addr.used_user_addr, 1180 .size = device_size - 1, 1181 .perm = IOMMU_RW, 1182 }; 1183 ok = vhost_vdpa_svq_map_ring(v, &device_region, errp); 1184 if (unlikely(!ok)) { 1185 error_prepend(errp, "Cannot create vq device region: "); 1186 vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr); 1187 } 1188 addr->used_user_addr = device_region.iova; 1189 1190 return ok; 1191 } 1192 1193 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev, 1194 VhostShadowVirtqueue *svq, unsigned idx, 1195 Error **errp) 1196 { 1197 uint16_t vq_index = dev->vq_index + idx; 1198 struct vhost_vring_state s = { 1199 .index = vq_index, 1200 }; 1201 int r; 1202 1203 r = vhost_vdpa_set_dev_vring_base(dev, &s); 1204 if (unlikely(r)) { 1205 error_setg_errno(errp, -r, "Cannot set vring base"); 1206 return false; 1207 } 1208 1209 r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp); 1210 return r == 0; 1211 } 1212 1213 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev) 1214 { 1215 struct vhost_vdpa *v = dev->opaque; 1216 Error *err = NULL; 1217 unsigned i; 1218 1219 if (!v->shadow_vqs_enabled) { 1220 return true; 1221 } 1222 1223 for (i = 0; i < v->shadow_vqs->len; ++i) { 1224 VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i); 1225 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1226 struct vhost_vring_addr addr = { 1227 .index = dev->vq_index + i, 1228 }; 1229 int r; 1230 bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err); 1231 if (unlikely(!ok)) { 1232 goto err; 1233 } 1234 1235 vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree); 1236 ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err); 1237 if (unlikely(!ok)) { 1238 goto err_map; 1239 } 1240 1241 /* Override vring GPA set by vhost subsystem */ 1242 r = vhost_vdpa_set_vring_dev_addr(dev, &addr); 1243 if (unlikely(r != 0)) { 1244 error_setg_errno(&err, -r, "Cannot set device address"); 1245 goto err_set_addr; 1246 } 1247 } 1248 1249 return true; 1250 1251 err_set_addr: 1252 vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i)); 1253 1254 err_map: 1255 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i)); 1256 1257 err: 1258 error_reportf_err(err, "Cannot setup SVQ %u: ", i); 1259 for (unsigned j = 0; j < i; ++j) { 1260 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j); 1261 vhost_vdpa_svq_unmap_rings(dev, svq); 1262 vhost_svq_stop(svq); 1263 } 1264 1265 return false; 1266 } 1267 1268 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) 1269 { 1270 struct vhost_vdpa *v = dev->opaque; 1271 1272 if (!v->shadow_vqs_enabled) { 1273 return; 1274 } 1275 1276 for (unsigned i = 0; i < v->shadow_vqs->len; ++i) { 1277 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1278 1279 vhost_svq_stop(svq); 1280 vhost_vdpa_svq_unmap_rings(dev, svq); 1281 1282 event_notifier_cleanup(&svq->hdev_kick); 1283 event_notifier_cleanup(&svq->hdev_call); 1284 } 1285 } 1286 1287 static void vhost_vdpa_suspend(struct vhost_dev *dev) 1288 { 1289 struct vhost_vdpa *v = dev->opaque; 1290 int r; 1291 1292 if (!vhost_vdpa_first_dev(dev)) { 1293 return; 1294 } 1295 1296 if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { 1297 trace_vhost_vdpa_suspend(dev); 1298 r = ioctl(v->shared->device_fd, VHOST_VDPA_SUSPEND); 1299 if (unlikely(r)) { 1300 error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); 1301 } else { 1302 v->suspended = true; 1303 return; 1304 } 1305 } 1306 1307 vhost_vdpa_reset_device(dev); 1308 } 1309 1310 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 1311 { 1312 struct vhost_vdpa *v = dev->opaque; 1313 bool ok; 1314 trace_vhost_vdpa_dev_start(dev, started); 1315 1316 if (started) { 1317 vhost_vdpa_host_notifiers_init(dev); 1318 ok = vhost_vdpa_svqs_start(dev); 1319 if (unlikely(!ok)) { 1320 return -1; 1321 } 1322 } else { 1323 vhost_vdpa_suspend(dev); 1324 vhost_vdpa_svqs_stop(dev); 1325 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 1326 } 1327 1328 if (!vhost_vdpa_last_dev(dev)) { 1329 return 0; 1330 } 1331 1332 if (started) { 1333 if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) { 1334 error_report("SVQ can not work while IOMMU enable, please disable" 1335 "IOMMU and try again"); 1336 return -1; 1337 } 1338 memory_listener_register(&v->shared->listener, dev->vdev->dma_as); 1339 1340 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 1341 } 1342 1343 return 0; 1344 } 1345 1346 static void vhost_vdpa_reset_status(struct vhost_dev *dev) 1347 { 1348 struct vhost_vdpa *v = dev->opaque; 1349 1350 if (!vhost_vdpa_last_dev(dev)) { 1351 return; 1352 } 1353 1354 vhost_vdpa_reset_device(dev); 1355 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 1356 VIRTIO_CONFIG_S_DRIVER); 1357 memory_listener_unregister(&v->shared->listener); 1358 } 1359 1360 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 1361 struct vhost_log *log) 1362 { 1363 struct vhost_vdpa *v = dev->opaque; 1364 if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) { 1365 return 0; 1366 } 1367 1368 trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd, 1369 log->log); 1370 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 1371 } 1372 1373 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 1374 struct vhost_vring_addr *addr) 1375 { 1376 struct vhost_vdpa *v = dev->opaque; 1377 1378 if (v->shadow_vqs_enabled) { 1379 /* 1380 * Device vring addr was set at device start. SVQ base is handled by 1381 * VirtQueue code. 1382 */ 1383 return 0; 1384 } 1385 1386 return vhost_vdpa_set_vring_dev_addr(dev, addr); 1387 } 1388 1389 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 1390 struct vhost_vring_state *ring) 1391 { 1392 trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num); 1393 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 1394 } 1395 1396 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 1397 struct vhost_vring_state *ring) 1398 { 1399 struct vhost_vdpa *v = dev->opaque; 1400 1401 if (v->shadow_vqs_enabled) { 1402 /* 1403 * Device vring base was set at device start. SVQ base is handled by 1404 * VirtQueue code. 1405 */ 1406 return 0; 1407 } 1408 1409 return vhost_vdpa_set_dev_vring_base(dev, ring); 1410 } 1411 1412 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 1413 struct vhost_vring_state *ring) 1414 { 1415 struct vhost_vdpa *v = dev->opaque; 1416 int ret; 1417 1418 if (v->shadow_vqs_enabled) { 1419 ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index); 1420 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num, true); 1421 return 0; 1422 } 1423 1424 if (!v->suspended) { 1425 /* 1426 * Cannot trust in value returned by device, let vhost recover used 1427 * idx from guest. 1428 */ 1429 return -1; 1430 } 1431 1432 ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 1433 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num, false); 1434 return ret; 1435 } 1436 1437 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 1438 struct vhost_vring_file *file) 1439 { 1440 struct vhost_vdpa *v = dev->opaque; 1441 int vdpa_idx = file->index - dev->vq_index; 1442 1443 if (v->shadow_vqs_enabled) { 1444 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1445 vhost_svq_set_svq_kick_fd(svq, file->fd); 1446 return 0; 1447 } else { 1448 return vhost_vdpa_set_vring_dev_kick(dev, file); 1449 } 1450 } 1451 1452 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 1453 struct vhost_vring_file *file) 1454 { 1455 struct vhost_vdpa *v = dev->opaque; 1456 int vdpa_idx = file->index - dev->vq_index; 1457 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1458 1459 /* Remember last call fd because we can switch to SVQ anytime. */ 1460 vhost_svq_set_svq_call_fd(svq, file->fd); 1461 /* 1462 * When SVQ is transitioning to off, shadow_vqs_enabled has 1463 * not been set back to false yet, but the underlying call fd 1464 * will have to switch back to the guest notifier to signal the 1465 * passthrough virtqueues. In other situations, SVQ's own call 1466 * fd shall be used to signal the device model. 1467 */ 1468 if (v->shadow_vqs_enabled && 1469 v->shared->svq_switching != SVQ_TSTATE_DISABLING) { 1470 return 0; 1471 } 1472 1473 return vhost_vdpa_set_vring_dev_call(dev, file); 1474 } 1475 1476 static int vhost_vdpa_get_features(struct vhost_dev *dev, 1477 uint64_t *features) 1478 { 1479 int ret = vhost_vdpa_get_dev_features(dev, features); 1480 1481 if (ret == 0) { 1482 /* Add SVQ logging capabilities */ 1483 *features |= BIT_ULL(VHOST_F_LOG_ALL); 1484 } 1485 1486 return ret; 1487 } 1488 1489 static int vhost_vdpa_set_owner(struct vhost_dev *dev) 1490 { 1491 if (!vhost_vdpa_first_dev(dev)) { 1492 return 0; 1493 } 1494 1495 trace_vhost_vdpa_set_owner(dev); 1496 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 1497 } 1498 1499 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 1500 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 1501 { 1502 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 1503 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 1504 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 1505 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 1506 trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, 1507 addr->avail_user_addr, addr->used_user_addr); 1508 return 0; 1509 } 1510 1511 static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 1512 { 1513 return true; 1514 } 1515 1516 const VhostOps vdpa_ops = { 1517 .backend_type = VHOST_BACKEND_TYPE_VDPA, 1518 .vhost_backend_init = vhost_vdpa_init, 1519 .vhost_backend_cleanup = vhost_vdpa_cleanup, 1520 .vhost_set_log_base = vhost_vdpa_set_log_base, 1521 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 1522 .vhost_set_vring_num = vhost_vdpa_set_vring_num, 1523 .vhost_set_vring_base = vhost_vdpa_set_vring_base, 1524 .vhost_get_vring_base = vhost_vdpa_get_vring_base, 1525 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 1526 .vhost_set_vring_call = vhost_vdpa_set_vring_call, 1527 .vhost_get_features = vhost_vdpa_get_features, 1528 .vhost_set_backend_cap = vhost_vdpa_set_backend_cap, 1529 .vhost_set_owner = vhost_vdpa_set_owner, 1530 .vhost_set_vring_endian = NULL, 1531 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 1532 .vhost_set_mem_table = vhost_vdpa_set_mem_table, 1533 .vhost_set_features = vhost_vdpa_set_features, 1534 .vhost_reset_device = vhost_vdpa_reset_device, 1535 .vhost_get_vq_index = vhost_vdpa_get_vq_index, 1536 .vhost_get_config = vhost_vdpa_get_config, 1537 .vhost_set_config = vhost_vdpa_set_config, 1538 .vhost_requires_shm_log = NULL, 1539 .vhost_migration_done = NULL, 1540 .vhost_net_set_mtu = NULL, 1541 .vhost_set_iotlb_callback = NULL, 1542 .vhost_send_device_iotlb_msg = NULL, 1543 .vhost_dev_start = vhost_vdpa_dev_start, 1544 .vhost_get_device_id = vhost_vdpa_get_device_id, 1545 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 1546 .vhost_force_iommu = vhost_vdpa_force_iommu, 1547 .vhost_set_config_call = vhost_vdpa_set_config_call, 1548 .vhost_reset_status = vhost_vdpa_reset_status, 1549 }; 1550