1108a6481SCindy Lu /* 2108a6481SCindy Lu * vhost-vdpa 3108a6481SCindy Lu * 4108a6481SCindy Lu * Copyright(c) 2017-2018 Intel Corporation. 5108a6481SCindy Lu * Copyright(c) 2020 Red Hat, Inc. 6108a6481SCindy Lu * 7108a6481SCindy Lu * This work is licensed under the terms of the GNU GPL, version 2 or later. 8108a6481SCindy Lu * See the COPYING file in the top-level directory. 9108a6481SCindy Lu * 10108a6481SCindy Lu */ 11108a6481SCindy Lu 12108a6481SCindy Lu #include "qemu/osdep.h" 13108a6481SCindy Lu #include <linux/vhost.h> 14108a6481SCindy Lu #include <linux/vfio.h> 15108a6481SCindy Lu #include <sys/eventfd.h> 16108a6481SCindy Lu #include <sys/ioctl.h> 1733f21860SPhilippe Mathieu-Daudé #include "exec/target_page.h" 18108a6481SCindy Lu #include "hw/virtio/vhost.h" 19108a6481SCindy Lu #include "hw/virtio/vhost-backend.h" 20108a6481SCindy Lu #include "hw/virtio/virtio-net.h" 21dff4426fSEugenio Pérez #include "hw/virtio/vhost-shadow-virtqueue.h" 22108a6481SCindy Lu #include "hw/virtio/vhost-vdpa.h" 23df77d45aSXie Yongji #include "exec/address-spaces.h" 24c156d5bfSEugenio Pérez #include "migration/blocker.h" 25415b7327SMarc-André Lureau #include "qemu/cutils.h" 26108a6481SCindy Lu #include "qemu/main-loop.h" 27778e67deSLaurent Vivier #include "trace.h" 28dff4426fSEugenio Pérez #include "qapi/error.h" 29108a6481SCindy Lu 30032e4d68SEugenio Pérez /* 31032e4d68SEugenio Pérez * Return one past the end of the end of section. Be careful with uint64_t 32032e4d68SEugenio Pérez * conversions! 33032e4d68SEugenio Pérez */ 348b1a8884SPhilippe Mathieu-Daudé static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section, 358b1a8884SPhilippe Mathieu-Daudé int page_mask) 36032e4d68SEugenio Pérez { 37032e4d68SEugenio Pérez Int128 llend = int128_make64(section->offset_within_address_space); 38032e4d68SEugenio Pérez llend = int128_add(llend, section->size); 398b1a8884SPhilippe Mathieu-Daudé llend = int128_and(llend, int128_exts64(page_mask)); 40032e4d68SEugenio Pérez 41032e4d68SEugenio Pérez return llend; 42032e4d68SEugenio Pérez } 43032e4d68SEugenio Pérez 44013108b6SEugenio Pérez static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section, 45013108b6SEugenio Pérez uint64_t iova_min, 46961d60e9SPhilippe Mathieu-Daudé uint64_t iova_max, 47961d60e9SPhilippe Mathieu-Daudé int page_mask) 48108a6481SCindy Lu { 49013108b6SEugenio Pérez Int128 llend; 50013108b6SEugenio Pérez 51013108b6SEugenio Pérez if ((!memory_region_is_ram(section->mr) && 52108a6481SCindy Lu !memory_region_is_iommu(section->mr)) || 53c64038c9SEugenio Pérez memory_region_is_protected(section->mr) || 54d60c75d2SJason Wang /* vhost-vDPA doesn't allow MMIO to be mapped */ 55013108b6SEugenio Pérez memory_region_is_ram_device(section->mr)) { 56013108b6SEugenio Pérez return true; 57013108b6SEugenio Pérez } 58013108b6SEugenio Pérez 59013108b6SEugenio Pérez if (section->offset_within_address_space < iova_min) { 60013108b6SEugenio Pérez error_report("RAM section out of device range (min=0x%" PRIx64 61013108b6SEugenio Pérez ", addr=0x%" HWADDR_PRIx ")", 62013108b6SEugenio Pérez iova_min, section->offset_within_address_space); 63013108b6SEugenio Pérez return true; 64013108b6SEugenio Pérez } 65bc7b0cacSCindy Lu /* 66bc7b0cacSCindy Lu * While using vIOMMU, sometimes the section will be larger than iova_max, 67bc7b0cacSCindy Lu * but the memory that actually maps is smaller, so move the check to 68bc7b0cacSCindy Lu * function vhost_vdpa_iommu_map_notify(). That function will use the actual 69bc7b0cacSCindy Lu * size that maps to the kernel 70bc7b0cacSCindy Lu */ 71013108b6SEugenio Pérez 72bc7b0cacSCindy Lu if (!memory_region_is_iommu(section->mr)) { 738b1a8884SPhilippe Mathieu-Daudé llend = vhost_vdpa_section_end(section, page_mask); 74013108b6SEugenio Pérez if (int128_gt(llend, int128_make64(iova_max))) { 75013108b6SEugenio Pérez error_report("RAM section out of device range (max=0x%" PRIx64 76013108b6SEugenio Pérez ", end addr=0x%" PRIx64 ")", 77013108b6SEugenio Pérez iova_max, int128_get64(llend)); 78013108b6SEugenio Pérez return true; 79013108b6SEugenio Pérez } 80bc7b0cacSCindy Lu } 81013108b6SEugenio Pérez 82013108b6SEugenio Pérez return false; 83108a6481SCindy Lu } 84108a6481SCindy Lu 85cd831ed5SEugenio Pérez /* 86cd831ed5SEugenio Pérez * The caller must set asid = 0 if the device does not support asid. 87cd831ed5SEugenio Pérez * This is not an ABI break since it is set to 0 by the initializer anyway. 88cd831ed5SEugenio Pérez */ 89*6f03d9efSEugenio Pérez int vhost_vdpa_dma_map(VhostVDPAShared *s, uint32_t asid, hwaddr iova, 90cd831ed5SEugenio Pérez hwaddr size, void *vaddr, bool readonly) 91108a6481SCindy Lu { 92386494f2SCindy Lu struct vhost_msg_v2 msg = {}; 93*6f03d9efSEugenio Pérez int fd = s->device_fd; 94108a6481SCindy Lu int ret = 0; 95108a6481SCindy Lu 9674e76c7dSEugenio Pérez msg.type = VHOST_IOTLB_MSG_V2; 97cd831ed5SEugenio Pérez msg.asid = asid; 98108a6481SCindy Lu msg.iotlb.iova = iova; 99108a6481SCindy Lu msg.iotlb.size = size; 100108a6481SCindy Lu msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 101108a6481SCindy Lu msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 102108a6481SCindy Lu msg.iotlb.type = VHOST_IOTLB_UPDATE; 103108a6481SCindy Lu 104*6f03d9efSEugenio Pérez trace_vhost_vdpa_dma_map(s, fd, msg.type, msg.asid, msg.iotlb.iova, 105cd831ed5SEugenio Pérez msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm, 106cd831ed5SEugenio Pérez msg.iotlb.type); 107778e67deSLaurent Vivier 108108a6481SCindy Lu if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 109108a6481SCindy Lu error_report("failed to write, fd=%d, errno=%d (%s)", 110108a6481SCindy Lu fd, errno, strerror(errno)); 111108a6481SCindy Lu return -EIO ; 112108a6481SCindy Lu } 113108a6481SCindy Lu 114108a6481SCindy Lu return ret; 115108a6481SCindy Lu } 116108a6481SCindy Lu 117cd831ed5SEugenio Pérez /* 118cd831ed5SEugenio Pérez * The caller must set asid = 0 if the device does not support asid. 119cd831ed5SEugenio Pérez * This is not an ABI break since it is set to 0 by the initializer anyway. 120cd831ed5SEugenio Pérez */ 121*6f03d9efSEugenio Pérez int vhost_vdpa_dma_unmap(VhostVDPAShared *s, uint32_t asid, hwaddr iova, 122cd831ed5SEugenio Pérez hwaddr size) 123108a6481SCindy Lu { 124386494f2SCindy Lu struct vhost_msg_v2 msg = {}; 125*6f03d9efSEugenio Pérez int fd = s->device_fd; 126108a6481SCindy Lu int ret = 0; 127108a6481SCindy Lu 12874e76c7dSEugenio Pérez msg.type = VHOST_IOTLB_MSG_V2; 129cd831ed5SEugenio Pérez msg.asid = asid; 130108a6481SCindy Lu msg.iotlb.iova = iova; 131108a6481SCindy Lu msg.iotlb.size = size; 132108a6481SCindy Lu msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 133108a6481SCindy Lu 134*6f03d9efSEugenio Pérez trace_vhost_vdpa_dma_unmap(s, fd, msg.type, msg.asid, msg.iotlb.iova, 135*6f03d9efSEugenio Pérez msg.iotlb.size, msg.iotlb.type); 136778e67deSLaurent Vivier 137108a6481SCindy Lu if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 138108a6481SCindy Lu error_report("failed to write, fd=%d, errno=%d (%s)", 139108a6481SCindy Lu fd, errno, strerror(errno)); 140108a6481SCindy Lu return -EIO ; 141108a6481SCindy Lu } 142108a6481SCindy Lu 143108a6481SCindy Lu return ret; 144108a6481SCindy Lu } 145108a6481SCindy Lu 146*6f03d9efSEugenio Pérez static void vhost_vdpa_listener_begin_batch(VhostVDPAShared *s) 147a5bd0580SJason Wang { 148*6f03d9efSEugenio Pérez int fd = s->device_fd; 149e6db5df7SEugenio Pérez struct vhost_msg_v2 msg = { 15074e76c7dSEugenio Pérez .type = VHOST_IOTLB_MSG_V2, 151e6db5df7SEugenio Pérez .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, 152e6db5df7SEugenio Pérez }; 153a5bd0580SJason Wang 154*6f03d9efSEugenio Pérez trace_vhost_vdpa_listener_begin_batch(s, fd, msg.type, msg.iotlb.type); 155a5bd0580SJason Wang if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 156a5bd0580SJason Wang error_report("failed to write, fd=%d, errno=%d (%s)", 157a5bd0580SJason Wang fd, errno, strerror(errno)); 158a5bd0580SJason Wang } 159a5bd0580SJason Wang } 160a5bd0580SJason Wang 161*6f03d9efSEugenio Pérez static void vhost_vdpa_iotlb_batch_begin_once(VhostVDPAShared *s) 162e6db5df7SEugenio Pérez { 163*6f03d9efSEugenio Pérez if (s->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && 164*6f03d9efSEugenio Pérez !s->iotlb_batch_begin_sent) { 165*6f03d9efSEugenio Pérez vhost_vdpa_listener_begin_batch(s); 166e6db5df7SEugenio Pérez } 167e6db5df7SEugenio Pérez 168*6f03d9efSEugenio Pérez s->iotlb_batch_begin_sent = true; 169e6db5df7SEugenio Pérez } 170e6db5df7SEugenio Pérez 171a5bd0580SJason Wang static void vhost_vdpa_listener_commit(MemoryListener *listener) 172a5bd0580SJason Wang { 173a5bd0580SJason Wang struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 1748acb3218SPhilippe Mathieu-Daudé struct vhost_msg_v2 msg = {}; 175f12b2498SEugenio Pérez int fd = v->shared->device_fd; 176a5bd0580SJason Wang 1774da38d1aSEugenio Pérez if (!(v->shared->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { 178a5bd0580SJason Wang return; 179a5bd0580SJason Wang } 180a5bd0580SJason Wang 1817627f0a2SEugenio Pérez if (!v->shared->iotlb_batch_begin_sent) { 182e6db5df7SEugenio Pérez return; 183e6db5df7SEugenio Pérez } 184e6db5df7SEugenio Pérez 18574e76c7dSEugenio Pérez msg.type = VHOST_IOTLB_MSG_V2; 186a5bd0580SJason Wang msg.iotlb.type = VHOST_IOTLB_BATCH_END; 187a5bd0580SJason Wang 188e36b9992SEugenio Pérez trace_vhost_vdpa_listener_commit(v->shared, fd, msg.type, msg.iotlb.type); 189a5bd0580SJason Wang if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 190a5bd0580SJason Wang error_report("failed to write, fd=%d, errno=%d (%s)", 191a5bd0580SJason Wang fd, errno, strerror(errno)); 192a5bd0580SJason Wang } 193e6db5df7SEugenio Pérez 1947627f0a2SEugenio Pérez v->shared->iotlb_batch_begin_sent = false; 195a5bd0580SJason Wang } 196a5bd0580SJason Wang 197bc7b0cacSCindy Lu static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 198bc7b0cacSCindy Lu { 199bc7b0cacSCindy Lu struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n); 200bc7b0cacSCindy Lu 201bc7b0cacSCindy Lu hwaddr iova = iotlb->iova + iommu->iommu_offset; 202bc7b0cacSCindy Lu struct vhost_vdpa *v = iommu->dev; 203bc7b0cacSCindy Lu void *vaddr; 204bc7b0cacSCindy Lu int ret; 205bc7b0cacSCindy Lu Int128 llend; 206bc7b0cacSCindy Lu 207bc7b0cacSCindy Lu if (iotlb->target_as != &address_space_memory) { 208bc7b0cacSCindy Lu error_report("Wrong target AS \"%s\", only system memory is allowed", 209bc7b0cacSCindy Lu iotlb->target_as->name ? iotlb->target_as->name : "none"); 210bc7b0cacSCindy Lu return; 211bc7b0cacSCindy Lu } 212bc7b0cacSCindy Lu RCU_READ_LOCK_GUARD(); 213bc7b0cacSCindy Lu /* check if RAM section out of device range */ 214bc7b0cacSCindy Lu llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova)); 215ae25ff41SEugenio Pérez if (int128_gt(llend, int128_make64(v->shared->iova_range.last))) { 216bc7b0cacSCindy Lu error_report("RAM section out of device range (max=0x%" PRIx64 217bc7b0cacSCindy Lu ", end addr=0x%" PRIx64 ")", 218ae25ff41SEugenio Pérez v->shared->iova_range.last, int128_get64(llend)); 219bc7b0cacSCindy Lu return; 220bc7b0cacSCindy Lu } 221bc7b0cacSCindy Lu 222bc7b0cacSCindy Lu if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 223bc7b0cacSCindy Lu bool read_only; 224bc7b0cacSCindy Lu 225bc7b0cacSCindy Lu if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) { 226bc7b0cacSCindy Lu return; 227bc7b0cacSCindy Lu } 228*6f03d9efSEugenio Pérez ret = vhost_vdpa_dma_map(v->shared, VHOST_VDPA_GUEST_PA_ASID, iova, 229bc7b0cacSCindy Lu iotlb->addr_mask + 1, vaddr, read_only); 230bc7b0cacSCindy Lu if (ret) { 231bc7b0cacSCindy Lu error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", " 232bc7b0cacSCindy Lu "0x%" HWADDR_PRIx ", %p) = %d (%m)", 233bc7b0cacSCindy Lu v, iova, iotlb->addr_mask + 1, vaddr, ret); 234bc7b0cacSCindy Lu } 235bc7b0cacSCindy Lu } else { 236*6f03d9efSEugenio Pérez ret = vhost_vdpa_dma_unmap(v->shared, VHOST_VDPA_GUEST_PA_ASID, iova, 237bc7b0cacSCindy Lu iotlb->addr_mask + 1); 238bc7b0cacSCindy Lu if (ret) { 239bc7b0cacSCindy Lu error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 240bc7b0cacSCindy Lu "0x%" HWADDR_PRIx ") = %d (%m)", 241bc7b0cacSCindy Lu v, iova, iotlb->addr_mask + 1, ret); 242bc7b0cacSCindy Lu } 243bc7b0cacSCindy Lu } 244bc7b0cacSCindy Lu } 245bc7b0cacSCindy Lu 246bc7b0cacSCindy Lu static void vhost_vdpa_iommu_region_add(MemoryListener *listener, 247bc7b0cacSCindy Lu MemoryRegionSection *section) 248bc7b0cacSCindy Lu { 249bc7b0cacSCindy Lu struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 250bc7b0cacSCindy Lu 251bc7b0cacSCindy Lu struct vdpa_iommu *iommu; 252bc7b0cacSCindy Lu Int128 end; 253bc7b0cacSCindy Lu int iommu_idx; 254bc7b0cacSCindy Lu IOMMUMemoryRegion *iommu_mr; 255bc7b0cacSCindy Lu int ret; 256bc7b0cacSCindy Lu 257bc7b0cacSCindy Lu iommu_mr = IOMMU_MEMORY_REGION(section->mr); 258bc7b0cacSCindy Lu 259bc7b0cacSCindy Lu iommu = g_malloc0(sizeof(*iommu)); 260bc7b0cacSCindy Lu end = int128_add(int128_make64(section->offset_within_region), 261bc7b0cacSCindy Lu section->size); 262bc7b0cacSCindy Lu end = int128_sub(end, int128_one()); 263bc7b0cacSCindy Lu iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 264bc7b0cacSCindy Lu MEMTXATTRS_UNSPECIFIED); 265bc7b0cacSCindy Lu iommu->iommu_mr = iommu_mr; 266bc7b0cacSCindy Lu iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify, 267bc7b0cacSCindy Lu IOMMU_NOTIFIER_IOTLB_EVENTS, 268bc7b0cacSCindy Lu section->offset_within_region, 269bc7b0cacSCindy Lu int128_get64(end), 270bc7b0cacSCindy Lu iommu_idx); 271bc7b0cacSCindy Lu iommu->iommu_offset = section->offset_within_address_space - 272bc7b0cacSCindy Lu section->offset_within_region; 273bc7b0cacSCindy Lu iommu->dev = v; 274bc7b0cacSCindy Lu 275bc7b0cacSCindy Lu ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 276bc7b0cacSCindy Lu if (ret) { 277bc7b0cacSCindy Lu g_free(iommu); 278bc7b0cacSCindy Lu return; 279bc7b0cacSCindy Lu } 280bc7b0cacSCindy Lu 2813c6d12a3SEugenio Pérez QLIST_INSERT_HEAD(&v->shared->iommu_list, iommu, iommu_next); 282bc7b0cacSCindy Lu memory_region_iommu_replay(iommu->iommu_mr, &iommu->n); 283bc7b0cacSCindy Lu 284bc7b0cacSCindy Lu return; 285bc7b0cacSCindy Lu } 286bc7b0cacSCindy Lu 287bc7b0cacSCindy Lu static void vhost_vdpa_iommu_region_del(MemoryListener *listener, 288bc7b0cacSCindy Lu MemoryRegionSection *section) 289bc7b0cacSCindy Lu { 290bc7b0cacSCindy Lu struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 291bc7b0cacSCindy Lu 292bc7b0cacSCindy Lu struct vdpa_iommu *iommu; 293bc7b0cacSCindy Lu 2943c6d12a3SEugenio Pérez QLIST_FOREACH(iommu, &v->shared->iommu_list, iommu_next) 295bc7b0cacSCindy Lu { 296bc7b0cacSCindy Lu if (MEMORY_REGION(iommu->iommu_mr) == section->mr && 297bc7b0cacSCindy Lu iommu->n.start == section->offset_within_region) { 298bc7b0cacSCindy Lu memory_region_unregister_iommu_notifier(section->mr, &iommu->n); 299bc7b0cacSCindy Lu QLIST_REMOVE(iommu, iommu_next); 300bc7b0cacSCindy Lu g_free(iommu); 301bc7b0cacSCindy Lu break; 302bc7b0cacSCindy Lu } 303bc7b0cacSCindy Lu } 304bc7b0cacSCindy Lu } 305bc7b0cacSCindy Lu 306108a6481SCindy Lu static void vhost_vdpa_listener_region_add(MemoryListener *listener, 307108a6481SCindy Lu MemoryRegionSection *section) 308108a6481SCindy Lu { 3097dab70beSEugenio Pérez DMAMap mem_region = {}; 310108a6481SCindy Lu struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 311108a6481SCindy Lu hwaddr iova; 312108a6481SCindy Lu Int128 llend, llsize; 313108a6481SCindy Lu void *vaddr; 314108a6481SCindy Lu int ret; 31533f21860SPhilippe Mathieu-Daudé int page_size = qemu_target_page_size(); 31633f21860SPhilippe Mathieu-Daudé int page_mask = -page_size; 317108a6481SCindy Lu 318ae25ff41SEugenio Pérez if (vhost_vdpa_listener_skipped_section(section, 319ae25ff41SEugenio Pérez v->shared->iova_range.first, 320ae25ff41SEugenio Pérez v->shared->iova_range.last, 321ae25ff41SEugenio Pérez page_mask)) { 322108a6481SCindy Lu return; 323108a6481SCindy Lu } 324bc7b0cacSCindy Lu if (memory_region_is_iommu(section->mr)) { 325bc7b0cacSCindy Lu vhost_vdpa_iommu_region_add(listener, section); 326bc7b0cacSCindy Lu return; 327bc7b0cacSCindy Lu } 328108a6481SCindy Lu 32933f21860SPhilippe Mathieu-Daudé if (unlikely((section->offset_within_address_space & ~page_mask) != 33033f21860SPhilippe Mathieu-Daudé (section->offset_within_region & ~page_mask))) { 331e36b9992SEugenio Pérez trace_vhost_vdpa_listener_region_add_unaligned(v->shared, 332e36b9992SEugenio Pérez section->mr->name, 33333f21860SPhilippe Mathieu-Daudé section->offset_within_address_space & ~page_mask, 33433f21860SPhilippe Mathieu-Daudé section->offset_within_region & ~page_mask); 335108a6481SCindy Lu return; 336108a6481SCindy Lu } 337108a6481SCindy Lu 33833f21860SPhilippe Mathieu-Daudé iova = ROUND_UP(section->offset_within_address_space, page_size); 33933f21860SPhilippe Mathieu-Daudé llend = vhost_vdpa_section_end(section, page_mask); 340108a6481SCindy Lu if (int128_ge(int128_make64(iova), llend)) { 341108a6481SCindy Lu return; 342108a6481SCindy Lu } 343108a6481SCindy Lu 344108a6481SCindy Lu memory_region_ref(section->mr); 345108a6481SCindy Lu 346108a6481SCindy Lu /* Here we assume that memory_region_is_ram(section->mr)==true */ 347108a6481SCindy Lu 348108a6481SCindy Lu vaddr = memory_region_get_ram_ptr(section->mr) + 349108a6481SCindy Lu section->offset_within_region + 350108a6481SCindy Lu (iova - section->offset_within_address_space); 351108a6481SCindy Lu 352e36b9992SEugenio Pérez trace_vhost_vdpa_listener_region_add(v->shared, iova, int128_get64(llend), 353778e67deSLaurent Vivier vaddr, section->readonly); 354778e67deSLaurent Vivier 355108a6481SCindy Lu llsize = int128_sub(llend, int128_make64(iova)); 356a6e823d4SEugenio Pérez if (v->shared->shadow_data) { 3577dab70beSEugenio Pérez int r; 35834e3c94eSEugenio Pérez 3597dab70beSEugenio Pérez mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr, 3607dab70beSEugenio Pérez mem_region.size = int128_get64(llsize) - 1, 3617dab70beSEugenio Pérez mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly), 3627dab70beSEugenio Pérez 3635edb02e8SEugenio Pérez r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &mem_region); 36434e3c94eSEugenio Pérez if (unlikely(r != IOVA_OK)) { 36534e3c94eSEugenio Pérez error_report("Can't allocate a mapping (%d)", r); 36634e3c94eSEugenio Pérez goto fail; 36734e3c94eSEugenio Pérez } 36834e3c94eSEugenio Pérez 36934e3c94eSEugenio Pérez iova = mem_region.iova; 37034e3c94eSEugenio Pérez } 371108a6481SCindy Lu 372*6f03d9efSEugenio Pérez vhost_vdpa_iotlb_batch_begin_once(v->shared); 373*6f03d9efSEugenio Pérez ret = vhost_vdpa_dma_map(v->shared, VHOST_VDPA_GUEST_PA_ASID, iova, 374cd831ed5SEugenio Pérez int128_get64(llsize), vaddr, section->readonly); 375108a6481SCindy Lu if (ret) { 376108a6481SCindy Lu error_report("vhost vdpa map fail!"); 3777dab70beSEugenio Pérez goto fail_map; 378108a6481SCindy Lu } 379108a6481SCindy Lu 380108a6481SCindy Lu return; 381108a6481SCindy Lu 3827dab70beSEugenio Pérez fail_map: 383a6e823d4SEugenio Pérez if (v->shared->shadow_data) { 3845edb02e8SEugenio Pérez vhost_iova_tree_remove(v->shared->iova_tree, mem_region); 3857dab70beSEugenio Pérez } 3867dab70beSEugenio Pérez 387108a6481SCindy Lu fail: 388108a6481SCindy Lu /* 389108a6481SCindy Lu * On the initfn path, store the first error in the container so we 390108a6481SCindy Lu * can gracefully fail. Runtime, there's not much we can do other 391108a6481SCindy Lu * than throw a hardware error. 392108a6481SCindy Lu */ 393108a6481SCindy Lu error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 394108a6481SCindy Lu return; 395108a6481SCindy Lu 396108a6481SCindy Lu } 397108a6481SCindy Lu 398108a6481SCindy Lu static void vhost_vdpa_listener_region_del(MemoryListener *listener, 399108a6481SCindy Lu MemoryRegionSection *section) 400108a6481SCindy Lu { 401108a6481SCindy Lu struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 402108a6481SCindy Lu hwaddr iova; 403108a6481SCindy Lu Int128 llend, llsize; 404108a6481SCindy Lu int ret; 40533f21860SPhilippe Mathieu-Daudé int page_size = qemu_target_page_size(); 40633f21860SPhilippe Mathieu-Daudé int page_mask = -page_size; 407108a6481SCindy Lu 408ae25ff41SEugenio Pérez if (vhost_vdpa_listener_skipped_section(section, 409ae25ff41SEugenio Pérez v->shared->iova_range.first, 410ae25ff41SEugenio Pérez v->shared->iova_range.last, 411ae25ff41SEugenio Pérez page_mask)) { 412108a6481SCindy Lu return; 413108a6481SCindy Lu } 414bc7b0cacSCindy Lu if (memory_region_is_iommu(section->mr)) { 415bc7b0cacSCindy Lu vhost_vdpa_iommu_region_del(listener, section); 416bc7b0cacSCindy Lu } 417108a6481SCindy Lu 41833f21860SPhilippe Mathieu-Daudé if (unlikely((section->offset_within_address_space & ~page_mask) != 41933f21860SPhilippe Mathieu-Daudé (section->offset_within_region & ~page_mask))) { 420e36b9992SEugenio Pérez trace_vhost_vdpa_listener_region_del_unaligned(v->shared, 421e36b9992SEugenio Pérez section->mr->name, 42233f21860SPhilippe Mathieu-Daudé section->offset_within_address_space & ~page_mask, 42333f21860SPhilippe Mathieu-Daudé section->offset_within_region & ~page_mask); 424108a6481SCindy Lu return; 425108a6481SCindy Lu } 426108a6481SCindy Lu 42733f21860SPhilippe Mathieu-Daudé iova = ROUND_UP(section->offset_within_address_space, page_size); 42833f21860SPhilippe Mathieu-Daudé llend = vhost_vdpa_section_end(section, page_mask); 429108a6481SCindy Lu 430e36b9992SEugenio Pérez trace_vhost_vdpa_listener_region_del(v->shared, iova, 4313d1e4d34SCindy Lu int128_get64(int128_sub(llend, int128_one()))); 432778e67deSLaurent Vivier 433108a6481SCindy Lu if (int128_ge(int128_make64(iova), llend)) { 434108a6481SCindy Lu return; 435108a6481SCindy Lu } 436108a6481SCindy Lu 437108a6481SCindy Lu llsize = int128_sub(llend, int128_make64(iova)); 438108a6481SCindy Lu 439a6e823d4SEugenio Pérez if (v->shared->shadow_data) { 44034e3c94eSEugenio Pérez const DMAMap *result; 44134e3c94eSEugenio Pérez const void *vaddr = memory_region_get_ram_ptr(section->mr) + 44234e3c94eSEugenio Pérez section->offset_within_region + 44334e3c94eSEugenio Pérez (iova - section->offset_within_address_space); 44434e3c94eSEugenio Pérez DMAMap mem_region = { 44534e3c94eSEugenio Pérez .translated_addr = (hwaddr)(uintptr_t)vaddr, 44634e3c94eSEugenio Pérez .size = int128_get64(llsize) - 1, 44734e3c94eSEugenio Pérez }; 44834e3c94eSEugenio Pérez 4495edb02e8SEugenio Pérez result = vhost_iova_tree_find_iova(v->shared->iova_tree, &mem_region); 45010dab9f2SEugenio Pérez if (!result) { 45110dab9f2SEugenio Pérez /* The memory listener map wasn't mapped */ 45210dab9f2SEugenio Pérez return; 45310dab9f2SEugenio Pérez } 45434e3c94eSEugenio Pérez iova = result->iova; 4555edb02e8SEugenio Pérez vhost_iova_tree_remove(v->shared->iova_tree, *result); 45634e3c94eSEugenio Pérez } 457*6f03d9efSEugenio Pérez vhost_vdpa_iotlb_batch_begin_once(v->shared); 4582fbef6aaSCindy Lu /* 4592fbef6aaSCindy Lu * The unmap ioctl doesn't accept a full 64-bit. need to check it 4602fbef6aaSCindy Lu */ 4612fbef6aaSCindy Lu if (int128_eq(llsize, int128_2_64())) { 4622fbef6aaSCindy Lu llsize = int128_rshift(llsize, 1); 463*6f03d9efSEugenio Pérez ret = vhost_vdpa_dma_unmap(v->shared, VHOST_VDPA_GUEST_PA_ASID, iova, 464cd831ed5SEugenio Pérez int128_get64(llsize)); 4652fbef6aaSCindy Lu 466108a6481SCindy Lu if (ret) { 4672fbef6aaSCindy Lu error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 4682fbef6aaSCindy Lu "0x%" HWADDR_PRIx ") = %d (%m)", 4692fbef6aaSCindy Lu v, iova, int128_get64(llsize), ret); 4702fbef6aaSCindy Lu } 4712fbef6aaSCindy Lu iova += int128_get64(llsize); 4722fbef6aaSCindy Lu } 473*6f03d9efSEugenio Pérez ret = vhost_vdpa_dma_unmap(v->shared, VHOST_VDPA_GUEST_PA_ASID, iova, 4742fbef6aaSCindy Lu int128_get64(llsize)); 4752fbef6aaSCindy Lu 4762fbef6aaSCindy Lu if (ret) { 4772fbef6aaSCindy Lu error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 4782fbef6aaSCindy Lu "0x%" HWADDR_PRIx ") = %d (%m)", 4792fbef6aaSCindy Lu v, iova, int128_get64(llsize), ret); 480108a6481SCindy Lu } 481108a6481SCindy Lu 482108a6481SCindy Lu memory_region_unref(section->mr); 483108a6481SCindy Lu } 484108a6481SCindy Lu /* 485ef4ff56cSStefano Garzarella * IOTLB API is used by vhost-vdpa which requires incremental updating 486108a6481SCindy Lu * of the mapping. So we can not use generic vhost memory listener which 487108a6481SCindy Lu * depends on the addnop(). 488108a6481SCindy Lu */ 489108a6481SCindy Lu static const MemoryListener vhost_vdpa_memory_listener = { 490142518bdSPeter Xu .name = "vhost-vdpa", 491a5bd0580SJason Wang .commit = vhost_vdpa_listener_commit, 492108a6481SCindy Lu .region_add = vhost_vdpa_listener_region_add, 493108a6481SCindy Lu .region_del = vhost_vdpa_listener_region_del, 494108a6481SCindy Lu }; 495108a6481SCindy Lu 496108a6481SCindy Lu static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 497108a6481SCindy Lu void *arg) 498108a6481SCindy Lu { 499108a6481SCindy Lu struct vhost_vdpa *v = dev->opaque; 500f12b2498SEugenio Pérez int fd = v->shared->device_fd; 501f2a6e6c4SKevin Wolf int ret; 502108a6481SCindy Lu 503108a6481SCindy Lu assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 504108a6481SCindy Lu 505f2a6e6c4SKevin Wolf ret = ioctl(fd, request, arg); 506f2a6e6c4SKevin Wolf return ret < 0 ? -errno : ret; 507108a6481SCindy Lu } 508108a6481SCindy Lu 5093631151bSRoman Kagan static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 510108a6481SCindy Lu { 511108a6481SCindy Lu uint8_t s; 5123631151bSRoman Kagan int ret; 513108a6481SCindy Lu 514778e67deSLaurent Vivier trace_vhost_vdpa_add_status(dev, status); 5153631151bSRoman Kagan ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 5163631151bSRoman Kagan if (ret < 0) { 5173631151bSRoman Kagan return ret; 518108a6481SCindy Lu } 519bc865bfeSEugenio Pérez if ((s & status) == status) { 520bc865bfeSEugenio Pérez /* Don't set bits already set */ 521bc865bfeSEugenio Pérez return 0; 522bc865bfeSEugenio Pérez } 523108a6481SCindy Lu 524108a6481SCindy Lu s |= status; 525108a6481SCindy Lu 5263631151bSRoman Kagan ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 5273631151bSRoman Kagan if (ret < 0) { 5283631151bSRoman Kagan return ret; 5293631151bSRoman Kagan } 5303631151bSRoman Kagan 5313631151bSRoman Kagan ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 5323631151bSRoman Kagan if (ret < 0) { 5333631151bSRoman Kagan return ret; 5343631151bSRoman Kagan } 5353631151bSRoman Kagan 5363631151bSRoman Kagan if (!(s & status)) { 5373631151bSRoman Kagan return -EIO; 5383631151bSRoman Kagan } 5393631151bSRoman Kagan 5403631151bSRoman Kagan return 0; 541108a6481SCindy Lu } 542108a6481SCindy Lu 543c672f348SLongpeng int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range) 544c672f348SLongpeng { 545c672f348SLongpeng int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range); 546c672f348SLongpeng 547c672f348SLongpeng return ret < 0 ? -errno : 0; 548c672f348SLongpeng } 549c672f348SLongpeng 550d71b0609SSi-Wei Liu /* 551d71b0609SSi-Wei Liu * The use of this function is for requests that only need to be 552d71b0609SSi-Wei Liu * applied once. Typically such request occurs at the beginning 553d71b0609SSi-Wei Liu * of operation, and before setting up queues. It should not be 554d71b0609SSi-Wei Liu * used for request that performs operation until all queues are 555d71b0609SSi-Wei Liu * set, which would need to check dev->vq_index_end instead. 556d71b0609SSi-Wei Liu */ 557d71b0609SSi-Wei Liu static bool vhost_vdpa_first_dev(struct vhost_dev *dev) 5584d191cfdSJason Wang { 5594d191cfdSJason Wang struct vhost_vdpa *v = dev->opaque; 5604d191cfdSJason Wang 561d71b0609SSi-Wei Liu return v->index == 0; 5624d191cfdSJason Wang } 5634d191cfdSJason Wang 56412a195faSEugenio Pérez static int vhost_vdpa_get_dev_features(struct vhost_dev *dev, 56512a195faSEugenio Pérez uint64_t *features) 56612a195faSEugenio Pérez { 56712a195faSEugenio Pérez int ret; 56812a195faSEugenio Pérez 56912a195faSEugenio Pérez ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 57012a195faSEugenio Pérez trace_vhost_vdpa_get_features(dev, *features); 57112a195faSEugenio Pérez return ret; 57212a195faSEugenio Pérez } 57312a195faSEugenio Pérez 574258a0394SEugenio Pérez static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v) 575dff4426fSEugenio Pérez { 576dff4426fSEugenio Pérez g_autoptr(GPtrArray) shadow_vqs = NULL; 5774725a418SEugenio Pérez 578dff4426fSEugenio Pérez shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free); 579dff4426fSEugenio Pérez for (unsigned n = 0; n < hdev->nvqs; ++n) { 5803cfb4d06SEugenio Pérez VhostShadowVirtqueue *svq; 581dff4426fSEugenio Pérez 5825fde952bSEugenio Pérez svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque); 5833cfb4d06SEugenio Pérez g_ptr_array_add(shadow_vqs, svq); 584dff4426fSEugenio Pérez } 585dff4426fSEugenio Pérez 586dff4426fSEugenio Pérez v->shadow_vqs = g_steal_pointer(&shadow_vqs); 587dff4426fSEugenio Pérez } 588dff4426fSEugenio Pérez 58928770ff9SKevin Wolf static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) 590108a6481SCindy Lu { 591e36b9992SEugenio Pérez struct vhost_vdpa *v = opaque; 592108a6481SCindy Lu assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 593e36b9992SEugenio Pérez trace_vhost_vdpa_init(dev, v->shared, opaque); 594e1c1915bSDavid Hildenbrand int ret; 595e1c1915bSDavid Hildenbrand 596a5bd0580SJason Wang v->dev = dev; 597108a6481SCindy Lu dev->opaque = opaque ; 598108a6481SCindy Lu v->listener = vhost_vdpa_memory_listener; 599258a0394SEugenio Pérez vhost_vdpa_init_svq(dev, v); 600108a6481SCindy Lu 6019c363cf6SEugenio Pérez error_propagate(&dev->migration_blocker, v->migration_blocker); 602d71b0609SSi-Wei Liu if (!vhost_vdpa_first_dev(dev)) { 6034d191cfdSJason Wang return 0; 6044d191cfdSJason Wang } 6054d191cfdSJason Wang 606a230c471SEugenio Pérez /* 60757ac8318SEugenio Pérez * If dev->shadow_vqs_enabled at initialization that means the device has 60857ac8318SEugenio Pérez * been started with x-svq=on, so don't block migration 60957ac8318SEugenio Pérez */ 61057ac8318SEugenio Pérez if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { 61157ac8318SEugenio Pérez /* We don't have dev->features yet */ 61257ac8318SEugenio Pérez uint64_t features; 61357ac8318SEugenio Pérez ret = vhost_vdpa_get_dev_features(dev, &features); 61457ac8318SEugenio Pérez if (unlikely(ret)) { 61557ac8318SEugenio Pérez error_setg_errno(errp, -ret, "Could not get device features"); 61657ac8318SEugenio Pérez return ret; 61757ac8318SEugenio Pérez } 61857ac8318SEugenio Pérez vhost_svq_valid_features(features, &dev->migration_blocker); 61957ac8318SEugenio Pérez } 62057ac8318SEugenio Pérez 62157ac8318SEugenio Pérez /* 622a230c471SEugenio Pérez * Similar to VFIO, we end up pinning all guest memory and have to 623a230c471SEugenio Pérez * disable discarding of RAM. 624a230c471SEugenio Pérez */ 625a230c471SEugenio Pérez ret = ram_block_discard_disable(true); 626a230c471SEugenio Pérez if (ret) { 627a230c471SEugenio Pérez error_report("Cannot set discarding of RAM broken"); 628a230c471SEugenio Pérez return ret; 629a230c471SEugenio Pérez } 630a230c471SEugenio Pérez 631108a6481SCindy Lu vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 632108a6481SCindy Lu VIRTIO_CONFIG_S_DRIVER); 633108a6481SCindy Lu 634108a6481SCindy Lu return 0; 635108a6481SCindy Lu } 636108a6481SCindy Lu 637d0416d48SJason Wang static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, 638d0416d48SJason Wang int queue_index) 639d0416d48SJason Wang { 6408e3b0cbbSMarc-André Lureau size_t page_size = qemu_real_host_page_size(); 641d0416d48SJason Wang struct vhost_vdpa *v = dev->opaque; 642d0416d48SJason Wang VirtIODevice *vdev = dev->vdev; 643d0416d48SJason Wang VhostVDPAHostNotifier *n; 644d0416d48SJason Wang 645d0416d48SJason Wang n = &v->notifier[queue_index]; 646d0416d48SJason Wang 647d0416d48SJason Wang if (n->addr) { 648d0416d48SJason Wang virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); 649d0416d48SJason Wang object_unparent(OBJECT(&n->mr)); 650d0416d48SJason Wang munmap(n->addr, page_size); 651d0416d48SJason Wang n->addr = NULL; 652d0416d48SJason Wang } 653d0416d48SJason Wang } 654d0416d48SJason Wang 655d0416d48SJason Wang static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) 656d0416d48SJason Wang { 6578e3b0cbbSMarc-André Lureau size_t page_size = qemu_real_host_page_size(); 658d0416d48SJason Wang struct vhost_vdpa *v = dev->opaque; 659d0416d48SJason Wang VirtIODevice *vdev = dev->vdev; 660d0416d48SJason Wang VhostVDPAHostNotifier *n; 661f12b2498SEugenio Pérez int fd = v->shared->device_fd; 662d0416d48SJason Wang void *addr; 663d0416d48SJason Wang char *name; 664d0416d48SJason Wang 665d0416d48SJason Wang vhost_vdpa_host_notifier_uninit(dev, queue_index); 666d0416d48SJason Wang 667d0416d48SJason Wang n = &v->notifier[queue_index]; 668d0416d48SJason Wang 669d0416d48SJason Wang addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, 670d0416d48SJason Wang queue_index * page_size); 671d0416d48SJason Wang if (addr == MAP_FAILED) { 672d0416d48SJason Wang goto err; 673d0416d48SJason Wang } 674d0416d48SJason Wang 675d0416d48SJason Wang name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", 676d0416d48SJason Wang v, queue_index); 677d0416d48SJason Wang memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 678d0416d48SJason Wang page_size, addr); 679d0416d48SJason Wang g_free(name); 680d0416d48SJason Wang 681d0416d48SJason Wang if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { 68298f7607eSLaurent Vivier object_unparent(OBJECT(&n->mr)); 683d0416d48SJason Wang munmap(addr, page_size); 684d0416d48SJason Wang goto err; 685d0416d48SJason Wang } 686d0416d48SJason Wang n->addr = addr; 687d0416d48SJason Wang 688d0416d48SJason Wang return 0; 689d0416d48SJason Wang 690d0416d48SJason Wang err: 691d0416d48SJason Wang return -1; 692d0416d48SJason Wang } 693d0416d48SJason Wang 694b1f030a0SLaurent Vivier static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) 695b1f030a0SLaurent Vivier { 696b1f030a0SLaurent Vivier int i; 697b1f030a0SLaurent Vivier 698e66f2311SLongpeng /* 699e66f2311SLongpeng * Pack all the changes to the memory regions in a single 700e66f2311SLongpeng * transaction to avoid a few updating of the address space 701e66f2311SLongpeng * topology. 702e66f2311SLongpeng */ 703e66f2311SLongpeng memory_region_transaction_begin(); 704e66f2311SLongpeng 705b1f030a0SLaurent Vivier for (i = dev->vq_index; i < dev->vq_index + n; i++) { 706b1f030a0SLaurent Vivier vhost_vdpa_host_notifier_uninit(dev, i); 707b1f030a0SLaurent Vivier } 708e66f2311SLongpeng 709e66f2311SLongpeng memory_region_transaction_commit(); 710b1f030a0SLaurent Vivier } 711b1f030a0SLaurent Vivier 712d0416d48SJason Wang static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) 713d0416d48SJason Wang { 714dff4426fSEugenio Pérez struct vhost_vdpa *v = dev->opaque; 715d0416d48SJason Wang int i; 716d0416d48SJason Wang 717dff4426fSEugenio Pérez if (v->shadow_vqs_enabled) { 718dff4426fSEugenio Pérez /* FIXME SVQ is not compatible with host notifiers mr */ 719dff4426fSEugenio Pérez return; 720dff4426fSEugenio Pérez } 721dff4426fSEugenio Pérez 722e66f2311SLongpeng /* 723e66f2311SLongpeng * Pack all the changes to the memory regions in a single 724e66f2311SLongpeng * transaction to avoid a few updating of the address space 725e66f2311SLongpeng * topology. 726e66f2311SLongpeng */ 727e66f2311SLongpeng memory_region_transaction_begin(); 728e66f2311SLongpeng 729d0416d48SJason Wang for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { 730d0416d48SJason Wang if (vhost_vdpa_host_notifier_init(dev, i)) { 731b1f030a0SLaurent Vivier vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index); 732e66f2311SLongpeng break; 733e66f2311SLongpeng } 734e66f2311SLongpeng } 735e66f2311SLongpeng 736e66f2311SLongpeng memory_region_transaction_commit(); 737d0416d48SJason Wang } 738d0416d48SJason Wang 739dff4426fSEugenio Pérez static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev) 740dff4426fSEugenio Pérez { 741dff4426fSEugenio Pérez struct vhost_vdpa *v = dev->opaque; 742dff4426fSEugenio Pérez size_t idx; 743dff4426fSEugenio Pérez 744dff4426fSEugenio Pérez for (idx = 0; idx < v->shadow_vqs->len; ++idx) { 745dff4426fSEugenio Pérez vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx)); 746dff4426fSEugenio Pérez } 747dff4426fSEugenio Pérez g_ptr_array_free(v->shadow_vqs, true); 748dff4426fSEugenio Pérez } 749dff4426fSEugenio Pérez 750108a6481SCindy Lu static int vhost_vdpa_cleanup(struct vhost_dev *dev) 751108a6481SCindy Lu { 752108a6481SCindy Lu struct vhost_vdpa *v; 753108a6481SCindy Lu assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 754108a6481SCindy Lu v = dev->opaque; 755778e67deSLaurent Vivier trace_vhost_vdpa_cleanup(dev, v); 756a230c471SEugenio Pérez if (vhost_vdpa_first_dev(dev)) { 757a230c471SEugenio Pérez ram_block_discard_disable(false); 758a230c471SEugenio Pérez } 759a230c471SEugenio Pérez 760d0416d48SJason Wang vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 761108a6481SCindy Lu memory_listener_unregister(&v->listener); 762dff4426fSEugenio Pérez vhost_vdpa_svq_cleanup(dev); 763108a6481SCindy Lu 764108a6481SCindy Lu dev->opaque = NULL; 765e1c1915bSDavid Hildenbrand 766108a6481SCindy Lu return 0; 767108a6481SCindy Lu } 768108a6481SCindy Lu 769108a6481SCindy Lu static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 770108a6481SCindy Lu { 771778e67deSLaurent Vivier trace_vhost_vdpa_memslots_limit(dev, INT_MAX); 772108a6481SCindy Lu return INT_MAX; 773108a6481SCindy Lu } 774108a6481SCindy Lu 775108a6481SCindy Lu static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 776108a6481SCindy Lu struct vhost_memory *mem) 777108a6481SCindy Lu { 778d71b0609SSi-Wei Liu if (!vhost_vdpa_first_dev(dev)) { 7794d191cfdSJason Wang return 0; 7804d191cfdSJason Wang } 7814d191cfdSJason Wang 782778e67deSLaurent Vivier trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding); 783778e67deSLaurent Vivier if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) && 784778e67deSLaurent Vivier trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) { 785778e67deSLaurent Vivier int i; 786778e67deSLaurent Vivier for (i = 0; i < mem->nregions; i++) { 787778e67deSLaurent Vivier trace_vhost_vdpa_dump_regions(dev, i, 788778e67deSLaurent Vivier mem->regions[i].guest_phys_addr, 789778e67deSLaurent Vivier mem->regions[i].memory_size, 790778e67deSLaurent Vivier mem->regions[i].userspace_addr, 791778e67deSLaurent Vivier mem->regions[i].flags_padding); 792778e67deSLaurent Vivier } 793778e67deSLaurent Vivier } 794108a6481SCindy Lu if (mem->padding) { 7953631151bSRoman Kagan return -EINVAL; 796108a6481SCindy Lu } 797108a6481SCindy Lu 798108a6481SCindy Lu return 0; 799108a6481SCindy Lu } 800108a6481SCindy Lu 801108a6481SCindy Lu static int vhost_vdpa_set_features(struct vhost_dev *dev, 802108a6481SCindy Lu uint64_t features) 803108a6481SCindy Lu { 80412a195faSEugenio Pérez struct vhost_vdpa *v = dev->opaque; 805108a6481SCindy Lu int ret; 8064d191cfdSJason Wang 807d71b0609SSi-Wei Liu if (!vhost_vdpa_first_dev(dev)) { 8084d191cfdSJason Wang return 0; 8094d191cfdSJason Wang } 8104d191cfdSJason Wang 81112a195faSEugenio Pérez if (v->shadow_vqs_enabled) { 81212a195faSEugenio Pérez if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) { 81312a195faSEugenio Pérez /* 81412a195faSEugenio Pérez * QEMU is just trying to enable or disable logging. SVQ handles 81512a195faSEugenio Pérez * this sepparately, so no need to forward this. 81612a195faSEugenio Pérez */ 81712a195faSEugenio Pérez v->acked_features = features; 81812a195faSEugenio Pérez return 0; 81912a195faSEugenio Pérez } 82012a195faSEugenio Pérez 82112a195faSEugenio Pérez v->acked_features = features; 82212a195faSEugenio Pérez 82312a195faSEugenio Pérez /* We must not ack _F_LOG if SVQ is enabled */ 82412a195faSEugenio Pérez features &= ~BIT_ULL(VHOST_F_LOG_ALL); 82512a195faSEugenio Pérez } 82612a195faSEugenio Pérez 827778e67deSLaurent Vivier trace_vhost_vdpa_set_features(dev, features); 828108a6481SCindy Lu ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 829108a6481SCindy Lu if (ret) { 830108a6481SCindy Lu return ret; 831108a6481SCindy Lu } 832108a6481SCindy Lu 8333631151bSRoman Kagan return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 834108a6481SCindy Lu } 835108a6481SCindy Lu 836a5bd0580SJason Wang static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) 837a5bd0580SJason Wang { 8384da38d1aSEugenio Pérez struct vhost_vdpa *v = dev->opaque; 8394da38d1aSEugenio Pérez 840a5bd0580SJason Wang uint64_t features; 841a5bd0580SJason Wang uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 842c1a10086SEugenio Pérez 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | 843d83b4945SEugenio Pérez 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | 844d83b4945SEugenio Pérez 0x1ULL << VHOST_BACKEND_F_SUSPEND; 845a5bd0580SJason Wang int r; 846a5bd0580SJason Wang 847a5bd0580SJason Wang if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { 8482a83e97eSJason Wang return -EFAULT; 849a5bd0580SJason Wang } 850a5bd0580SJason Wang 851a5bd0580SJason Wang features &= f; 8524d191cfdSJason Wang 853d71b0609SSi-Wei Liu if (vhost_vdpa_first_dev(dev)) { 854a5bd0580SJason Wang r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); 855a5bd0580SJason Wang if (r) { 8562a83e97eSJason Wang return -EFAULT; 857a5bd0580SJason Wang } 8584d191cfdSJason Wang } 859a5bd0580SJason Wang 860a5bd0580SJason Wang dev->backend_cap = features; 8614da38d1aSEugenio Pérez v->shared->backend_cap = features; 862a5bd0580SJason Wang 863a5bd0580SJason Wang return 0; 864a5bd0580SJason Wang } 865a5bd0580SJason Wang 866c232b8f4SZenghui Yu static int vhost_vdpa_get_device_id(struct vhost_dev *dev, 867108a6481SCindy Lu uint32_t *device_id) 868108a6481SCindy Lu { 869778e67deSLaurent Vivier int ret; 870778e67deSLaurent Vivier ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 871778e67deSLaurent Vivier trace_vhost_vdpa_get_device_id(dev, *device_id); 872778e67deSLaurent Vivier return ret; 873108a6481SCindy Lu } 874108a6481SCindy Lu 875108a6481SCindy Lu static int vhost_vdpa_reset_device(struct vhost_dev *dev) 876108a6481SCindy Lu { 8770bb302a9SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 878778e67deSLaurent Vivier int ret; 879108a6481SCindy Lu uint8_t status = 0; 880108a6481SCindy Lu 881778e67deSLaurent Vivier ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 8822b5de4d7SEugenio Pérez trace_vhost_vdpa_reset_device(dev); 8830bb302a9SEugenio Pérez v->suspended = false; 884778e67deSLaurent Vivier return ret; 885108a6481SCindy Lu } 886108a6481SCindy Lu 887108a6481SCindy Lu static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 888108a6481SCindy Lu { 889108a6481SCindy Lu assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 890108a6481SCindy Lu 891353244d8SJason Wang trace_vhost_vdpa_get_vq_index(dev, idx, idx); 892353244d8SJason Wang return idx; 893108a6481SCindy Lu } 894108a6481SCindy Lu 895d7ce0841SEugenio Pérez int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx) 896108a6481SCindy Lu { 897d7ce0841SEugenio Pérez struct vhost_dev *dev = v->dev; 898108a6481SCindy Lu struct vhost_vring_state state = { 899d7ce0841SEugenio Pérez .index = idx, 900108a6481SCindy Lu .num = 1, 901108a6481SCindy Lu }; 902d7ce0841SEugenio Pérez int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 903d7ce0841SEugenio Pérez 904d7ce0841SEugenio Pérez trace_vhost_vdpa_set_vring_ready(dev, idx, r); 905d7ce0841SEugenio Pérez return r; 906108a6481SCindy Lu } 907108a6481SCindy Lu 908259f3accSCindy Lu static int vhost_vdpa_set_config_call(struct vhost_dev *dev, 909259f3accSCindy Lu int fd) 910259f3accSCindy Lu { 911259f3accSCindy Lu trace_vhost_vdpa_set_config_call(dev, fd); 912259f3accSCindy Lu return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd); 913259f3accSCindy Lu } 914259f3accSCindy Lu 915778e67deSLaurent Vivier static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config, 916778e67deSLaurent Vivier uint32_t config_len) 917778e67deSLaurent Vivier { 918778e67deSLaurent Vivier int b, len; 919778e67deSLaurent Vivier char line[QEMU_HEXDUMP_LINE_LEN]; 920778e67deSLaurent Vivier 921778e67deSLaurent Vivier for (b = 0; b < config_len; b += 16) { 922778e67deSLaurent Vivier len = config_len - b; 923778e67deSLaurent Vivier qemu_hexdump_line(line, b, config, len, false); 924778e67deSLaurent Vivier trace_vhost_vdpa_dump_config(dev, line); 925778e67deSLaurent Vivier } 926778e67deSLaurent Vivier } 927778e67deSLaurent Vivier 928108a6481SCindy Lu static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 929108a6481SCindy Lu uint32_t offset, uint32_t size, 930108a6481SCindy Lu uint32_t flags) 931108a6481SCindy Lu { 932108a6481SCindy Lu struct vhost_vdpa_config *config; 933108a6481SCindy Lu int ret; 934108a6481SCindy Lu unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 935986d4f78SLi Qiang 936778e67deSLaurent Vivier trace_vhost_vdpa_set_config(dev, offset, size, flags); 937108a6481SCindy Lu config = g_malloc(size + config_size); 938108a6481SCindy Lu config->off = offset; 939108a6481SCindy Lu config->len = size; 940108a6481SCindy Lu memcpy(config->buf, data, size); 941778e67deSLaurent Vivier if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) && 942778e67deSLaurent Vivier trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 943778e67deSLaurent Vivier vhost_vdpa_dump_config(dev, data, size); 944778e67deSLaurent Vivier } 945108a6481SCindy Lu ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 946108a6481SCindy Lu g_free(config); 947108a6481SCindy Lu return ret; 948108a6481SCindy Lu } 949108a6481SCindy Lu 950108a6481SCindy Lu static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 95150de5138SKevin Wolf uint32_t config_len, Error **errp) 952108a6481SCindy Lu { 953108a6481SCindy Lu struct vhost_vdpa_config *v_config; 954108a6481SCindy Lu unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 955108a6481SCindy Lu int ret; 956108a6481SCindy Lu 957778e67deSLaurent Vivier trace_vhost_vdpa_get_config(dev, config, config_len); 958108a6481SCindy Lu v_config = g_malloc(config_len + config_size); 959108a6481SCindy Lu v_config->len = config_len; 960108a6481SCindy Lu v_config->off = 0; 961108a6481SCindy Lu ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 962108a6481SCindy Lu memcpy(config, v_config->buf, config_len); 963108a6481SCindy Lu g_free(v_config); 964778e67deSLaurent Vivier if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) && 965778e67deSLaurent Vivier trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 966778e67deSLaurent Vivier vhost_vdpa_dump_config(dev, config, config_len); 967778e67deSLaurent Vivier } 968108a6481SCindy Lu return ret; 969108a6481SCindy Lu } 970108a6481SCindy Lu 971d96be4c8SEugenio Pérez static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev, 972d96be4c8SEugenio Pérez struct vhost_vring_state *ring) 973d96be4c8SEugenio Pérez { 974d96be4c8SEugenio Pérez trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num); 975d96be4c8SEugenio Pérez return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 976d96be4c8SEugenio Pérez } 977d96be4c8SEugenio Pérez 978dff4426fSEugenio Pérez static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev, 979dff4426fSEugenio Pérez struct vhost_vring_file *file) 980dff4426fSEugenio Pérez { 981dff4426fSEugenio Pérez trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd); 982dff4426fSEugenio Pérez return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 983dff4426fSEugenio Pérez } 984dff4426fSEugenio Pérez 985a8ac8858SEugenio Pérez static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev, 986a8ac8858SEugenio Pérez struct vhost_vring_file *file) 987a8ac8858SEugenio Pérez { 988a8ac8858SEugenio Pérez trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd); 989a8ac8858SEugenio Pérez return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 990a8ac8858SEugenio Pérez } 991a8ac8858SEugenio Pérez 992d96be4c8SEugenio Pérez static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev, 993d96be4c8SEugenio Pérez struct vhost_vring_addr *addr) 994d96be4c8SEugenio Pérez { 995d96be4c8SEugenio Pérez trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags, 996d96be4c8SEugenio Pérez addr->desc_user_addr, addr->used_user_addr, 997d96be4c8SEugenio Pérez addr->avail_user_addr, 998d96be4c8SEugenio Pérez addr->log_guest_addr); 999d96be4c8SEugenio Pérez 1000d96be4c8SEugenio Pérez return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 1001d96be4c8SEugenio Pérez 1002d96be4c8SEugenio Pérez } 1003d96be4c8SEugenio Pérez 1004dff4426fSEugenio Pérez /** 1005dff4426fSEugenio Pérez * Set the shadow virtqueue descriptors to the device 1006dff4426fSEugenio Pérez * 1007dff4426fSEugenio Pérez * @dev: The vhost device model 1008dff4426fSEugenio Pérez * @svq: The shadow virtqueue 1009dff4426fSEugenio Pérez * @idx: The index of the virtqueue in the vhost device 1010dff4426fSEugenio Pérez * @errp: Error 1011a8ac8858SEugenio Pérez * 1012a8ac8858SEugenio Pérez * Note that this function does not rewind kick file descriptor if cannot set 1013a8ac8858SEugenio Pérez * call one. 1014dff4426fSEugenio Pérez */ 1015100890f7SEugenio Pérez static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev, 1016dff4426fSEugenio Pérez VhostShadowVirtqueue *svq, unsigned idx, 1017dff4426fSEugenio Pérez Error **errp) 1018dff4426fSEugenio Pérez { 1019dff4426fSEugenio Pérez struct vhost_vring_file file = { 1020dff4426fSEugenio Pérez .index = dev->vq_index + idx, 1021dff4426fSEugenio Pérez }; 1022dff4426fSEugenio Pérez const EventNotifier *event_notifier = &svq->hdev_kick; 1023dff4426fSEugenio Pérez int r; 1024dff4426fSEugenio Pérez 10253cfb4d06SEugenio Pérez r = event_notifier_init(&svq->hdev_kick, 0); 10263cfb4d06SEugenio Pérez if (r != 0) { 10273cfb4d06SEugenio Pérez error_setg_errno(errp, -r, "Couldn't create kick event notifier"); 10283cfb4d06SEugenio Pérez goto err_init_hdev_kick; 10293cfb4d06SEugenio Pérez } 10303cfb4d06SEugenio Pérez 10313cfb4d06SEugenio Pérez r = event_notifier_init(&svq->hdev_call, 0); 10323cfb4d06SEugenio Pérez if (r != 0) { 10333cfb4d06SEugenio Pérez error_setg_errno(errp, -r, "Couldn't create call event notifier"); 10343cfb4d06SEugenio Pérez goto err_init_hdev_call; 10353cfb4d06SEugenio Pérez } 10363cfb4d06SEugenio Pérez 1037dff4426fSEugenio Pérez file.fd = event_notifier_get_fd(event_notifier); 1038dff4426fSEugenio Pérez r = vhost_vdpa_set_vring_dev_kick(dev, &file); 1039dff4426fSEugenio Pérez if (unlikely(r != 0)) { 1040dff4426fSEugenio Pérez error_setg_errno(errp, -r, "Can't set device kick fd"); 10413cfb4d06SEugenio Pérez goto err_init_set_dev_fd; 1042a8ac8858SEugenio Pérez } 1043a8ac8858SEugenio Pérez 1044a8ac8858SEugenio Pérez event_notifier = &svq->hdev_call; 1045a8ac8858SEugenio Pérez file.fd = event_notifier_get_fd(event_notifier); 1046a8ac8858SEugenio Pérez r = vhost_vdpa_set_vring_dev_call(dev, &file); 1047a8ac8858SEugenio Pérez if (unlikely(r != 0)) { 1048a8ac8858SEugenio Pérez error_setg_errno(errp, -r, "Can't set device call fd"); 10493cfb4d06SEugenio Pérez goto err_init_set_dev_fd; 1050dff4426fSEugenio Pérez } 1051dff4426fSEugenio Pérez 10523cfb4d06SEugenio Pérez return 0; 10533cfb4d06SEugenio Pérez 10543cfb4d06SEugenio Pérez err_init_set_dev_fd: 10553cfb4d06SEugenio Pérez event_notifier_set_handler(&svq->hdev_call, NULL); 10563cfb4d06SEugenio Pérez 10573cfb4d06SEugenio Pérez err_init_hdev_call: 10583cfb4d06SEugenio Pérez event_notifier_cleanup(&svq->hdev_kick); 10593cfb4d06SEugenio Pérez 10603cfb4d06SEugenio Pérez err_init_hdev_kick: 1061100890f7SEugenio Pérez return r; 1062100890f7SEugenio Pérez } 1063100890f7SEugenio Pérez 1064100890f7SEugenio Pérez /** 1065100890f7SEugenio Pérez * Unmap a SVQ area in the device 1066100890f7SEugenio Pérez */ 10678b6d6119SEugenio Pérez static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr) 1068100890f7SEugenio Pérez { 10698b6d6119SEugenio Pérez const DMAMap needle = { 10708b6d6119SEugenio Pérez .translated_addr = addr, 10718b6d6119SEugenio Pérez }; 10725edb02e8SEugenio Pérez const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree, 10735edb02e8SEugenio Pérez &needle); 107434e3c94eSEugenio Pérez hwaddr size; 1075100890f7SEugenio Pérez int r; 1076100890f7SEugenio Pérez 107734e3c94eSEugenio Pérez if (unlikely(!result)) { 107834e3c94eSEugenio Pérez error_report("Unable to find SVQ address to unmap"); 10795b590f51SEugenio Pérez return; 108034e3c94eSEugenio Pérez } 108134e3c94eSEugenio Pérez 10828e3b0cbbSMarc-André Lureau size = ROUND_UP(result->size, qemu_real_host_page_size()); 1083*6f03d9efSEugenio Pérez r = vhost_vdpa_dma_unmap(v->shared, v->address_space_id, result->iova, 1084*6f03d9efSEugenio Pérez size); 1085b37c12beSEugenio Pérez if (unlikely(r < 0)) { 1086b37c12beSEugenio Pérez error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r); 10875b590f51SEugenio Pérez return; 1088b37c12beSEugenio Pérez } 1089b37c12beSEugenio Pérez 10905edb02e8SEugenio Pérez vhost_iova_tree_remove(v->shared->iova_tree, *result); 1091100890f7SEugenio Pérez } 1092100890f7SEugenio Pérez 10935b590f51SEugenio Pérez static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev, 1094100890f7SEugenio Pérez const VhostShadowVirtqueue *svq) 1095100890f7SEugenio Pérez { 1096100890f7SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1097100890f7SEugenio Pérez struct vhost_vring_addr svq_addr; 1098100890f7SEugenio Pérez 1099100890f7SEugenio Pérez vhost_svq_get_vring_addr(svq, &svq_addr); 1100100890f7SEugenio Pérez 11018b6d6119SEugenio Pérez vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr); 1102100890f7SEugenio Pérez 11038b6d6119SEugenio Pérez vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr); 110434e3c94eSEugenio Pérez } 110534e3c94eSEugenio Pérez 110634e3c94eSEugenio Pérez /** 110734e3c94eSEugenio Pérez * Map the SVQ area in the device 110834e3c94eSEugenio Pérez * 110934e3c94eSEugenio Pérez * @v: Vhost-vdpa device 111034e3c94eSEugenio Pérez * @needle: The area to search iova 111134e3c94eSEugenio Pérez * @errorp: Error pointer 111234e3c94eSEugenio Pérez */ 111334e3c94eSEugenio Pérez static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle, 111434e3c94eSEugenio Pérez Error **errp) 111534e3c94eSEugenio Pérez { 111634e3c94eSEugenio Pérez int r; 111734e3c94eSEugenio Pérez 11185edb02e8SEugenio Pérez r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle); 111934e3c94eSEugenio Pérez if (unlikely(r != IOVA_OK)) { 112034e3c94eSEugenio Pérez error_setg(errp, "Cannot allocate iova (%d)", r); 112134e3c94eSEugenio Pérez return false; 112234e3c94eSEugenio Pérez } 112334e3c94eSEugenio Pérez 1124*6f03d9efSEugenio Pérez r = vhost_vdpa_dma_map(v->shared, v->address_space_id, needle->iova, 1125cd831ed5SEugenio Pérez needle->size + 1, 112634e3c94eSEugenio Pérez (void *)(uintptr_t)needle->translated_addr, 112734e3c94eSEugenio Pérez needle->perm == IOMMU_RO); 112834e3c94eSEugenio Pérez if (unlikely(r != 0)) { 112934e3c94eSEugenio Pérez error_setg_errno(errp, -r, "Cannot map region to device"); 11305edb02e8SEugenio Pérez vhost_iova_tree_remove(v->shared->iova_tree, *needle); 113134e3c94eSEugenio Pérez } 113234e3c94eSEugenio Pérez 113334e3c94eSEugenio Pérez return r == 0; 1134100890f7SEugenio Pérez } 1135100890f7SEugenio Pérez 1136100890f7SEugenio Pérez /** 1137100890f7SEugenio Pérez * Map the shadow virtqueue rings in the device 1138100890f7SEugenio Pérez * 1139100890f7SEugenio Pérez * @dev: The vhost device 1140100890f7SEugenio Pérez * @svq: The shadow virtqueue 1141100890f7SEugenio Pérez * @addr: Assigned IOVA addresses 1142100890f7SEugenio Pérez * @errp: Error pointer 1143100890f7SEugenio Pérez */ 1144100890f7SEugenio Pérez static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev, 1145100890f7SEugenio Pérez const VhostShadowVirtqueue *svq, 1146100890f7SEugenio Pérez struct vhost_vring_addr *addr, 1147100890f7SEugenio Pérez Error **errp) 1148100890f7SEugenio Pérez { 114905e385d2SMarkus Armbruster ERRP_GUARD(); 115034e3c94eSEugenio Pérez DMAMap device_region, driver_region; 115134e3c94eSEugenio Pérez struct vhost_vring_addr svq_addr; 1152100890f7SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1153100890f7SEugenio Pérez size_t device_size = vhost_svq_device_area_size(svq); 1154100890f7SEugenio Pérez size_t driver_size = vhost_svq_driver_area_size(svq); 115534e3c94eSEugenio Pérez size_t avail_offset; 115634e3c94eSEugenio Pérez bool ok; 1157100890f7SEugenio Pérez 115834e3c94eSEugenio Pérez vhost_svq_get_vring_addr(svq, &svq_addr); 1159100890f7SEugenio Pérez 116034e3c94eSEugenio Pérez driver_region = (DMAMap) { 116134e3c94eSEugenio Pérez .translated_addr = svq_addr.desc_user_addr, 116234e3c94eSEugenio Pérez .size = driver_size - 1, 116334e3c94eSEugenio Pérez .perm = IOMMU_RO, 116434e3c94eSEugenio Pérez }; 116534e3c94eSEugenio Pérez ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp); 116634e3c94eSEugenio Pérez if (unlikely(!ok)) { 116734e3c94eSEugenio Pérez error_prepend(errp, "Cannot create vq driver region: "); 1168100890f7SEugenio Pérez return false; 1169100890f7SEugenio Pérez } 117034e3c94eSEugenio Pérez addr->desc_user_addr = driver_region.iova; 117134e3c94eSEugenio Pérez avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr; 117234e3c94eSEugenio Pérez addr->avail_user_addr = driver_region.iova + avail_offset; 1173100890f7SEugenio Pérez 117434e3c94eSEugenio Pérez device_region = (DMAMap) { 117534e3c94eSEugenio Pérez .translated_addr = svq_addr.used_user_addr, 117634e3c94eSEugenio Pérez .size = device_size - 1, 117734e3c94eSEugenio Pérez .perm = IOMMU_RW, 117834e3c94eSEugenio Pérez }; 117934e3c94eSEugenio Pérez ok = vhost_vdpa_svq_map_ring(v, &device_region, errp); 118034e3c94eSEugenio Pérez if (unlikely(!ok)) { 118134e3c94eSEugenio Pérez error_prepend(errp, "Cannot create vq device region: "); 11828b6d6119SEugenio Pérez vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr); 1183100890f7SEugenio Pérez } 118434e3c94eSEugenio Pérez addr->used_user_addr = device_region.iova; 1185100890f7SEugenio Pérez 118634e3c94eSEugenio Pérez return ok; 1187100890f7SEugenio Pérez } 1188100890f7SEugenio Pérez 1189100890f7SEugenio Pérez static bool vhost_vdpa_svq_setup(struct vhost_dev *dev, 1190100890f7SEugenio Pérez VhostShadowVirtqueue *svq, unsigned idx, 1191100890f7SEugenio Pérez Error **errp) 1192100890f7SEugenio Pérez { 1193100890f7SEugenio Pérez uint16_t vq_index = dev->vq_index + idx; 1194100890f7SEugenio Pérez struct vhost_vring_state s = { 1195100890f7SEugenio Pérez .index = vq_index, 1196100890f7SEugenio Pérez }; 1197100890f7SEugenio Pérez int r; 1198100890f7SEugenio Pérez 1199100890f7SEugenio Pérez r = vhost_vdpa_set_dev_vring_base(dev, &s); 1200100890f7SEugenio Pérez if (unlikely(r)) { 1201100890f7SEugenio Pérez error_setg_errno(errp, -r, "Cannot set vring base"); 1202100890f7SEugenio Pérez return false; 1203100890f7SEugenio Pérez } 1204100890f7SEugenio Pérez 1205100890f7SEugenio Pérez r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp); 1206dff4426fSEugenio Pérez return r == 0; 1207dff4426fSEugenio Pérez } 1208dff4426fSEugenio Pérez 1209dff4426fSEugenio Pérez static bool vhost_vdpa_svqs_start(struct vhost_dev *dev) 1210dff4426fSEugenio Pérez { 1211dff4426fSEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1212dff4426fSEugenio Pérez Error *err = NULL; 1213dff4426fSEugenio Pérez unsigned i; 1214dff4426fSEugenio Pérez 1215712c1a31SEugenio Pérez if (!v->shadow_vqs_enabled) { 1216dff4426fSEugenio Pérez return true; 1217dff4426fSEugenio Pérez } 1218dff4426fSEugenio Pérez 1219dff4426fSEugenio Pérez for (i = 0; i < v->shadow_vqs->len; ++i) { 1220100890f7SEugenio Pérez VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i); 1221dff4426fSEugenio Pérez VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1222100890f7SEugenio Pérez struct vhost_vring_addr addr = { 12231c82fdfeSEugenio Pérez .index = dev->vq_index + i, 1224100890f7SEugenio Pérez }; 1225100890f7SEugenio Pérez int r; 1226dff4426fSEugenio Pérez bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err); 1227dff4426fSEugenio Pérez if (unlikely(!ok)) { 1228100890f7SEugenio Pérez goto err; 1229100890f7SEugenio Pérez } 1230100890f7SEugenio Pérez 12315edb02e8SEugenio Pérez vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree); 1232100890f7SEugenio Pérez ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err); 1233100890f7SEugenio Pérez if (unlikely(!ok)) { 1234100890f7SEugenio Pérez goto err_map; 1235100890f7SEugenio Pérez } 1236100890f7SEugenio Pérez 1237100890f7SEugenio Pérez /* Override vring GPA set by vhost subsystem */ 1238100890f7SEugenio Pérez r = vhost_vdpa_set_vring_dev_addr(dev, &addr); 1239100890f7SEugenio Pérez if (unlikely(r != 0)) { 1240100890f7SEugenio Pérez error_setg_errno(&err, -r, "Cannot set device address"); 1241100890f7SEugenio Pérez goto err_set_addr; 1242100890f7SEugenio Pérez } 1243100890f7SEugenio Pérez } 1244100890f7SEugenio Pérez 1245100890f7SEugenio Pérez return true; 1246100890f7SEugenio Pérez 1247100890f7SEugenio Pérez err_set_addr: 1248100890f7SEugenio Pérez vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i)); 1249100890f7SEugenio Pérez 1250100890f7SEugenio Pérez err_map: 1251100890f7SEugenio Pérez vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i)); 1252100890f7SEugenio Pérez 1253100890f7SEugenio Pérez err: 1254dff4426fSEugenio Pérez error_reportf_err(err, "Cannot setup SVQ %u: ", i); 1255100890f7SEugenio Pérez for (unsigned j = 0; j < i; ++j) { 1256100890f7SEugenio Pérez VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j); 1257100890f7SEugenio Pérez vhost_vdpa_svq_unmap_rings(dev, svq); 1258100890f7SEugenio Pérez vhost_svq_stop(svq); 1259100890f7SEugenio Pérez } 1260100890f7SEugenio Pérez 1261100890f7SEugenio Pérez return false; 1262100890f7SEugenio Pérez } 1263100890f7SEugenio Pérez 12645b590f51SEugenio Pérez static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) 1265100890f7SEugenio Pérez { 1266100890f7SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1267100890f7SEugenio Pérez 1268712c1a31SEugenio Pérez if (!v->shadow_vqs_enabled) { 12695b590f51SEugenio Pérez return; 1270100890f7SEugenio Pérez } 1271100890f7SEugenio Pérez 1272100890f7SEugenio Pérez for (unsigned i = 0; i < v->shadow_vqs->len; ++i) { 1273100890f7SEugenio Pérez VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 12742e1a9de9SEugenio Pérez 12752e1a9de9SEugenio Pérez vhost_svq_stop(svq); 12765b590f51SEugenio Pérez vhost_vdpa_svq_unmap_rings(dev, svq); 12773cfb4d06SEugenio Pérez 12783cfb4d06SEugenio Pérez event_notifier_cleanup(&svq->hdev_kick); 12793cfb4d06SEugenio Pérez event_notifier_cleanup(&svq->hdev_call); 1280dff4426fSEugenio Pérez } 1281dff4426fSEugenio Pérez } 1282dff4426fSEugenio Pérez 12830bb302a9SEugenio Pérez static void vhost_vdpa_suspend(struct vhost_dev *dev) 12840bb302a9SEugenio Pérez { 12850bb302a9SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 12860bb302a9SEugenio Pérez int r; 12870bb302a9SEugenio Pérez 12880bb302a9SEugenio Pérez if (!vhost_vdpa_first_dev(dev)) { 12890bb302a9SEugenio Pérez return; 12900bb302a9SEugenio Pérez } 12910bb302a9SEugenio Pérez 12920bb302a9SEugenio Pérez if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { 12930bb302a9SEugenio Pérez trace_vhost_vdpa_suspend(dev); 1294f12b2498SEugenio Pérez r = ioctl(v->shared->device_fd, VHOST_VDPA_SUSPEND); 12950bb302a9SEugenio Pérez if (unlikely(r)) { 12960bb302a9SEugenio Pérez error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); 12970bb302a9SEugenio Pérez } else { 12980bb302a9SEugenio Pérez v->suspended = true; 12990bb302a9SEugenio Pérez return; 13000bb302a9SEugenio Pérez } 13010bb302a9SEugenio Pérez } 13020bb302a9SEugenio Pérez 13030bb302a9SEugenio Pérez vhost_vdpa_reset_device(dev); 13040bb302a9SEugenio Pérez } 13050bb302a9SEugenio Pérez 1306108a6481SCindy Lu static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 1307108a6481SCindy Lu { 1308108a6481SCindy Lu struct vhost_vdpa *v = dev->opaque; 1309dff4426fSEugenio Pérez bool ok; 1310778e67deSLaurent Vivier trace_vhost_vdpa_dev_start(dev, started); 13114d191cfdSJason Wang 13124d191cfdSJason Wang if (started) { 13134d191cfdSJason Wang vhost_vdpa_host_notifiers_init(dev); 1314dff4426fSEugenio Pérez ok = vhost_vdpa_svqs_start(dev); 1315dff4426fSEugenio Pérez if (unlikely(!ok)) { 1316dff4426fSEugenio Pérez return -1; 1317dff4426fSEugenio Pérez } 13184d191cfdSJason Wang } else { 13190bb302a9SEugenio Pérez vhost_vdpa_suspend(dev); 13205b590f51SEugenio Pérez vhost_vdpa_svqs_stop(dev); 13214d191cfdSJason Wang vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 13224d191cfdSJason Wang } 13234d191cfdSJason Wang 1324245cf2c2SEugenio Pérez if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 13254d191cfdSJason Wang return 0; 13264d191cfdSJason Wang } 13274d191cfdSJason Wang 1328108a6481SCindy Lu if (started) { 1329bc7b0cacSCindy Lu if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) { 1330bc7b0cacSCindy Lu error_report("SVQ can not work while IOMMU enable, please disable" 1331bc7b0cacSCindy Lu "IOMMU and try again"); 1332bc7b0cacSCindy Lu return -1; 1333bc7b0cacSCindy Lu } 1334bc7b0cacSCindy Lu memory_listener_register(&v->listener, dev->vdev->dma_as); 1335bc7b0cacSCindy Lu 13363631151bSRoman Kagan return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 1337c3716f26SEugenio Pérez } 1338c3716f26SEugenio Pérez 1339c3716f26SEugenio Pérez return 0; 1340c3716f26SEugenio Pérez } 1341c3716f26SEugenio Pérez 1342c3716f26SEugenio Pérez static void vhost_vdpa_reset_status(struct vhost_dev *dev) 1343c3716f26SEugenio Pérez { 1344c3716f26SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1345c3716f26SEugenio Pérez 1346c3716f26SEugenio Pérez if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1347c3716f26SEugenio Pérez return; 1348c3716f26SEugenio Pérez } 1349c3716f26SEugenio Pérez 1350108a6481SCindy Lu vhost_vdpa_reset_device(dev); 1351108a6481SCindy Lu vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 1352108a6481SCindy Lu VIRTIO_CONFIG_S_DRIVER); 1353108a6481SCindy Lu memory_listener_unregister(&v->listener); 1354108a6481SCindy Lu } 1355108a6481SCindy Lu 1356108a6481SCindy Lu static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 1357108a6481SCindy Lu struct vhost_log *log) 1358108a6481SCindy Lu { 1359773ebc95SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1360d71b0609SSi-Wei Liu if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) { 13614d191cfdSJason Wang return 0; 13624d191cfdSJason Wang } 13634d191cfdSJason Wang 1364778e67deSLaurent Vivier trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd, 1365778e67deSLaurent Vivier log->log); 1366108a6481SCindy Lu return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 1367108a6481SCindy Lu } 1368108a6481SCindy Lu 1369108a6481SCindy Lu static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 1370108a6481SCindy Lu struct vhost_vring_addr *addr) 1371108a6481SCindy Lu { 1372d96be4c8SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1373d96be4c8SEugenio Pérez 1374d96be4c8SEugenio Pérez if (v->shadow_vqs_enabled) { 1375d96be4c8SEugenio Pérez /* 1376d96be4c8SEugenio Pérez * Device vring addr was set at device start. SVQ base is handled by 1377d96be4c8SEugenio Pérez * VirtQueue code. 1378d96be4c8SEugenio Pérez */ 1379d96be4c8SEugenio Pérez return 0; 1380d96be4c8SEugenio Pérez } 1381d96be4c8SEugenio Pérez 1382d96be4c8SEugenio Pérez return vhost_vdpa_set_vring_dev_addr(dev, addr); 1383108a6481SCindy Lu } 1384108a6481SCindy Lu 1385108a6481SCindy Lu static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 1386108a6481SCindy Lu struct vhost_vring_state *ring) 1387108a6481SCindy Lu { 1388778e67deSLaurent Vivier trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num); 1389108a6481SCindy Lu return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 1390108a6481SCindy Lu } 1391108a6481SCindy Lu 1392108a6481SCindy Lu static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 1393108a6481SCindy Lu struct vhost_vring_state *ring) 1394108a6481SCindy Lu { 1395d96be4c8SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1396d96be4c8SEugenio Pérez 1397d96be4c8SEugenio Pérez if (v->shadow_vqs_enabled) { 1398d96be4c8SEugenio Pérez /* 1399d96be4c8SEugenio Pérez * Device vring base was set at device start. SVQ base is handled by 1400d96be4c8SEugenio Pérez * VirtQueue code. 1401d96be4c8SEugenio Pérez */ 1402d96be4c8SEugenio Pérez return 0; 1403d96be4c8SEugenio Pérez } 1404d96be4c8SEugenio Pérez 1405d96be4c8SEugenio Pérez return vhost_vdpa_set_dev_vring_base(dev, ring); 1406108a6481SCindy Lu } 1407108a6481SCindy Lu 1408108a6481SCindy Lu static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 1409108a6481SCindy Lu struct vhost_vring_state *ring) 1410108a6481SCindy Lu { 14116d0b2226SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1412778e67deSLaurent Vivier int ret; 1413778e67deSLaurent Vivier 14146d0b2226SEugenio Pérez if (v->shadow_vqs_enabled) { 14152fdac348SEugenio Pérez ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index); 14166d0b2226SEugenio Pérez return 0; 14176d0b2226SEugenio Pérez } 14186d0b2226SEugenio Pérez 1419b6662cb7SEugenio Pérez if (!v->suspended) { 1420b6662cb7SEugenio Pérez /* 1421b6662cb7SEugenio Pérez * Cannot trust in value returned by device, let vhost recover used 1422b6662cb7SEugenio Pérez * idx from guest. 1423b6662cb7SEugenio Pérez */ 1424b6662cb7SEugenio Pérez return -1; 1425b6662cb7SEugenio Pérez } 1426b6662cb7SEugenio Pérez 1427778e67deSLaurent Vivier ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 1428778e67deSLaurent Vivier trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); 1429778e67deSLaurent Vivier return ret; 1430108a6481SCindy Lu } 1431108a6481SCindy Lu 1432108a6481SCindy Lu static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 1433108a6481SCindy Lu struct vhost_vring_file *file) 1434108a6481SCindy Lu { 1435dff4426fSEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1436dff4426fSEugenio Pérez int vdpa_idx = file->index - dev->vq_index; 1437dff4426fSEugenio Pérez 1438dff4426fSEugenio Pérez if (v->shadow_vqs_enabled) { 1439dff4426fSEugenio Pérez VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1440dff4426fSEugenio Pérez vhost_svq_set_svq_kick_fd(svq, file->fd); 1441dff4426fSEugenio Pérez return 0; 1442dff4426fSEugenio Pérez } else { 1443dff4426fSEugenio Pérez return vhost_vdpa_set_vring_dev_kick(dev, file); 1444dff4426fSEugenio Pérez } 1445108a6481SCindy Lu } 1446108a6481SCindy Lu 1447108a6481SCindy Lu static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 1448108a6481SCindy Lu struct vhost_vring_file *file) 1449108a6481SCindy Lu { 1450a8ac8858SEugenio Pérez struct vhost_vdpa *v = dev->opaque; 1451a8ac8858SEugenio Pérez int vdpa_idx = file->index - dev->vq_index; 1452a8ac8858SEugenio Pérez VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1453a8ac8858SEugenio Pérez 1454b2765243SEugenio Pérez /* Remember last call fd because we can switch to SVQ anytime. */ 1455a8ac8858SEugenio Pérez vhost_svq_set_svq_call_fd(svq, file->fd); 1456b2765243SEugenio Pérez if (v->shadow_vqs_enabled) { 1457a8ac8858SEugenio Pérez return 0; 1458a8ac8858SEugenio Pérez } 1459b2765243SEugenio Pérez 1460b2765243SEugenio Pérez return vhost_vdpa_set_vring_dev_call(dev, file); 1461108a6481SCindy Lu } 1462108a6481SCindy Lu 1463108a6481SCindy Lu static int vhost_vdpa_get_features(struct vhost_dev *dev, 1464108a6481SCindy Lu uint64_t *features) 1465108a6481SCindy Lu { 146612a195faSEugenio Pérez int ret = vhost_vdpa_get_dev_features(dev, features); 1467778e67deSLaurent Vivier 1468ab7337e3SEugenio Pérez if (ret == 0) { 146912a195faSEugenio Pérez /* Add SVQ logging capabilities */ 147012a195faSEugenio Pérez *features |= BIT_ULL(VHOST_F_LOG_ALL); 147112a195faSEugenio Pérez } 147212a195faSEugenio Pérez 1473778e67deSLaurent Vivier return ret; 1474108a6481SCindy Lu } 1475108a6481SCindy Lu 1476108a6481SCindy Lu static int vhost_vdpa_set_owner(struct vhost_dev *dev) 1477108a6481SCindy Lu { 1478d71b0609SSi-Wei Liu if (!vhost_vdpa_first_dev(dev)) { 14794d191cfdSJason Wang return 0; 14804d191cfdSJason Wang } 14814d191cfdSJason Wang 1482778e67deSLaurent Vivier trace_vhost_vdpa_set_owner(dev); 1483108a6481SCindy Lu return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 1484108a6481SCindy Lu } 1485108a6481SCindy Lu 1486108a6481SCindy Lu static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 1487108a6481SCindy Lu struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 1488108a6481SCindy Lu { 1489108a6481SCindy Lu assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 1490108a6481SCindy Lu addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 1491108a6481SCindy Lu addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 1492108a6481SCindy Lu addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 1493778e67deSLaurent Vivier trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, 1494778e67deSLaurent Vivier addr->avail_user_addr, addr->used_user_addr); 1495108a6481SCindy Lu return 0; 1496108a6481SCindy Lu } 1497108a6481SCindy Lu 1498108a6481SCindy Lu static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 1499108a6481SCindy Lu { 1500108a6481SCindy Lu return true; 1501108a6481SCindy Lu } 1502108a6481SCindy Lu 1503108a6481SCindy Lu const VhostOps vdpa_ops = { 1504108a6481SCindy Lu .backend_type = VHOST_BACKEND_TYPE_VDPA, 1505108a6481SCindy Lu .vhost_backend_init = vhost_vdpa_init, 1506108a6481SCindy Lu .vhost_backend_cleanup = vhost_vdpa_cleanup, 1507108a6481SCindy Lu .vhost_set_log_base = vhost_vdpa_set_log_base, 1508108a6481SCindy Lu .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 1509108a6481SCindy Lu .vhost_set_vring_num = vhost_vdpa_set_vring_num, 1510108a6481SCindy Lu .vhost_set_vring_base = vhost_vdpa_set_vring_base, 1511108a6481SCindy Lu .vhost_get_vring_base = vhost_vdpa_get_vring_base, 1512108a6481SCindy Lu .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 1513108a6481SCindy Lu .vhost_set_vring_call = vhost_vdpa_set_vring_call, 1514108a6481SCindy Lu .vhost_get_features = vhost_vdpa_get_features, 1515a5bd0580SJason Wang .vhost_set_backend_cap = vhost_vdpa_set_backend_cap, 1516108a6481SCindy Lu .vhost_set_owner = vhost_vdpa_set_owner, 1517108a6481SCindy Lu .vhost_set_vring_endian = NULL, 1518108a6481SCindy Lu .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 1519108a6481SCindy Lu .vhost_set_mem_table = vhost_vdpa_set_mem_table, 1520108a6481SCindy Lu .vhost_set_features = vhost_vdpa_set_features, 1521108a6481SCindy Lu .vhost_reset_device = vhost_vdpa_reset_device, 1522108a6481SCindy Lu .vhost_get_vq_index = vhost_vdpa_get_vq_index, 1523108a6481SCindy Lu .vhost_get_config = vhost_vdpa_get_config, 1524108a6481SCindy Lu .vhost_set_config = vhost_vdpa_set_config, 1525108a6481SCindy Lu .vhost_requires_shm_log = NULL, 1526108a6481SCindy Lu .vhost_migration_done = NULL, 1527108a6481SCindy Lu .vhost_net_set_mtu = NULL, 1528108a6481SCindy Lu .vhost_set_iotlb_callback = NULL, 1529108a6481SCindy Lu .vhost_send_device_iotlb_msg = NULL, 1530108a6481SCindy Lu .vhost_dev_start = vhost_vdpa_dev_start, 1531108a6481SCindy Lu .vhost_get_device_id = vhost_vdpa_get_device_id, 1532108a6481SCindy Lu .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 1533108a6481SCindy Lu .vhost_force_iommu = vhost_vdpa_force_iommu, 1534259f3accSCindy Lu .vhost_set_config_call = vhost_vdpa_set_config_call, 1535c3716f26SEugenio Pérez .vhost_reset_status = vhost_vdpa_reset_status, 1536108a6481SCindy Lu }; 1537