1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-device.h" 29 #include "hw/vfio/pci.h" 30 #include "system/address-spaces.h" 31 #include "system/memory.h" 32 #include "system/ram_addr.h" 33 #include "hw/hw.h" 34 #include "qemu/error-report.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/range.h" 37 #include "system/kvm.h" 38 #include "system/reset.h" 39 #include "system/runstate.h" 40 #include "trace.h" 41 #include "qapi/error.h" 42 #include "migration/misc.h" 43 #include "migration/qemu-file.h" 44 #include "system/tcg.h" 45 #include "system/tpm.h" 46 #include "vfio-migration-internal.h" 47 #include "vfio-helpers.h" 48 #include "vfio-listener.h" 49 50 /* 51 * Device state interfaces 52 */ 53 54 55 static bool vfio_log_sync_needed(const VFIOContainerBase *bcontainer) 56 { 57 VFIODevice *vbasedev; 58 59 if (!vfio_container_dirty_tracking_is_started(bcontainer)) { 60 return false; 61 } 62 63 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 64 VFIOMigration *migration = vbasedev->migration; 65 66 if (!migration) { 67 return false; 68 } 69 70 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 71 (vfio_device_state_is_running(vbasedev) || 72 vfio_device_state_is_precopy(vbasedev))) { 73 return false; 74 } 75 } 76 return true; 77 } 78 79 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 80 { 81 return (!memory_region_is_ram(section->mr) && 82 !memory_region_is_iommu(section->mr)) || 83 memory_region_is_protected(section->mr) || 84 /* 85 * Sizing an enabled 64-bit BAR can cause spurious mappings to 86 * addresses in the upper part of the 64-bit address space. These 87 * are never accessed by the CPU and beyond the address width of 88 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 89 */ 90 section->offset_within_address_space & (1ULL << 63); 91 } 92 93 /* 94 * Called with rcu_read_lock held. 95 * The returned MemoryRegion must not be accessed after calling rcu_read_unlock. 96 */ 97 static MemoryRegion *vfio_translate_iotlb(IOMMUTLBEntry *iotlb, hwaddr *xlat_p, 98 Error **errp) 99 { 100 MemoryRegion *mr; 101 102 mr = memory_translate_iotlb(iotlb, xlat_p, errp); 103 if (mr && memory_region_has_ram_discard_manager(mr)) { 104 /* 105 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 106 * pages will remain pinned inside vfio until unmapped, resulting in a 107 * higher memory consumption than expected. If memory would get 108 * populated again later, there would be an inconsistency between pages 109 * pinned by vfio and pages seen by QEMU. This is the case until 110 * unmapped from the IOMMU (e.g., during device reset). 111 * 112 * With malicious guests, we really only care about pinning more memory 113 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 114 * exceeded and can be used to mitigate this problem. 115 */ 116 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 117 " RAM (e.g., virtio-mem) works, however, malicious" 118 " guests can trigger pinning of more memory than" 119 " intended via an IOMMU. It's possible to mitigate " 120 " by setting/adjusting RLIMIT_MEMLOCK."); 121 } 122 return mr; 123 } 124 125 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 126 { 127 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 128 VFIOContainerBase *bcontainer = giommu->bcontainer; 129 hwaddr iova = iotlb->iova + giommu->iommu_offset; 130 MemoryRegion *mr; 131 hwaddr xlat; 132 void *vaddr; 133 int ret; 134 Error *local_err = NULL; 135 136 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 137 iova, iova + iotlb->addr_mask); 138 139 if (iotlb->target_as != &address_space_memory) { 140 error_setg(&local_err, 141 "Wrong target AS \"%s\", only system memory is allowed", 142 iotlb->target_as->name ? iotlb->target_as->name : "none"); 143 if (migration_is_running()) { 144 migration_file_set_error(-EINVAL, local_err); 145 } else { 146 error_report_err(local_err); 147 } 148 return; 149 } 150 151 rcu_read_lock(); 152 153 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 154 bool read_only; 155 156 mr = vfio_translate_iotlb(iotlb, &xlat, &local_err); 157 if (!mr) { 158 error_report_err(local_err); 159 goto out; 160 } 161 vaddr = memory_region_get_ram_ptr(mr) + xlat; 162 read_only = !(iotlb->perm & IOMMU_WO) || mr->readonly; 163 164 /* 165 * vaddr is only valid until rcu_read_unlock(). But after 166 * vfio_dma_map has set up the mapping the pages will be 167 * pinned by the kernel. This makes sure that the RAM backend 168 * of vaddr will always be there, even if the memory object is 169 * destroyed and its backing memory munmap-ed. 170 */ 171 ret = vfio_container_dma_map(bcontainer, iova, 172 iotlb->addr_mask + 1, vaddr, 173 read_only, mr); 174 if (ret) { 175 error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 176 "0x%"HWADDR_PRIx", %p) = %d (%s)", 177 bcontainer, iova, 178 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 179 } 180 } else { 181 ret = vfio_container_dma_unmap(bcontainer, iova, 182 iotlb->addr_mask + 1, iotlb, false); 183 if (ret) { 184 error_setg(&local_err, 185 "vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 186 "0x%"HWADDR_PRIx") = %d (%s)", 187 bcontainer, iova, 188 iotlb->addr_mask + 1, ret, strerror(-ret)); 189 if (migration_is_running()) { 190 migration_file_set_error(ret, local_err); 191 } else { 192 error_report_err(local_err); 193 } 194 } 195 } 196 out: 197 rcu_read_unlock(); 198 } 199 200 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 201 MemoryRegionSection *section) 202 { 203 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 204 listener); 205 VFIOContainerBase *bcontainer = vrdl->bcontainer; 206 const hwaddr size = int128_get64(section->size); 207 const hwaddr iova = section->offset_within_address_space; 208 int ret; 209 210 /* Unmap with a single call. */ 211 ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL, false); 212 if (ret) { 213 error_report("%s: vfio_container_dma_unmap() failed: %s", __func__, 214 strerror(-ret)); 215 } 216 } 217 218 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 219 MemoryRegionSection *section) 220 { 221 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 222 listener); 223 VFIOContainerBase *bcontainer = vrdl->bcontainer; 224 const hwaddr end = section->offset_within_region + 225 int128_get64(section->size); 226 hwaddr start, next, iova; 227 void *vaddr; 228 int ret; 229 230 /* 231 * Map in (aligned within memory region) minimum granularity, so we can 232 * unmap in minimum granularity later. 233 */ 234 for (start = section->offset_within_region; start < end; start = next) { 235 next = ROUND_UP(start + 1, vrdl->granularity); 236 next = MIN(next, end); 237 238 iova = start - section->offset_within_region + 239 section->offset_within_address_space; 240 vaddr = memory_region_get_ram_ptr(section->mr) + start; 241 242 ret = vfio_container_dma_map(bcontainer, iova, next - start, 243 vaddr, section->readonly, section->mr); 244 if (ret) { 245 /* Rollback */ 246 vfio_ram_discard_notify_discard(rdl, section); 247 return ret; 248 } 249 } 250 return 0; 251 } 252 253 static void vfio_ram_discard_register_listener(VFIOContainerBase *bcontainer, 254 MemoryRegionSection *section) 255 { 256 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 257 int target_page_size = qemu_target_page_size(); 258 VFIORamDiscardListener *vrdl; 259 260 /* Ignore some corner cases not relevant in practice. */ 261 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, target_page_size)); 262 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 263 target_page_size)); 264 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), target_page_size)); 265 266 vrdl = g_new0(VFIORamDiscardListener, 1); 267 vrdl->bcontainer = bcontainer; 268 vrdl->mr = section->mr; 269 vrdl->offset_within_address_space = section->offset_within_address_space; 270 vrdl->size = int128_get64(section->size); 271 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 272 section->mr); 273 274 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 275 g_assert(bcontainer->pgsizes && 276 vrdl->granularity >= 1ULL << ctz64(bcontainer->pgsizes)); 277 278 ram_discard_listener_init(&vrdl->listener, 279 vfio_ram_discard_notify_populate, 280 vfio_ram_discard_notify_discard, true); 281 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 282 QLIST_INSERT_HEAD(&bcontainer->vrdl_list, vrdl, next); 283 284 /* 285 * Sanity-check if we have a theoretically problematic setup where we could 286 * exceed the maximum number of possible DMA mappings over time. We assume 287 * that each mapped section in the same address space as a RamDiscardManager 288 * section consumes exactly one DMA mapping, with the exception of 289 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 290 * in the same address space as RamDiscardManager sections. 291 * 292 * We assume that each section in the address space consumes one memslot. 293 * We take the number of KVM memory slots as a best guess for the maximum 294 * number of sections in the address space we could have over time, 295 * also consuming DMA mappings. 296 */ 297 if (bcontainer->dma_max_mappings) { 298 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 299 300 #ifdef CONFIG_KVM 301 if (kvm_enabled()) { 302 max_memslots = kvm_get_max_memslots(); 303 } 304 #endif 305 306 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 307 hwaddr start, end; 308 309 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 310 vrdl->granularity); 311 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 312 vrdl->granularity); 313 vrdl_mappings += (end - start) / vrdl->granularity; 314 vrdl_count++; 315 } 316 317 if (vrdl_mappings + max_memslots - vrdl_count > 318 bcontainer->dma_max_mappings) { 319 warn_report("%s: possibly running out of DMA mappings. E.g., try" 320 " increasing the 'block-size' of virtio-mem devies." 321 " Maximum possible DMA mappings: %d, Maximum possible" 322 " memslots: %d", __func__, bcontainer->dma_max_mappings, 323 max_memslots); 324 } 325 } 326 } 327 328 static void vfio_ram_discard_unregister_listener(VFIOContainerBase *bcontainer, 329 MemoryRegionSection *section) 330 { 331 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 332 VFIORamDiscardListener *vrdl = NULL; 333 334 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 335 if (vrdl->mr == section->mr && 336 vrdl->offset_within_address_space == 337 section->offset_within_address_space) { 338 break; 339 } 340 } 341 342 if (!vrdl) { 343 hw_error("vfio: Trying to unregister missing RAM discard listener"); 344 } 345 346 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 347 QLIST_REMOVE(vrdl, next); 348 g_free(vrdl); 349 } 350 351 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 352 { 353 MemoryRegion *mr = section->mr; 354 355 if (!TPM_IS_CRB(mr->owner)) { 356 return false; 357 } 358 359 /* this is a known safe misaligned region, just trace for debug purpose */ 360 trace_vfio_known_safe_misalignment(memory_region_name(mr), 361 section->offset_within_address_space, 362 section->offset_within_region, 363 qemu_real_host_page_size()); 364 return true; 365 } 366 367 static bool vfio_listener_valid_section(MemoryRegionSection *section, 368 const char *name) 369 { 370 if (vfio_listener_skipped_section(section)) { 371 trace_vfio_listener_region_skip(name, 372 section->offset_within_address_space, 373 section->offset_within_address_space + 374 int128_get64(int128_sub(section->size, int128_one()))); 375 return false; 376 } 377 378 if (unlikely((section->offset_within_address_space & 379 ~qemu_real_host_page_mask()) != 380 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 381 if (!vfio_known_safe_misalignment(section)) { 382 error_report("%s received unaligned region %s iova=0x%"PRIx64 383 " offset_within_region=0x%"PRIx64 384 " qemu_real_host_page_size=0x%"PRIxPTR, 385 __func__, memory_region_name(section->mr), 386 section->offset_within_address_space, 387 section->offset_within_region, 388 qemu_real_host_page_size()); 389 } 390 return false; 391 } 392 393 return true; 394 } 395 396 static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer, 397 MemoryRegionSection *section, 398 hwaddr *out_iova, hwaddr *out_end, 399 Int128 *out_llend) 400 { 401 Int128 llend; 402 hwaddr iova; 403 404 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 405 llend = int128_make64(section->offset_within_address_space); 406 llend = int128_add(llend, section->size); 407 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 408 409 if (int128_ge(int128_make64(iova), llend)) { 410 return false; 411 } 412 413 *out_iova = iova; 414 *out_end = int128_get64(int128_sub(llend, int128_one())); 415 if (out_llend) { 416 *out_llend = llend; 417 } 418 return true; 419 } 420 421 static void vfio_listener_begin(MemoryListener *listener) 422 { 423 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 424 listener); 425 void (*listener_begin)(VFIOContainerBase *bcontainer); 426 427 listener_begin = VFIO_IOMMU_GET_CLASS(bcontainer)->listener_begin; 428 429 if (listener_begin) { 430 listener_begin(bcontainer); 431 } 432 } 433 434 static void vfio_listener_commit(MemoryListener *listener) 435 { 436 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 437 listener); 438 void (*listener_commit)(VFIOContainerBase *bcontainer); 439 440 listener_commit = VFIO_IOMMU_GET_CLASS(bcontainer)->listener_commit; 441 442 if (listener_commit) { 443 listener_commit(bcontainer); 444 } 445 } 446 447 static void vfio_device_error_append(VFIODevice *vbasedev, Error **errp) 448 { 449 /* 450 * MMIO region mapping failures are not fatal but in this case PCI 451 * peer-to-peer transactions are broken. 452 */ 453 if (vbasedev && vbasedev->type == VFIO_DEVICE_TYPE_PCI) { 454 error_append_hint(errp, "%s: PCI peer-to-peer transactions " 455 "on BARs are not supported.\n", vbasedev->name); 456 } 457 } 458 459 VFIORamDiscardListener *vfio_find_ram_discard_listener( 460 VFIOContainerBase *bcontainer, MemoryRegionSection *section) 461 { 462 VFIORamDiscardListener *vrdl = NULL; 463 464 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 465 if (vrdl->mr == section->mr && 466 vrdl->offset_within_address_space == 467 section->offset_within_address_space) { 468 break; 469 } 470 } 471 472 if (!vrdl) { 473 hw_error("vfio: Trying to sync missing RAM discard listener"); 474 /* does not return */ 475 } 476 return vrdl; 477 } 478 479 static void vfio_listener_region_add(MemoryListener *listener, 480 MemoryRegionSection *section) 481 { 482 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 483 listener); 484 vfio_container_region_add(bcontainer, section, false); 485 } 486 487 void vfio_container_region_add(VFIOContainerBase *bcontainer, 488 MemoryRegionSection *section, 489 bool cpr_remap) 490 { 491 hwaddr iova, end; 492 Int128 llend, llsize; 493 void *vaddr; 494 int ret; 495 Error *err = NULL; 496 497 if (!vfio_listener_valid_section(section, "region_add")) { 498 return; 499 } 500 501 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 502 &llend)) { 503 if (memory_region_is_ram_device(section->mr)) { 504 trace_vfio_listener_region_add_no_dma_map( 505 memory_region_name(section->mr), 506 section->offset_within_address_space, 507 int128_getlo(section->size), 508 qemu_real_host_page_size()); 509 } 510 return; 511 } 512 513 /* PPC64/pseries machine only */ 514 if (!vfio_container_add_section_window(bcontainer, section, &err)) { 515 goto mmio_dma_error; 516 } 517 518 memory_region_ref(section->mr); 519 520 if (memory_region_is_iommu(section->mr)) { 521 VFIOGuestIOMMU *giommu; 522 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 523 int iommu_idx; 524 525 trace_vfio_listener_region_add_iommu(section->mr->name, iova, end); 526 527 if (cpr_remap) { 528 vfio_cpr_giommu_remap(bcontainer, section); 529 } 530 531 /* 532 * FIXME: For VFIO iommu types which have KVM acceleration to 533 * avoid bouncing all map/unmaps through qemu this way, this 534 * would be the right place to wire that up (tell the KVM 535 * device emulation the VFIO iommu handles to use). 536 */ 537 giommu = g_malloc0(sizeof(*giommu)); 538 giommu->iommu_mr = iommu_mr; 539 giommu->iommu_offset = section->offset_within_address_space - 540 section->offset_within_region; 541 giommu->bcontainer = bcontainer; 542 llend = int128_add(int128_make64(section->offset_within_region), 543 section->size); 544 llend = int128_sub(llend, int128_one()); 545 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 546 MEMTXATTRS_UNSPECIFIED); 547 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 548 IOMMU_NOTIFIER_IOTLB_EVENTS, 549 section->offset_within_region, 550 int128_get64(llend), 551 iommu_idx); 552 553 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 554 &err); 555 if (ret) { 556 g_free(giommu); 557 goto fail; 558 } 559 QLIST_INSERT_HEAD(&bcontainer->giommu_list, giommu, giommu_next); 560 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 561 562 return; 563 } 564 565 /* Here we assume that memory_region_is_ram(section->mr)==true */ 566 567 /* 568 * For RAM memory regions with a RamDiscardManager, we only want to map the 569 * actually populated parts - and update the mapping whenever we're notified 570 * about changes. 571 */ 572 if (memory_region_has_ram_discard_manager(section->mr)) { 573 if (!cpr_remap) { 574 vfio_ram_discard_register_listener(bcontainer, section); 575 } else if (!vfio_cpr_ram_discard_register_listener(bcontainer, 576 section)) { 577 goto fail; 578 } 579 return; 580 } 581 582 vaddr = memory_region_get_ram_ptr(section->mr) + 583 section->offset_within_region + 584 (iova - section->offset_within_address_space); 585 586 trace_vfio_listener_region_add_ram(iova, end, vaddr); 587 588 llsize = int128_sub(llend, int128_make64(iova)); 589 590 if (memory_region_is_ram_device(section->mr)) { 591 hwaddr pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 592 593 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 594 trace_vfio_listener_region_add_no_dma_map( 595 memory_region_name(section->mr), 596 section->offset_within_address_space, 597 int128_getlo(section->size), 598 pgmask + 1); 599 return; 600 } 601 } 602 603 ret = vfio_container_dma_map(bcontainer, iova, int128_get64(llsize), 604 vaddr, section->readonly, section->mr); 605 if (ret) { 606 error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 607 "0x%"HWADDR_PRIx", %p) = %d (%s)", 608 bcontainer, iova, int128_get64(llsize), vaddr, ret, 609 strerror(-ret)); 610 mmio_dma_error: 611 if (memory_region_is_ram_device(section->mr)) { 612 /* Allow unexpected mappings not to be fatal for RAM devices */ 613 VFIODevice *vbasedev = 614 vfio_get_vfio_device(memory_region_owner(section->mr)); 615 vfio_device_error_append(vbasedev, &err); 616 warn_report_err_once(err); 617 return; 618 } 619 goto fail; 620 } 621 622 return; 623 624 fail: 625 if (!bcontainer->initialized) { 626 /* 627 * At machine init time or when the device is attached to the 628 * VM, store the first error in the container so we can 629 * gracefully fail the device realize routine. 630 */ 631 if (!bcontainer->error) { 632 error_propagate_prepend(&bcontainer->error, err, 633 "Region %s: ", 634 memory_region_name(section->mr)); 635 } else { 636 error_free(err); 637 } 638 } else { 639 /* 640 * At runtime, there's not much we can do other than throw a 641 * hardware error. 642 */ 643 error_report_err(err); 644 hw_error("vfio: DMA mapping failed, unable to continue"); 645 } 646 } 647 648 static void vfio_listener_region_del(MemoryListener *listener, 649 MemoryRegionSection *section) 650 { 651 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 652 listener); 653 hwaddr iova, end; 654 Int128 llend, llsize; 655 int ret; 656 bool try_unmap = true; 657 658 if (!vfio_listener_valid_section(section, "region_del")) { 659 return; 660 } 661 662 if (memory_region_is_iommu(section->mr)) { 663 VFIOGuestIOMMU *giommu; 664 665 trace_vfio_listener_region_del_iommu(section->mr->name); 666 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 667 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 668 giommu->n.start == section->offset_within_region) { 669 memory_region_unregister_iommu_notifier(section->mr, 670 &giommu->n); 671 QLIST_REMOVE(giommu, giommu_next); 672 g_free(giommu); 673 break; 674 } 675 } 676 677 /* 678 * FIXME: We assume the one big unmap below is adequate to 679 * remove any individual page mappings in the IOMMU which 680 * might have been copied into VFIO. This works for a page table 681 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 682 * That may not be true for all IOMMU types. 683 */ 684 } 685 686 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 687 &llend)) { 688 return; 689 } 690 691 llsize = int128_sub(llend, int128_make64(iova)); 692 693 trace_vfio_listener_region_del(iova, end); 694 695 if (memory_region_is_ram_device(section->mr)) { 696 hwaddr pgmask; 697 698 pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 699 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 700 } else if (memory_region_has_ram_discard_manager(section->mr)) { 701 vfio_ram_discard_unregister_listener(bcontainer, section); 702 /* Unregistering will trigger an unmap. */ 703 try_unmap = false; 704 } 705 706 if (try_unmap) { 707 bool unmap_all = false; 708 709 if (int128_eq(llsize, int128_2_64())) { 710 unmap_all = true; 711 llsize = int128_zero(); 712 } 713 ret = vfio_container_dma_unmap(bcontainer, iova, int128_get64(llsize), 714 NULL, unmap_all); 715 if (ret) { 716 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 717 "0x%"HWADDR_PRIx") = %d (%s)", 718 bcontainer, iova, int128_get64(llsize), ret, 719 strerror(-ret)); 720 } 721 } 722 723 memory_region_unref(section->mr); 724 725 /* PPC64/pseries machine only */ 726 vfio_container_del_section_window(bcontainer, section); 727 } 728 729 typedef struct VFIODirtyRanges { 730 hwaddr min32; 731 hwaddr max32; 732 hwaddr min64; 733 hwaddr max64; 734 hwaddr minpci64; 735 hwaddr maxpci64; 736 } VFIODirtyRanges; 737 738 typedef struct VFIODirtyRangesListener { 739 VFIOContainerBase *bcontainer; 740 VFIODirtyRanges ranges; 741 MemoryListener listener; 742 } VFIODirtyRangesListener; 743 744 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 745 VFIOContainerBase *bcontainer) 746 { 747 VFIOPCIDevice *pcidev; 748 VFIODevice *vbasedev; 749 Object *owner; 750 751 owner = memory_region_owner(section->mr); 752 753 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 754 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 755 continue; 756 } 757 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 758 if (OBJECT(pcidev) == owner) { 759 return true; 760 } 761 } 762 763 return false; 764 } 765 766 static void vfio_dirty_tracking_update_range(VFIODirtyRanges *range, 767 hwaddr iova, hwaddr end, 768 bool update_pci) 769 { 770 hwaddr *min, *max; 771 772 /* 773 * The address space passed to the dirty tracker is reduced to three ranges: 774 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 775 * PCI 64-bit hole. 776 * 777 * The underlying reports of dirty will query a sub-interval of each of 778 * these ranges. 779 * 780 * The purpose of the three range handling is to handle known cases of big 781 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 782 * OVMF) which may relocate the pci-hole64 to the end of the address space. 783 * The latter would otherwise generate large ranges for tracking, stressing 784 * the limits of supported hardware. The pci-hole32 will always be below 4G 785 * (overlapping or not) so it doesn't need special handling and is part of 786 * the 32-bit range. 787 * 788 * The alternative would be an IOVATree but that has a much bigger runtime 789 * overhead and unnecessary complexity. 790 */ 791 if (update_pci && iova >= UINT32_MAX) { 792 min = &range->minpci64; 793 max = &range->maxpci64; 794 } else { 795 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 796 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 797 } 798 if (*min > iova) { 799 *min = iova; 800 } 801 if (*max < end) { 802 *max = end; 803 } 804 805 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 806 } 807 808 static void vfio_dirty_tracking_update(MemoryListener *listener, 809 MemoryRegionSection *section) 810 { 811 VFIODirtyRangesListener *dirty = 812 container_of(listener, VFIODirtyRangesListener, listener); 813 hwaddr iova, end; 814 815 if (!vfio_listener_valid_section(section, "tracking_update") || 816 !vfio_get_section_iova_range(dirty->bcontainer, section, 817 &iova, &end, NULL)) { 818 return; 819 } 820 821 vfio_dirty_tracking_update_range(&dirty->ranges, iova, end, 822 vfio_section_is_vfio_pci(section, dirty->bcontainer)); 823 } 824 825 static const MemoryListener vfio_dirty_tracking_listener = { 826 .name = "vfio-tracking", 827 .region_add = vfio_dirty_tracking_update, 828 }; 829 830 static void vfio_dirty_tracking_init(VFIOContainerBase *bcontainer, 831 VFIODirtyRanges *ranges) 832 { 833 VFIODirtyRangesListener dirty; 834 835 memset(&dirty, 0, sizeof(dirty)); 836 dirty.ranges.min32 = UINT32_MAX; 837 dirty.ranges.min64 = UINT64_MAX; 838 dirty.ranges.minpci64 = UINT64_MAX; 839 dirty.listener = vfio_dirty_tracking_listener; 840 dirty.bcontainer = bcontainer; 841 842 memory_listener_register(&dirty.listener, 843 bcontainer->space->as); 844 845 *ranges = dirty.ranges; 846 847 /* 848 * The memory listener is synchronous, and used to calculate the range 849 * to dirty tracking. Unregister it after we are done as we are not 850 * interested in any follow-up updates. 851 */ 852 memory_listener_unregister(&dirty.listener); 853 } 854 855 static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer) 856 { 857 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 858 sizeof(uint64_t))] = {}; 859 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 860 VFIODevice *vbasedev; 861 862 feature->argsz = sizeof(buf); 863 feature->flags = VFIO_DEVICE_FEATURE_SET | 864 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 865 866 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 867 int ret; 868 869 if (!vbasedev->dirty_tracking) { 870 continue; 871 } 872 873 ret = vbasedev->io_ops->device_feature(vbasedev, feature); 874 875 if (ret != 0) { 876 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 877 vbasedev->name, -ret, strerror(-ret)); 878 } 879 vbasedev->dirty_tracking = false; 880 } 881 } 882 883 static struct vfio_device_feature * 884 vfio_device_feature_dma_logging_start_create(VFIOContainerBase *bcontainer, 885 VFIODirtyRanges *tracking) 886 { 887 struct vfio_device_feature *feature; 888 size_t feature_size; 889 struct vfio_device_feature_dma_logging_control *control; 890 struct vfio_device_feature_dma_logging_range *ranges; 891 892 feature_size = sizeof(struct vfio_device_feature) + 893 sizeof(struct vfio_device_feature_dma_logging_control); 894 feature = g_try_malloc0(feature_size); 895 if (!feature) { 896 errno = ENOMEM; 897 return NULL; 898 } 899 feature->argsz = feature_size; 900 feature->flags = VFIO_DEVICE_FEATURE_SET | 901 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 902 903 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 904 control->page_size = qemu_real_host_page_size(); 905 906 /* 907 * DMA logging uAPI guarantees to support at least a number of ranges that 908 * fits into a single host kernel base page. 909 */ 910 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 911 !!tracking->maxpci64; 912 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 913 control->num_ranges); 914 if (!ranges) { 915 g_free(feature); 916 errno = ENOMEM; 917 918 return NULL; 919 } 920 921 control->ranges = (uintptr_t)ranges; 922 if (tracking->max32) { 923 ranges->iova = tracking->min32; 924 ranges->length = (tracking->max32 - tracking->min32) + 1; 925 ranges++; 926 } 927 if (tracking->max64) { 928 ranges->iova = tracking->min64; 929 ranges->length = (tracking->max64 - tracking->min64) + 1; 930 ranges++; 931 } 932 if (tracking->maxpci64) { 933 ranges->iova = tracking->minpci64; 934 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 935 } 936 937 trace_vfio_device_dirty_tracking_start(control->num_ranges, 938 tracking->min32, tracking->max32, 939 tracking->min64, tracking->max64, 940 tracking->minpci64, tracking->maxpci64); 941 942 return feature; 943 } 944 945 static void vfio_device_feature_dma_logging_start_destroy( 946 struct vfio_device_feature *feature) 947 { 948 struct vfio_device_feature_dma_logging_control *control = 949 (struct vfio_device_feature_dma_logging_control *)feature->data; 950 struct vfio_device_feature_dma_logging_range *ranges = 951 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 952 953 g_free(ranges); 954 g_free(feature); 955 } 956 957 static bool vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer, 958 Error **errp) 959 { 960 struct vfio_device_feature *feature; 961 VFIODirtyRanges ranges; 962 VFIODevice *vbasedev; 963 int ret = 0; 964 965 vfio_dirty_tracking_init(bcontainer, &ranges); 966 feature = vfio_device_feature_dma_logging_start_create(bcontainer, 967 &ranges); 968 if (!feature) { 969 error_setg_errno(errp, errno, "Failed to prepare DMA logging"); 970 return false; 971 } 972 973 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 974 if (vbasedev->dirty_tracking) { 975 continue; 976 } 977 978 ret = vbasedev->io_ops->device_feature(vbasedev, feature); 979 if (ret) { 980 error_setg_errno(errp, -ret, "%s: Failed to start DMA logging", 981 vbasedev->name); 982 goto out; 983 } 984 vbasedev->dirty_tracking = true; 985 } 986 987 out: 988 if (ret) { 989 vfio_devices_dma_logging_stop(bcontainer); 990 } 991 992 vfio_device_feature_dma_logging_start_destroy(feature); 993 994 return ret == 0; 995 } 996 997 static bool vfio_listener_log_global_start(MemoryListener *listener, 998 Error **errp) 999 { 1000 ERRP_GUARD(); 1001 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1002 listener); 1003 bool ret; 1004 1005 if (vfio_container_devices_dirty_tracking_is_supported(bcontainer)) { 1006 ret = vfio_devices_dma_logging_start(bcontainer, errp); 1007 } else { 1008 ret = vfio_container_set_dirty_page_tracking(bcontainer, true, errp) == 0; 1009 } 1010 1011 if (!ret) { 1012 error_prepend(errp, "vfio: Could not start dirty page tracking - "); 1013 } 1014 return ret; 1015 } 1016 1017 static void vfio_listener_log_global_stop(MemoryListener *listener) 1018 { 1019 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1020 listener); 1021 Error *local_err = NULL; 1022 int ret = 0; 1023 1024 if (vfio_container_devices_dirty_tracking_is_supported(bcontainer)) { 1025 vfio_devices_dma_logging_stop(bcontainer); 1026 } else { 1027 ret = vfio_container_set_dirty_page_tracking(bcontainer, false, 1028 &local_err); 1029 } 1030 1031 if (ret) { 1032 error_prepend(&local_err, 1033 "vfio: Could not stop dirty page tracking - "); 1034 if (migration_is_running()) { 1035 migration_file_set_error(ret, local_err); 1036 } else { 1037 error_report_err(local_err); 1038 } 1039 } 1040 } 1041 1042 typedef struct { 1043 IOMMUNotifier n; 1044 VFIOGuestIOMMU *giommu; 1045 } vfio_giommu_dirty_notifier; 1046 1047 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1048 { 1049 vfio_giommu_dirty_notifier *gdn = container_of(n, 1050 vfio_giommu_dirty_notifier, n); 1051 VFIOGuestIOMMU *giommu = gdn->giommu; 1052 VFIOContainerBase *bcontainer = giommu->bcontainer; 1053 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1054 ram_addr_t translated_addr; 1055 Error *local_err = NULL; 1056 int ret = -EINVAL; 1057 MemoryRegion *mr; 1058 hwaddr xlat; 1059 1060 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1061 1062 if (iotlb->target_as != &address_space_memory) { 1063 error_setg(&local_err, 1064 "Wrong target AS \"%s\", only system memory is allowed", 1065 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1066 goto out; 1067 } 1068 1069 rcu_read_lock(); 1070 mr = vfio_translate_iotlb(iotlb, &xlat, &local_err); 1071 if (!mr) { 1072 goto out_unlock; 1073 } 1074 translated_addr = memory_region_get_ram_addr(mr) + xlat; 1075 1076 ret = vfio_container_query_dirty_bitmap(bcontainer, iova, iotlb->addr_mask + 1, 1077 translated_addr, &local_err); 1078 if (ret) { 1079 error_prepend(&local_err, 1080 "vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1081 "0x%"HWADDR_PRIx") failed - ", bcontainer, iova, 1082 iotlb->addr_mask + 1); 1083 } 1084 1085 out_unlock: 1086 rcu_read_unlock(); 1087 1088 out: 1089 if (ret) { 1090 if (migration_is_running()) { 1091 migration_file_set_error(ret, local_err); 1092 } else { 1093 error_report_err(local_err); 1094 } 1095 } 1096 } 1097 1098 static int vfio_ram_discard_query_dirty_bitmap(MemoryRegionSection *section, 1099 void *opaque) 1100 { 1101 const hwaddr size = int128_get64(section->size); 1102 const hwaddr iova = section->offset_within_address_space; 1103 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1104 section->offset_within_region; 1105 VFIORamDiscardListener *vrdl = opaque; 1106 Error *local_err = NULL; 1107 int ret; 1108 1109 /* 1110 * Sync the whole mapped region (spanning multiple individual mappings) 1111 * in one go. 1112 */ 1113 ret = vfio_container_query_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr, 1114 &local_err); 1115 if (ret) { 1116 error_report_err(local_err); 1117 } 1118 return ret; 1119 } 1120 1121 static int 1122 vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer, 1123 MemoryRegionSection *section) 1124 { 1125 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1126 VFIORamDiscardListener *vrdl = 1127 vfio_find_ram_discard_listener(bcontainer, section); 1128 1129 /* 1130 * We only want/can synchronize the bitmap for actually mapped parts - 1131 * which correspond to populated parts. Replay all populated parts. 1132 */ 1133 return ram_discard_manager_replay_populated(rdm, section, 1134 vfio_ram_discard_query_dirty_bitmap, 1135 &vrdl); 1136 } 1137 1138 static int vfio_sync_iommu_dirty_bitmap(VFIOContainerBase *bcontainer, 1139 MemoryRegionSection *section) 1140 { 1141 VFIOGuestIOMMU *giommu; 1142 bool found = false; 1143 Int128 llend; 1144 vfio_giommu_dirty_notifier gdn; 1145 int idx; 1146 1147 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 1148 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1149 giommu->n.start == section->offset_within_region) { 1150 found = true; 1151 break; 1152 } 1153 } 1154 1155 if (!found) { 1156 return 0; 1157 } 1158 1159 gdn.giommu = giommu; 1160 idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1161 MEMTXATTRS_UNSPECIFIED); 1162 1163 llend = int128_add(int128_make64(section->offset_within_region), 1164 section->size); 1165 llend = int128_sub(llend, int128_one()); 1166 1167 iommu_notifier_init(&gdn.n, vfio_iommu_map_dirty_notify, IOMMU_NOTIFIER_MAP, 1168 section->offset_within_region, int128_get64(llend), 1169 idx); 1170 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1171 1172 return 0; 1173 } 1174 1175 static int vfio_sync_dirty_bitmap(VFIOContainerBase *bcontainer, 1176 MemoryRegionSection *section, Error **errp) 1177 { 1178 ram_addr_t ram_addr; 1179 1180 if (memory_region_is_iommu(section->mr)) { 1181 return vfio_sync_iommu_dirty_bitmap(bcontainer, section); 1182 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1183 int ret; 1184 1185 ret = vfio_sync_ram_discard_listener_dirty_bitmap(bcontainer, section); 1186 if (ret) { 1187 error_setg(errp, 1188 "Failed to sync dirty bitmap with RAM discard listener"); 1189 } 1190 return ret; 1191 } 1192 1193 ram_addr = memory_region_get_ram_addr(section->mr) + 1194 section->offset_within_region; 1195 1196 return vfio_container_query_dirty_bitmap(bcontainer, 1197 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1198 int128_get64(section->size), ram_addr, errp); 1199 } 1200 1201 static void vfio_listener_log_sync(MemoryListener *listener, 1202 MemoryRegionSection *section) 1203 { 1204 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1205 listener); 1206 int ret; 1207 Error *local_err = NULL; 1208 1209 if (vfio_listener_skipped_section(section)) { 1210 return; 1211 } 1212 1213 if (vfio_log_sync_needed(bcontainer)) { 1214 ret = vfio_sync_dirty_bitmap(bcontainer, section, &local_err); 1215 if (ret) { 1216 if (migration_is_running()) { 1217 migration_file_set_error(ret, local_err); 1218 } else { 1219 error_report_err(local_err); 1220 } 1221 } 1222 } 1223 } 1224 1225 static const MemoryListener vfio_memory_listener = { 1226 .name = "vfio", 1227 .begin = vfio_listener_begin, 1228 .commit = vfio_listener_commit, 1229 .region_add = vfio_listener_region_add, 1230 .region_del = vfio_listener_region_del, 1231 .log_global_start = vfio_listener_log_global_start, 1232 .log_global_stop = vfio_listener_log_global_stop, 1233 .log_sync = vfio_listener_log_sync, 1234 }; 1235 1236 bool vfio_listener_register(VFIOContainerBase *bcontainer, Error **errp) 1237 { 1238 bcontainer->listener = vfio_memory_listener; 1239 memory_listener_register(&bcontainer->listener, bcontainer->space->as); 1240 1241 if (bcontainer->error) { 1242 error_propagate_prepend(errp, bcontainer->error, 1243 "memory listener initialization failed: "); 1244 return false; 1245 } 1246 1247 return true; 1248 } 1249 1250 void vfio_listener_unregister(VFIOContainerBase *bcontainer) 1251 { 1252 memory_listener_unregister(&bcontainer->listener); 1253 } 1254