1 /* 2 * QEMU KVM support 3 * 4 * Copyright IBM, Corp. 2008 5 * Red Hat, Inc. 2008 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * Glauber Costa <gcosta@redhat.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2 or later. 12 * See the COPYING file in the top-level directory. 13 * 14 */ 15 16 #include "qemu/osdep.h" 17 #include <sys/ioctl.h> 18 #include <poll.h> 19 20 #include <linux/kvm.h> 21 22 #include "qemu/atomic.h" 23 #include "qemu/option.h" 24 #include "qemu/config-file.h" 25 #include "qemu/error-report.h" 26 #include "qapi/error.h" 27 #include "hw/pci/msi.h" 28 #include "hw/pci/msix.h" 29 #include "hw/s390x/adapter.h" 30 #include "gdbstub/enums.h" 31 #include "system/kvm_int.h" 32 #include "system/runstate.h" 33 #include "system/cpus.h" 34 #include "system/accel-blocker.h" 35 #include "qemu/bswap.h" 36 #include "exec/tswap.h" 37 #include "system/memory.h" 38 #include "system/ram_addr.h" 39 #include "qemu/event_notifier.h" 40 #include "qemu/main-loop.h" 41 #include "trace.h" 42 #include "hw/irq.h" 43 #include "qapi/visitor.h" 44 #include "qapi/qapi-types-common.h" 45 #include "qapi/qapi-visit-common.h" 46 #include "system/reset.h" 47 #include "qemu/guest-random.h" 48 #include "system/hw_accel.h" 49 #include "kvm-cpus.h" 50 #include "system/dirtylimit.h" 51 #include "qemu/range.h" 52 53 #include "hw/boards.h" 54 #include "system/stats.h" 55 56 /* This check must be after config-host.h is included */ 57 #ifdef CONFIG_EVENTFD 58 #include <sys/eventfd.h> 59 #endif 60 61 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) 62 # define KVM_HAVE_MCE_INJECTION 1 63 #endif 64 65 66 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We 67 * need to use the real host PAGE_SIZE, as that's what KVM will use. 68 */ 69 #ifdef PAGE_SIZE 70 #undef PAGE_SIZE 71 #endif 72 #define PAGE_SIZE qemu_real_host_page_size() 73 74 #ifndef KVM_GUESTDBG_BLOCKIRQ 75 #define KVM_GUESTDBG_BLOCKIRQ 0 76 #endif 77 78 /* Default num of memslots to be allocated when VM starts */ 79 #define KVM_MEMSLOTS_NR_ALLOC_DEFAULT 16 80 /* Default max allowed memslots if kernel reported nothing */ 81 #define KVM_MEMSLOTS_NR_MAX_DEFAULT 32 82 83 struct KVMParkedVcpu { 84 unsigned long vcpu_id; 85 int kvm_fd; 86 QLIST_ENTRY(KVMParkedVcpu) node; 87 }; 88 89 KVMState *kvm_state; 90 bool kvm_kernel_irqchip; 91 bool kvm_split_irqchip; 92 bool kvm_async_interrupts_allowed; 93 bool kvm_halt_in_kernel_allowed; 94 bool kvm_resamplefds_allowed; 95 bool kvm_msi_via_irqfd_allowed; 96 bool kvm_gsi_routing_allowed; 97 bool kvm_gsi_direct_mapping; 98 bool kvm_allowed; 99 bool kvm_readonly_mem_allowed; 100 bool kvm_vm_attributes_allowed; 101 bool kvm_msi_use_devid; 102 bool kvm_pre_fault_memory_supported; 103 static bool kvm_has_guest_debug; 104 static int kvm_sstep_flags; 105 static bool kvm_immediate_exit; 106 static uint64_t kvm_supported_memory_attributes; 107 static bool kvm_guest_memfd_supported; 108 static hwaddr kvm_max_slot_size = ~0; 109 110 static const KVMCapabilityInfo kvm_required_capabilites[] = { 111 KVM_CAP_INFO(USER_MEMORY), 112 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS), 113 KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS), 114 KVM_CAP_INFO(INTERNAL_ERROR_DATA), 115 KVM_CAP_INFO(IOEVENTFD), 116 KVM_CAP_INFO(IOEVENTFD_ANY_LENGTH), 117 KVM_CAP_LAST_INFO 118 }; 119 120 static NotifierList kvm_irqchip_change_notifiers = 121 NOTIFIER_LIST_INITIALIZER(kvm_irqchip_change_notifiers); 122 123 struct KVMResampleFd { 124 int gsi; 125 EventNotifier *resample_event; 126 QLIST_ENTRY(KVMResampleFd) node; 127 }; 128 typedef struct KVMResampleFd KVMResampleFd; 129 130 /* 131 * Only used with split irqchip where we need to do the resample fd 132 * kick for the kernel from userspace. 133 */ 134 static QLIST_HEAD(, KVMResampleFd) kvm_resample_fd_list = 135 QLIST_HEAD_INITIALIZER(kvm_resample_fd_list); 136 137 static QemuMutex kml_slots_lock; 138 139 #define kvm_slots_lock() qemu_mutex_lock(&kml_slots_lock) 140 #define kvm_slots_unlock() qemu_mutex_unlock(&kml_slots_lock) 141 142 static void kvm_slot_init_dirty_bitmap(KVMSlot *mem); 143 144 static inline void kvm_resample_fd_remove(int gsi) 145 { 146 KVMResampleFd *rfd; 147 148 QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { 149 if (rfd->gsi == gsi) { 150 QLIST_REMOVE(rfd, node); 151 g_free(rfd); 152 break; 153 } 154 } 155 } 156 157 static inline void kvm_resample_fd_insert(int gsi, EventNotifier *event) 158 { 159 KVMResampleFd *rfd = g_new0(KVMResampleFd, 1); 160 161 rfd->gsi = gsi; 162 rfd->resample_event = event; 163 164 QLIST_INSERT_HEAD(&kvm_resample_fd_list, rfd, node); 165 } 166 167 void kvm_resample_fd_notify(int gsi) 168 { 169 KVMResampleFd *rfd; 170 171 QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { 172 if (rfd->gsi == gsi) { 173 event_notifier_set(rfd->resample_event); 174 trace_kvm_resample_fd_notify(gsi); 175 return; 176 } 177 } 178 } 179 180 /** 181 * kvm_slots_grow(): Grow the slots[] array in the KVMMemoryListener 182 * 183 * @kml: The KVMMemoryListener* to grow the slots[] array 184 * @nr_slots_new: The new size of slots[] array 185 * 186 * Returns: True if the array grows larger, false otherwise. 187 */ 188 static bool kvm_slots_grow(KVMMemoryListener *kml, unsigned int nr_slots_new) 189 { 190 unsigned int i, cur = kml->nr_slots_allocated; 191 KVMSlot *slots; 192 193 if (nr_slots_new > kvm_state->nr_slots_max) { 194 nr_slots_new = kvm_state->nr_slots_max; 195 } 196 197 if (cur >= nr_slots_new) { 198 /* Big enough, no need to grow, or we reached max */ 199 return false; 200 } 201 202 if (cur == 0) { 203 slots = g_new0(KVMSlot, nr_slots_new); 204 } else { 205 assert(kml->slots); 206 slots = g_renew(KVMSlot, kml->slots, nr_slots_new); 207 /* 208 * g_renew() doesn't initialize extended buffers, however kvm 209 * memslots require fields to be zero-initialized. E.g. pointers, 210 * memory_size field, etc. 211 */ 212 memset(&slots[cur], 0x0, sizeof(slots[0]) * (nr_slots_new - cur)); 213 } 214 215 for (i = cur; i < nr_slots_new; i++) { 216 slots[i].slot = i; 217 } 218 219 kml->slots = slots; 220 kml->nr_slots_allocated = nr_slots_new; 221 trace_kvm_slots_grow(cur, nr_slots_new); 222 223 return true; 224 } 225 226 static bool kvm_slots_double(KVMMemoryListener *kml) 227 { 228 return kvm_slots_grow(kml, kml->nr_slots_allocated * 2); 229 } 230 231 unsigned int kvm_get_max_memslots(void) 232 { 233 KVMState *s = KVM_STATE(current_accel()); 234 235 return s->nr_slots_max; 236 } 237 238 unsigned int kvm_get_free_memslots(void) 239 { 240 unsigned int used_slots = 0; 241 KVMState *s = kvm_state; 242 int i; 243 244 kvm_slots_lock(); 245 for (i = 0; i < s->nr_as; i++) { 246 if (!s->as[i].ml) { 247 continue; 248 } 249 used_slots = MAX(used_slots, s->as[i].ml->nr_slots_used); 250 } 251 kvm_slots_unlock(); 252 253 return s->nr_slots_max - used_slots; 254 } 255 256 /* Called with KVMMemoryListener.slots_lock held */ 257 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml) 258 { 259 unsigned int n; 260 int i; 261 262 for (i = 0; i < kml->nr_slots_allocated; i++) { 263 if (kml->slots[i].memory_size == 0) { 264 return &kml->slots[i]; 265 } 266 } 267 268 /* 269 * If no free slots, try to grow first by doubling. Cache the old size 270 * here to avoid another round of search: if the grow succeeded, it 271 * means slots[] now must have the existing "n" slots occupied, 272 * followed by one or more free slots starting from slots[n]. 273 */ 274 n = kml->nr_slots_allocated; 275 if (kvm_slots_double(kml)) { 276 return &kml->slots[n]; 277 } 278 279 return NULL; 280 } 281 282 /* Called with KVMMemoryListener.slots_lock held */ 283 static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml) 284 { 285 KVMSlot *slot = kvm_get_free_slot(kml); 286 287 if (slot) { 288 return slot; 289 } 290 291 fprintf(stderr, "%s: no free slot available\n", __func__); 292 abort(); 293 } 294 295 static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml, 296 hwaddr start_addr, 297 hwaddr size) 298 { 299 int i; 300 301 for (i = 0; i < kml->nr_slots_allocated; i++) { 302 KVMSlot *mem = &kml->slots[i]; 303 304 if (start_addr == mem->start_addr && size == mem->memory_size) { 305 return mem; 306 } 307 } 308 309 return NULL; 310 } 311 312 /* 313 * Calculate and align the start address and the size of the section. 314 * Return the size. If the size is 0, the aligned section is empty. 315 */ 316 static hwaddr kvm_align_section(MemoryRegionSection *section, 317 hwaddr *start) 318 { 319 hwaddr size = int128_get64(section->size); 320 hwaddr delta, aligned; 321 322 /* kvm works in page size chunks, but the function may be called 323 with sub-page size and unaligned start address. Pad the start 324 address to next and truncate size to previous page boundary. */ 325 aligned = ROUND_UP(section->offset_within_address_space, 326 qemu_real_host_page_size()); 327 delta = aligned - section->offset_within_address_space; 328 *start = aligned; 329 if (delta > size) { 330 return 0; 331 } 332 333 return (size - delta) & qemu_real_host_page_mask(); 334 } 335 336 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram, 337 hwaddr *phys_addr) 338 { 339 KVMMemoryListener *kml = &s->memory_listener; 340 int i, ret = 0; 341 342 kvm_slots_lock(); 343 for (i = 0; i < kml->nr_slots_allocated; i++) { 344 KVMSlot *mem = &kml->slots[i]; 345 346 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) { 347 *phys_addr = mem->start_addr + (ram - mem->ram); 348 ret = 1; 349 break; 350 } 351 } 352 kvm_slots_unlock(); 353 354 return ret; 355 } 356 357 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new) 358 { 359 KVMState *s = kvm_state; 360 struct kvm_userspace_memory_region2 mem; 361 int ret; 362 363 mem.slot = slot->slot | (kml->as_id << 16); 364 mem.guest_phys_addr = slot->start_addr; 365 mem.userspace_addr = (unsigned long)slot->ram; 366 mem.flags = slot->flags; 367 mem.guest_memfd = slot->guest_memfd; 368 mem.guest_memfd_offset = slot->guest_memfd_offset; 369 370 if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) { 371 /* Set the slot size to 0 before setting the slot to the desired 372 * value. This is needed based on KVM commit 75d61fbc. */ 373 mem.memory_size = 0; 374 375 if (kvm_guest_memfd_supported) { 376 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem); 377 } else { 378 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 379 } 380 if (ret < 0) { 381 goto err; 382 } 383 } 384 mem.memory_size = slot->memory_size; 385 if (kvm_guest_memfd_supported) { 386 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem); 387 } else { 388 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 389 } 390 slot->old_flags = mem.flags; 391 err: 392 trace_kvm_set_user_memory(mem.slot >> 16, (uint16_t)mem.slot, mem.flags, 393 mem.guest_phys_addr, mem.memory_size, 394 mem.userspace_addr, mem.guest_memfd, 395 mem.guest_memfd_offset, ret); 396 if (ret < 0) { 397 if (kvm_guest_memfd_supported) { 398 error_report("%s: KVM_SET_USER_MEMORY_REGION2 failed, slot=%d," 399 " start=0x%" PRIx64 ", size=0x%" PRIx64 "," 400 " flags=0x%" PRIx32 ", guest_memfd=%" PRId32 "," 401 " guest_memfd_offset=0x%" PRIx64 ": %s", 402 __func__, mem.slot, slot->start_addr, 403 (uint64_t)mem.memory_size, mem.flags, 404 mem.guest_memfd, (uint64_t)mem.guest_memfd_offset, 405 strerror(errno)); 406 } else { 407 error_report("%s: KVM_SET_USER_MEMORY_REGION failed, slot=%d," 408 " start=0x%" PRIx64 ", size=0x%" PRIx64 ": %s", 409 __func__, mem.slot, slot->start_addr, 410 (uint64_t)mem.memory_size, strerror(errno)); 411 } 412 } 413 return ret; 414 } 415 416 void kvm_park_vcpu(CPUState *cpu) 417 { 418 struct KVMParkedVcpu *vcpu; 419 420 trace_kvm_park_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); 421 422 vcpu = g_malloc0(sizeof(*vcpu)); 423 vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); 424 vcpu->kvm_fd = cpu->kvm_fd; 425 QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); 426 } 427 428 int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id) 429 { 430 struct KVMParkedVcpu *cpu; 431 int kvm_fd = -ENOENT; 432 433 QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { 434 if (cpu->vcpu_id == vcpu_id) { 435 QLIST_REMOVE(cpu, node); 436 kvm_fd = cpu->kvm_fd; 437 g_free(cpu); 438 break; 439 } 440 } 441 442 trace_kvm_unpark_vcpu(vcpu_id, kvm_fd > 0 ? "unparked" : "!found parked"); 443 444 return kvm_fd; 445 } 446 447 static void kvm_reset_parked_vcpus(KVMState *s) 448 { 449 struct KVMParkedVcpu *cpu; 450 451 QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { 452 kvm_arch_reset_parked_vcpu(cpu->vcpu_id, cpu->kvm_fd); 453 } 454 } 455 456 int kvm_create_vcpu(CPUState *cpu) 457 { 458 unsigned long vcpu_id = kvm_arch_vcpu_id(cpu); 459 KVMState *s = kvm_state; 460 int kvm_fd; 461 462 /* check if the KVM vCPU already exist but is parked */ 463 kvm_fd = kvm_unpark_vcpu(s, vcpu_id); 464 if (kvm_fd < 0) { 465 /* vCPU not parked: create a new KVM vCPU */ 466 kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id); 467 if (kvm_fd < 0) { 468 error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu", vcpu_id); 469 return kvm_fd; 470 } 471 } 472 473 cpu->kvm_fd = kvm_fd; 474 cpu->kvm_state = s; 475 if (!s->guest_state_protected) { 476 cpu->vcpu_dirty = true; 477 } 478 cpu->dirty_pages = 0; 479 cpu->throttle_us_per_full = 0; 480 481 trace_kvm_create_vcpu(cpu->cpu_index, vcpu_id, kvm_fd); 482 483 return 0; 484 } 485 486 int kvm_create_and_park_vcpu(CPUState *cpu) 487 { 488 int ret = 0; 489 490 ret = kvm_create_vcpu(cpu); 491 if (!ret) { 492 kvm_park_vcpu(cpu); 493 } 494 495 return ret; 496 } 497 498 static int do_kvm_destroy_vcpu(CPUState *cpu) 499 { 500 KVMState *s = kvm_state; 501 int mmap_size; 502 int ret = 0; 503 504 trace_kvm_destroy_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); 505 506 ret = kvm_arch_destroy_vcpu(cpu); 507 if (ret < 0) { 508 goto err; 509 } 510 511 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 512 if (mmap_size < 0) { 513 ret = mmap_size; 514 trace_kvm_failed_get_vcpu_mmap_size(); 515 goto err; 516 } 517 518 ret = munmap(cpu->kvm_run, mmap_size); 519 if (ret < 0) { 520 goto err; 521 } 522 523 if (cpu->kvm_dirty_gfns) { 524 ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes); 525 if (ret < 0) { 526 goto err; 527 } 528 } 529 530 kvm_park_vcpu(cpu); 531 err: 532 return ret; 533 } 534 535 void kvm_destroy_vcpu(CPUState *cpu) 536 { 537 if (do_kvm_destroy_vcpu(cpu) < 0) { 538 error_report("kvm_destroy_vcpu failed"); 539 exit(EXIT_FAILURE); 540 } 541 } 542 543 int kvm_init_vcpu(CPUState *cpu, Error **errp) 544 { 545 KVMState *s = kvm_state; 546 int mmap_size; 547 int ret; 548 549 trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); 550 551 ret = kvm_arch_pre_create_vcpu(cpu, errp); 552 if (ret < 0) { 553 goto err; 554 } 555 556 ret = kvm_create_vcpu(cpu); 557 if (ret < 0) { 558 error_setg_errno(errp, -ret, 559 "kvm_init_vcpu: kvm_create_vcpu failed (%lu)", 560 kvm_arch_vcpu_id(cpu)); 561 goto err; 562 } 563 564 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 565 if (mmap_size < 0) { 566 ret = mmap_size; 567 error_setg_errno(errp, -mmap_size, 568 "kvm_init_vcpu: KVM_GET_VCPU_MMAP_SIZE failed"); 569 goto err; 570 } 571 572 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, 573 cpu->kvm_fd, 0); 574 if (cpu->kvm_run == MAP_FAILED) { 575 ret = -errno; 576 error_setg_errno(errp, ret, 577 "kvm_init_vcpu: mmap'ing vcpu state failed (%lu)", 578 kvm_arch_vcpu_id(cpu)); 579 goto err; 580 } 581 582 if (s->coalesced_mmio && !s->coalesced_mmio_ring) { 583 s->coalesced_mmio_ring = 584 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE; 585 } 586 587 if (s->kvm_dirty_ring_size) { 588 /* Use MAP_SHARED to share pages with the kernel */ 589 cpu->kvm_dirty_gfns = mmap(NULL, s->kvm_dirty_ring_bytes, 590 PROT_READ | PROT_WRITE, MAP_SHARED, 591 cpu->kvm_fd, 592 PAGE_SIZE * KVM_DIRTY_LOG_PAGE_OFFSET); 593 if (cpu->kvm_dirty_gfns == MAP_FAILED) { 594 ret = -errno; 595 goto err; 596 } 597 } 598 599 ret = kvm_arch_init_vcpu(cpu); 600 if (ret < 0) { 601 error_setg_errno(errp, -ret, 602 "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)", 603 kvm_arch_vcpu_id(cpu)); 604 } 605 cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL); 606 607 err: 608 return ret; 609 } 610 611 /* 612 * dirty pages logging control 613 */ 614 615 static int kvm_mem_flags(MemoryRegion *mr) 616 { 617 bool readonly = mr->readonly || memory_region_is_romd(mr); 618 int flags = 0; 619 620 if (memory_region_get_dirty_log_mask(mr) != 0) { 621 flags |= KVM_MEM_LOG_DIRTY_PAGES; 622 } 623 if (readonly && kvm_readonly_mem_allowed) { 624 flags |= KVM_MEM_READONLY; 625 } 626 if (memory_region_has_guest_memfd(mr)) { 627 assert(kvm_guest_memfd_supported); 628 flags |= KVM_MEM_GUEST_MEMFD; 629 } 630 return flags; 631 } 632 633 /* Called with KVMMemoryListener.slots_lock held */ 634 static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem, 635 MemoryRegion *mr) 636 { 637 mem->flags = kvm_mem_flags(mr); 638 639 /* If nothing changed effectively, no need to issue ioctl */ 640 if (mem->flags == mem->old_flags) { 641 return 0; 642 } 643 644 kvm_slot_init_dirty_bitmap(mem); 645 return kvm_set_user_memory_region(kml, mem, false); 646 } 647 648 static int kvm_section_update_flags(KVMMemoryListener *kml, 649 MemoryRegionSection *section) 650 { 651 hwaddr start_addr, size, slot_size; 652 KVMSlot *mem; 653 int ret = 0; 654 655 size = kvm_align_section(section, &start_addr); 656 if (!size) { 657 return 0; 658 } 659 660 kvm_slots_lock(); 661 662 while (size && !ret) { 663 slot_size = MIN(kvm_max_slot_size, size); 664 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 665 if (!mem) { 666 /* We don't have a slot if we want to trap every access. */ 667 goto out; 668 } 669 670 ret = kvm_slot_update_flags(kml, mem, section->mr); 671 start_addr += slot_size; 672 size -= slot_size; 673 } 674 675 out: 676 kvm_slots_unlock(); 677 return ret; 678 } 679 680 static void kvm_log_start(MemoryListener *listener, 681 MemoryRegionSection *section, 682 int old, int new) 683 { 684 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 685 int r; 686 687 if (old != 0) { 688 return; 689 } 690 691 r = kvm_section_update_flags(kml, section); 692 if (r < 0) { 693 abort(); 694 } 695 } 696 697 static void kvm_log_stop(MemoryListener *listener, 698 MemoryRegionSection *section, 699 int old, int new) 700 { 701 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 702 int r; 703 704 if (new != 0) { 705 return; 706 } 707 708 r = kvm_section_update_flags(kml, section); 709 if (r < 0) { 710 abort(); 711 } 712 } 713 714 /* get kvm's dirty pages bitmap and update qemu's */ 715 static void kvm_slot_sync_dirty_pages(KVMSlot *slot) 716 { 717 ram_addr_t start = slot->ram_start_offset; 718 ram_addr_t pages = slot->memory_size / qemu_real_host_page_size(); 719 720 cpu_physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); 721 } 722 723 static void kvm_slot_reset_dirty_pages(KVMSlot *slot) 724 { 725 memset(slot->dirty_bmap, 0, slot->dirty_bmap_size); 726 } 727 728 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1)) 729 730 /* Allocate the dirty bitmap for a slot */ 731 static void kvm_slot_init_dirty_bitmap(KVMSlot *mem) 732 { 733 if (!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) || mem->dirty_bmap) { 734 return; 735 } 736 737 /* 738 * XXX bad kernel interface alert 739 * For dirty bitmap, kernel allocates array of size aligned to 740 * bits-per-long. But for case when the kernel is 64bits and 741 * the userspace is 32bits, userspace can't align to the same 742 * bits-per-long, since sizeof(long) is different between kernel 743 * and user space. This way, userspace will provide buffer which 744 * may be 4 bytes less than the kernel will use, resulting in 745 * userspace memory corruption (which is not detectable by valgrind 746 * too, in most cases). 747 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in 748 * a hope that sizeof(long) won't become >8 any time soon. 749 * 750 * Note: the granule of kvm dirty log is qemu_real_host_page_size. 751 * And mem->memory_size is aligned to it (otherwise this mem can't 752 * be registered to KVM). 753 */ 754 hwaddr bitmap_size = ALIGN(mem->memory_size / qemu_real_host_page_size(), 755 /*HOST_LONG_BITS*/ 64) / 8; 756 mem->dirty_bmap = g_malloc0(bitmap_size); 757 mem->dirty_bmap_size = bitmap_size; 758 } 759 760 /* 761 * Sync dirty bitmap from kernel to KVMSlot.dirty_bmap, return true if 762 * succeeded, false otherwise 763 */ 764 static bool kvm_slot_get_dirty_log(KVMState *s, KVMSlot *slot) 765 { 766 struct kvm_dirty_log d = {}; 767 int ret; 768 769 d.dirty_bitmap = slot->dirty_bmap; 770 d.slot = slot->slot | (slot->as_id << 16); 771 ret = kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d); 772 773 if (ret == -ENOENT) { 774 /* kernel does not have dirty bitmap in this slot */ 775 ret = 0; 776 } 777 if (ret) { 778 error_report_once("%s: KVM_GET_DIRTY_LOG failed with %d", 779 __func__, ret); 780 } 781 return ret == 0; 782 } 783 784 /* Should be with all slots_lock held for the address spaces. */ 785 static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id, 786 uint32_t slot_id, uint64_t offset) 787 { 788 KVMMemoryListener *kml; 789 KVMSlot *mem; 790 791 if (as_id >= s->nr_as) { 792 return; 793 } 794 795 kml = s->as[as_id].ml; 796 mem = &kml->slots[slot_id]; 797 798 if (!mem->memory_size || offset >= 799 (mem->memory_size / qemu_real_host_page_size())) { 800 return; 801 } 802 803 set_bit(offset, mem->dirty_bmap); 804 } 805 806 static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn) 807 { 808 /* 809 * Read the flags before the value. Pairs with barrier in 810 * KVM's kvm_dirty_ring_push() function. 811 */ 812 return qatomic_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY; 813 } 814 815 static void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn) 816 { 817 /* 818 * Use a store-release so that the CPU that executes KVM_RESET_DIRTY_RINGS 819 * sees the full content of the ring: 820 * 821 * CPU0 CPU1 CPU2 822 * ------------------------------------------------------------------------------ 823 * fill gfn0 824 * store-rel flags for gfn0 825 * load-acq flags for gfn0 826 * store-rel RESET for gfn0 827 * ioctl(RESET_RINGS) 828 * load-acq flags for gfn0 829 * check if flags have RESET 830 * 831 * The synchronization goes from CPU2 to CPU0 to CPU1. 832 */ 833 qatomic_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET); 834 } 835 836 /* 837 * Should be with all slots_lock held for the address spaces. It returns the 838 * dirty page we've collected on this dirty ring. 839 */ 840 static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu) 841 { 842 struct kvm_dirty_gfn *dirty_gfns = cpu->kvm_dirty_gfns, *cur; 843 uint32_t ring_size = s->kvm_dirty_ring_size; 844 uint32_t count = 0, fetch = cpu->kvm_fetch_index; 845 846 /* 847 * It's possible that we race with vcpu creation code where the vcpu is 848 * put onto the vcpus list but not yet initialized the dirty ring 849 * structures. If so, skip it. 850 */ 851 if (!cpu->created) { 852 return 0; 853 } 854 855 assert(dirty_gfns && ring_size); 856 trace_kvm_dirty_ring_reap_vcpu(cpu->cpu_index); 857 858 while (true) { 859 cur = &dirty_gfns[fetch % ring_size]; 860 if (!dirty_gfn_is_dirtied(cur)) { 861 break; 862 } 863 kvm_dirty_ring_mark_page(s, cur->slot >> 16, cur->slot & 0xffff, 864 cur->offset); 865 dirty_gfn_set_collected(cur); 866 trace_kvm_dirty_ring_page(cpu->cpu_index, fetch, cur->offset); 867 fetch++; 868 count++; 869 } 870 cpu->kvm_fetch_index = fetch; 871 cpu->dirty_pages += count; 872 873 return count; 874 } 875 876 /* Must be with slots_lock held */ 877 static uint64_t kvm_dirty_ring_reap_locked(KVMState *s, CPUState* cpu) 878 { 879 int ret; 880 uint64_t total = 0; 881 int64_t stamp; 882 883 stamp = get_clock(); 884 885 if (cpu) { 886 total = kvm_dirty_ring_reap_one(s, cpu); 887 } else { 888 CPU_FOREACH(cpu) { 889 total += kvm_dirty_ring_reap_one(s, cpu); 890 } 891 } 892 893 if (total) { 894 ret = kvm_vm_ioctl(s, KVM_RESET_DIRTY_RINGS); 895 assert(ret == total); 896 } 897 898 stamp = get_clock() - stamp; 899 900 if (total) { 901 trace_kvm_dirty_ring_reap(total, stamp / 1000); 902 } 903 904 return total; 905 } 906 907 /* 908 * Currently for simplicity, we must hold BQL before calling this. We can 909 * consider to drop the BQL if we're clear with all the race conditions. 910 */ 911 static uint64_t kvm_dirty_ring_reap(KVMState *s, CPUState *cpu) 912 { 913 uint64_t total; 914 915 /* 916 * We need to lock all kvm slots for all address spaces here, 917 * because: 918 * 919 * (1) We need to mark dirty for dirty bitmaps in multiple slots 920 * and for tons of pages, so it's better to take the lock here 921 * once rather than once per page. And more importantly, 922 * 923 * (2) We must _NOT_ publish dirty bits to the other threads 924 * (e.g., the migration thread) via the kvm memory slot dirty 925 * bitmaps before correctly re-protect those dirtied pages. 926 * Otherwise we can have potential risk of data corruption if 927 * the page data is read in the other thread before we do 928 * reset below. 929 */ 930 kvm_slots_lock(); 931 total = kvm_dirty_ring_reap_locked(s, cpu); 932 kvm_slots_unlock(); 933 934 return total; 935 } 936 937 static void do_kvm_cpu_synchronize_kick(CPUState *cpu, run_on_cpu_data arg) 938 { 939 /* No need to do anything */ 940 } 941 942 /* 943 * Kick all vcpus out in a synchronized way. When returned, we 944 * guarantee that every vcpu has been kicked and at least returned to 945 * userspace once. 946 */ 947 static void kvm_cpu_synchronize_kick_all(void) 948 { 949 CPUState *cpu; 950 951 CPU_FOREACH(cpu) { 952 run_on_cpu(cpu, do_kvm_cpu_synchronize_kick, RUN_ON_CPU_NULL); 953 } 954 } 955 956 /* 957 * Flush all the existing dirty pages to the KVM slot buffers. When 958 * this call returns, we guarantee that all the touched dirty pages 959 * before calling this function have been put into the per-kvmslot 960 * dirty bitmap. 961 * 962 * This function must be called with BQL held. 963 */ 964 static void kvm_dirty_ring_flush(void) 965 { 966 trace_kvm_dirty_ring_flush(0); 967 /* 968 * The function needs to be serialized. Since this function 969 * should always be with BQL held, serialization is guaranteed. 970 * However, let's be sure of it. 971 */ 972 assert(bql_locked()); 973 /* 974 * First make sure to flush the hardware buffers by kicking all 975 * vcpus out in a synchronous way. 976 */ 977 kvm_cpu_synchronize_kick_all(); 978 kvm_dirty_ring_reap(kvm_state, NULL); 979 trace_kvm_dirty_ring_flush(1); 980 } 981 982 /** 983 * kvm_physical_sync_dirty_bitmap - Sync dirty bitmap from kernel space 984 * 985 * This function will first try to fetch dirty bitmap from the kernel, 986 * and then updates qemu's dirty bitmap. 987 * 988 * NOTE: caller must be with kml->slots_lock held. 989 * 990 * @kml: the KVM memory listener object 991 * @section: the memory section to sync the dirty bitmap with 992 */ 993 static void kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, 994 MemoryRegionSection *section) 995 { 996 KVMState *s = kvm_state; 997 KVMSlot *mem; 998 hwaddr start_addr, size; 999 hwaddr slot_size; 1000 1001 size = kvm_align_section(section, &start_addr); 1002 while (size) { 1003 slot_size = MIN(kvm_max_slot_size, size); 1004 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 1005 if (!mem) { 1006 /* We don't have a slot if we want to trap every access. */ 1007 return; 1008 } 1009 if (kvm_slot_get_dirty_log(s, mem)) { 1010 kvm_slot_sync_dirty_pages(mem); 1011 } 1012 start_addr += slot_size; 1013 size -= slot_size; 1014 } 1015 } 1016 1017 /* Alignment requirement for KVM_CLEAR_DIRTY_LOG - 64 pages */ 1018 #define KVM_CLEAR_LOG_SHIFT 6 1019 #define KVM_CLEAR_LOG_ALIGN (qemu_real_host_page_size() << KVM_CLEAR_LOG_SHIFT) 1020 #define KVM_CLEAR_LOG_MASK (-KVM_CLEAR_LOG_ALIGN) 1021 1022 static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start, 1023 uint64_t size) 1024 { 1025 KVMState *s = kvm_state; 1026 uint64_t end, bmap_start, start_delta, bmap_npages; 1027 struct kvm_clear_dirty_log d; 1028 unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size(); 1029 int ret; 1030 1031 /* 1032 * We need to extend either the start or the size or both to 1033 * satisfy the KVM interface requirement. Firstly, do the start 1034 * page alignment on 64 host pages 1035 */ 1036 bmap_start = start & KVM_CLEAR_LOG_MASK; 1037 start_delta = start - bmap_start; 1038 bmap_start /= psize; 1039 1040 /* 1041 * The kernel interface has restriction on the size too, that either: 1042 * 1043 * (1) the size is 64 host pages aligned (just like the start), or 1044 * (2) the size fills up until the end of the KVM memslot. 1045 */ 1046 bmap_npages = DIV_ROUND_UP(size + start_delta, KVM_CLEAR_LOG_ALIGN) 1047 << KVM_CLEAR_LOG_SHIFT; 1048 end = mem->memory_size / psize; 1049 if (bmap_npages > end - bmap_start) { 1050 bmap_npages = end - bmap_start; 1051 } 1052 start_delta /= psize; 1053 1054 /* 1055 * Prepare the bitmap to clear dirty bits. Here we must guarantee 1056 * that we won't clear any unknown dirty bits otherwise we might 1057 * accidentally clear some set bits which are not yet synced from 1058 * the kernel into QEMU's bitmap, then we'll lose track of the 1059 * guest modifications upon those pages (which can directly lead 1060 * to guest data loss or panic after migration). 1061 * 1062 * Layout of the KVMSlot.dirty_bmap: 1063 * 1064 * |<-------- bmap_npages -----------..>| 1065 * [1] 1066 * start_delta size 1067 * |----------------|-------------|------------------|------------| 1068 * ^ ^ ^ ^ 1069 * | | | | 1070 * start bmap_start (start) end 1071 * of memslot of memslot 1072 * 1073 * [1] bmap_npages can be aligned to either 64 pages or the end of slot 1074 */ 1075 1076 assert(bmap_start % BITS_PER_LONG == 0); 1077 /* We should never do log_clear before log_sync */ 1078 assert(mem->dirty_bmap); 1079 if (start_delta || bmap_npages - size / psize) { 1080 /* Slow path - we need to manipulate a temp bitmap */ 1081 bmap_clear = bitmap_new(bmap_npages); 1082 bitmap_copy_with_src_offset(bmap_clear, mem->dirty_bmap, 1083 bmap_start, start_delta + size / psize); 1084 /* 1085 * We need to fill the holes at start because that was not 1086 * specified by the caller and we extended the bitmap only for 1087 * 64 pages alignment 1088 */ 1089 bitmap_clear(bmap_clear, 0, start_delta); 1090 d.dirty_bitmap = bmap_clear; 1091 } else { 1092 /* 1093 * Fast path - both start and size align well with BITS_PER_LONG 1094 * (or the end of memory slot) 1095 */ 1096 d.dirty_bitmap = mem->dirty_bmap + BIT_WORD(bmap_start); 1097 } 1098 1099 d.first_page = bmap_start; 1100 /* It should never overflow. If it happens, say something */ 1101 assert(bmap_npages <= UINT32_MAX); 1102 d.num_pages = bmap_npages; 1103 d.slot = mem->slot | (as_id << 16); 1104 1105 ret = kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d); 1106 if (ret < 0 && ret != -ENOENT) { 1107 error_report("%s: KVM_CLEAR_DIRTY_LOG failed, slot=%d, " 1108 "start=0x%"PRIx64", size=0x%"PRIx32", errno=%d", 1109 __func__, d.slot, (uint64_t)d.first_page, 1110 (uint32_t)d.num_pages, ret); 1111 } else { 1112 ret = 0; 1113 trace_kvm_clear_dirty_log(d.slot, d.first_page, d.num_pages); 1114 } 1115 1116 /* 1117 * After we have updated the remote dirty bitmap, we update the 1118 * cached bitmap as well for the memslot, then if another user 1119 * clears the same region we know we shouldn't clear it again on 1120 * the remote otherwise it's data loss as well. 1121 */ 1122 bitmap_clear(mem->dirty_bmap, bmap_start + start_delta, 1123 size / psize); 1124 /* This handles the NULL case well */ 1125 g_free(bmap_clear); 1126 return ret; 1127 } 1128 1129 1130 /** 1131 * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range 1132 * 1133 * NOTE: this will be a no-op if we haven't enabled manual dirty log 1134 * protection in the host kernel because in that case this operation 1135 * will be done within log_sync(). 1136 * 1137 * @kml: the kvm memory listener 1138 * @section: the memory range to clear dirty bitmap 1139 */ 1140 static int kvm_physical_log_clear(KVMMemoryListener *kml, 1141 MemoryRegionSection *section) 1142 { 1143 KVMState *s = kvm_state; 1144 uint64_t start, size, offset, count; 1145 KVMSlot *mem; 1146 int ret = 0, i; 1147 1148 if (!s->manual_dirty_log_protect) { 1149 /* No need to do explicit clear */ 1150 return ret; 1151 } 1152 1153 start = section->offset_within_address_space; 1154 size = int128_get64(section->size); 1155 1156 if (!size) { 1157 /* Nothing more we can do... */ 1158 return ret; 1159 } 1160 1161 kvm_slots_lock(); 1162 1163 for (i = 0; i < kml->nr_slots_allocated; i++) { 1164 mem = &kml->slots[i]; 1165 /* Discard slots that are empty or do not overlap the section */ 1166 if (!mem->memory_size || 1167 mem->start_addr > start + size - 1 || 1168 start > mem->start_addr + mem->memory_size - 1) { 1169 continue; 1170 } 1171 1172 if (start >= mem->start_addr) { 1173 /* The slot starts before section or is aligned to it. */ 1174 offset = start - mem->start_addr; 1175 count = MIN(mem->memory_size - offset, size); 1176 } else { 1177 /* The slot starts after section. */ 1178 offset = 0; 1179 count = MIN(mem->memory_size, size - (mem->start_addr - start)); 1180 } 1181 ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count); 1182 if (ret < 0) { 1183 break; 1184 } 1185 } 1186 1187 kvm_slots_unlock(); 1188 1189 return ret; 1190 } 1191 1192 static void kvm_coalesce_mmio_region(MemoryListener *listener, 1193 MemoryRegionSection *secion, 1194 hwaddr start, hwaddr size) 1195 { 1196 KVMState *s = kvm_state; 1197 1198 if (s->coalesced_mmio) { 1199 struct kvm_coalesced_mmio_zone zone; 1200 1201 zone.addr = start; 1202 zone.size = size; 1203 zone.pad = 0; 1204 1205 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 1206 } 1207 } 1208 1209 static void kvm_uncoalesce_mmio_region(MemoryListener *listener, 1210 MemoryRegionSection *secion, 1211 hwaddr start, hwaddr size) 1212 { 1213 KVMState *s = kvm_state; 1214 1215 if (s->coalesced_mmio) { 1216 struct kvm_coalesced_mmio_zone zone; 1217 1218 zone.addr = start; 1219 zone.size = size; 1220 zone.pad = 0; 1221 1222 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 1223 } 1224 } 1225 1226 static void kvm_coalesce_pio_add(MemoryListener *listener, 1227 MemoryRegionSection *section, 1228 hwaddr start, hwaddr size) 1229 { 1230 KVMState *s = kvm_state; 1231 1232 if (s->coalesced_pio) { 1233 struct kvm_coalesced_mmio_zone zone; 1234 1235 zone.addr = start; 1236 zone.size = size; 1237 zone.pio = 1; 1238 1239 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 1240 } 1241 } 1242 1243 static void kvm_coalesce_pio_del(MemoryListener *listener, 1244 MemoryRegionSection *section, 1245 hwaddr start, hwaddr size) 1246 { 1247 KVMState *s = kvm_state; 1248 1249 if (s->coalesced_pio) { 1250 struct kvm_coalesced_mmio_zone zone; 1251 1252 zone.addr = start; 1253 zone.size = size; 1254 zone.pio = 1; 1255 1256 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 1257 } 1258 } 1259 1260 int kvm_check_extension(KVMState *s, unsigned int extension) 1261 { 1262 int ret; 1263 1264 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension); 1265 if (ret < 0) { 1266 ret = 0; 1267 } 1268 1269 return ret; 1270 } 1271 1272 int kvm_vm_check_extension(KVMState *s, unsigned int extension) 1273 { 1274 int ret; 1275 1276 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension); 1277 if (ret < 0) { 1278 /* VM wide version not implemented, use global one instead */ 1279 ret = kvm_check_extension(s, extension); 1280 } 1281 1282 return ret; 1283 } 1284 1285 /* 1286 * We track the poisoned pages to be able to: 1287 * - replace them on VM reset 1288 * - block a migration for a VM with a poisoned page 1289 */ 1290 typedef struct HWPoisonPage { 1291 ram_addr_t ram_addr; 1292 QLIST_ENTRY(HWPoisonPage) list; 1293 } HWPoisonPage; 1294 1295 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list = 1296 QLIST_HEAD_INITIALIZER(hwpoison_page_list); 1297 1298 static void kvm_unpoison_all(void *param) 1299 { 1300 HWPoisonPage *page, *next_page; 1301 1302 QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) { 1303 QLIST_REMOVE(page, list); 1304 qemu_ram_remap(page->ram_addr); 1305 g_free(page); 1306 } 1307 } 1308 1309 void kvm_hwpoison_page_add(ram_addr_t ram_addr) 1310 { 1311 HWPoisonPage *page; 1312 1313 QLIST_FOREACH(page, &hwpoison_page_list, list) { 1314 if (page->ram_addr == ram_addr) { 1315 return; 1316 } 1317 } 1318 page = g_new(HWPoisonPage, 1); 1319 page->ram_addr = ram_addr; 1320 QLIST_INSERT_HEAD(&hwpoison_page_list, page, list); 1321 } 1322 1323 bool kvm_hwpoisoned_mem(void) 1324 { 1325 return !QLIST_EMPTY(&hwpoison_page_list); 1326 } 1327 1328 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) 1329 { 1330 if (target_needs_bswap()) { 1331 /* 1332 * The kernel expects ioeventfd values in HOST_BIG_ENDIAN 1333 * endianness, but the memory core hands them in target endianness. 1334 * For example, PPC is always treated as big-endian even if running 1335 * on KVM and on PPC64LE. Correct here, swapping back. 1336 */ 1337 switch (size) { 1338 case 2: 1339 val = bswap16(val); 1340 break; 1341 case 4: 1342 val = bswap32(val); 1343 break; 1344 } 1345 } 1346 return val; 1347 } 1348 1349 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val, 1350 bool assign, uint32_t size, bool datamatch) 1351 { 1352 int ret; 1353 struct kvm_ioeventfd iofd = { 1354 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 1355 .addr = addr, 1356 .len = size, 1357 .flags = 0, 1358 .fd = fd, 1359 }; 1360 1361 trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size, 1362 datamatch); 1363 if (!kvm_enabled()) { 1364 return -ENOSYS; 1365 } 1366 1367 if (datamatch) { 1368 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 1369 } 1370 if (!assign) { 1371 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 1372 } 1373 1374 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd); 1375 1376 if (ret < 0) { 1377 return -errno; 1378 } 1379 1380 return 0; 1381 } 1382 1383 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val, 1384 bool assign, uint32_t size, bool datamatch) 1385 { 1386 struct kvm_ioeventfd kick = { 1387 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 1388 .addr = addr, 1389 .flags = KVM_IOEVENTFD_FLAG_PIO, 1390 .len = size, 1391 .fd = fd, 1392 }; 1393 int r; 1394 trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch); 1395 if (!kvm_enabled()) { 1396 return -ENOSYS; 1397 } 1398 if (datamatch) { 1399 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 1400 } 1401 if (!assign) { 1402 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 1403 } 1404 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick); 1405 if (r < 0) { 1406 return r; 1407 } 1408 return 0; 1409 } 1410 1411 1412 static const KVMCapabilityInfo * 1413 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list) 1414 { 1415 while (list->name) { 1416 if (!kvm_check_extension(s, list->value)) { 1417 return list; 1418 } 1419 list++; 1420 } 1421 return NULL; 1422 } 1423 1424 void kvm_set_max_memslot_size(hwaddr max_slot_size) 1425 { 1426 g_assert( 1427 ROUND_UP(max_slot_size, qemu_real_host_page_size()) == max_slot_size 1428 ); 1429 kvm_max_slot_size = max_slot_size; 1430 } 1431 1432 static int kvm_set_memory_attributes(hwaddr start, uint64_t size, uint64_t attr) 1433 { 1434 struct kvm_memory_attributes attrs; 1435 int r; 1436 1437 assert((attr & kvm_supported_memory_attributes) == attr); 1438 attrs.attributes = attr; 1439 attrs.address = start; 1440 attrs.size = size; 1441 attrs.flags = 0; 1442 1443 r = kvm_vm_ioctl(kvm_state, KVM_SET_MEMORY_ATTRIBUTES, &attrs); 1444 if (r) { 1445 error_report("failed to set memory (0x%" HWADDR_PRIx "+0x%" PRIx64 ") " 1446 "with attr 0x%" PRIx64 " error '%s'", 1447 start, size, attr, strerror(errno)); 1448 } 1449 return r; 1450 } 1451 1452 int kvm_set_memory_attributes_private(hwaddr start, uint64_t size) 1453 { 1454 return kvm_set_memory_attributes(start, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); 1455 } 1456 1457 int kvm_set_memory_attributes_shared(hwaddr start, uint64_t size) 1458 { 1459 return kvm_set_memory_attributes(start, size, 0); 1460 } 1461 1462 /* Called with KVMMemoryListener.slots_lock held */ 1463 static void kvm_set_phys_mem(KVMMemoryListener *kml, 1464 MemoryRegionSection *section, bool add) 1465 { 1466 KVMSlot *mem; 1467 int err; 1468 MemoryRegion *mr = section->mr; 1469 bool writable = !mr->readonly && !mr->rom_device; 1470 hwaddr start_addr, size, slot_size, mr_offset; 1471 ram_addr_t ram_start_offset; 1472 void *ram; 1473 1474 if (!memory_region_is_ram(mr)) { 1475 if (writable || !kvm_readonly_mem_allowed) { 1476 return; 1477 } else if (!mr->romd_mode) { 1478 /* If the memory device is not in romd_mode, then we actually want 1479 * to remove the kvm memory slot so all accesses will trap. */ 1480 add = false; 1481 } 1482 } 1483 1484 size = kvm_align_section(section, &start_addr); 1485 if (!size) { 1486 return; 1487 } 1488 1489 /* The offset of the kvmslot within the memory region */ 1490 mr_offset = section->offset_within_region + start_addr - 1491 section->offset_within_address_space; 1492 1493 /* use aligned delta to align the ram address and offset */ 1494 ram = memory_region_get_ram_ptr(mr) + mr_offset; 1495 ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset; 1496 1497 if (!add) { 1498 do { 1499 slot_size = MIN(kvm_max_slot_size, size); 1500 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 1501 if (!mem) { 1502 return; 1503 } 1504 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1505 /* 1506 * NOTE: We should be aware of the fact that here we're only 1507 * doing a best effort to sync dirty bits. No matter whether 1508 * we're using dirty log or dirty ring, we ignored two facts: 1509 * 1510 * (1) dirty bits can reside in hardware buffers (PML) 1511 * 1512 * (2) after we collected dirty bits here, pages can be dirtied 1513 * again before we do the final KVM_SET_USER_MEMORY_REGION to 1514 * remove the slot. 1515 * 1516 * Not easy. Let's cross the fingers until it's fixed. 1517 */ 1518 if (kvm_state->kvm_dirty_ring_size) { 1519 kvm_dirty_ring_reap_locked(kvm_state, NULL); 1520 if (kvm_state->kvm_dirty_ring_with_bitmap) { 1521 kvm_slot_sync_dirty_pages(mem); 1522 kvm_slot_get_dirty_log(kvm_state, mem); 1523 } 1524 } else { 1525 kvm_slot_get_dirty_log(kvm_state, mem); 1526 } 1527 kvm_slot_sync_dirty_pages(mem); 1528 } 1529 1530 /* unregister the slot */ 1531 g_free(mem->dirty_bmap); 1532 mem->dirty_bmap = NULL; 1533 mem->memory_size = 0; 1534 mem->flags = 0; 1535 err = kvm_set_user_memory_region(kml, mem, false); 1536 if (err) { 1537 fprintf(stderr, "%s: error unregistering slot: %s\n", 1538 __func__, strerror(-err)); 1539 abort(); 1540 } 1541 start_addr += slot_size; 1542 size -= slot_size; 1543 kml->nr_slots_used--; 1544 } while (size); 1545 return; 1546 } 1547 1548 /* register the new slot */ 1549 do { 1550 slot_size = MIN(kvm_max_slot_size, size); 1551 mem = kvm_alloc_slot(kml); 1552 mem->as_id = kml->as_id; 1553 mem->memory_size = slot_size; 1554 mem->start_addr = start_addr; 1555 mem->ram_start_offset = ram_start_offset; 1556 mem->ram = ram; 1557 mem->flags = kvm_mem_flags(mr); 1558 mem->guest_memfd = mr->ram_block->guest_memfd; 1559 mem->guest_memfd_offset = (uint8_t*)ram - mr->ram_block->host; 1560 1561 kvm_slot_init_dirty_bitmap(mem); 1562 err = kvm_set_user_memory_region(kml, mem, true); 1563 if (err) { 1564 fprintf(stderr, "%s: error registering slot: %s\n", __func__, 1565 strerror(-err)); 1566 abort(); 1567 } 1568 1569 if (memory_region_has_guest_memfd(mr)) { 1570 err = kvm_set_memory_attributes_private(start_addr, slot_size); 1571 if (err) { 1572 error_report("%s: failed to set memory attribute private: %s", 1573 __func__, strerror(-err)); 1574 exit(1); 1575 } 1576 } 1577 1578 start_addr += slot_size; 1579 ram_start_offset += slot_size; 1580 ram += slot_size; 1581 size -= slot_size; 1582 kml->nr_slots_used++; 1583 } while (size); 1584 } 1585 1586 static void *kvm_dirty_ring_reaper_thread(void *data) 1587 { 1588 KVMState *s = data; 1589 struct KVMDirtyRingReaper *r = &s->reaper; 1590 1591 rcu_register_thread(); 1592 1593 trace_kvm_dirty_ring_reaper("init"); 1594 1595 while (true) { 1596 r->reaper_state = KVM_DIRTY_RING_REAPER_WAIT; 1597 trace_kvm_dirty_ring_reaper("wait"); 1598 /* 1599 * TODO: provide a smarter timeout rather than a constant? 1600 */ 1601 sleep(1); 1602 1603 /* keep sleeping so that dirtylimit not be interfered by reaper */ 1604 if (dirtylimit_in_service()) { 1605 continue; 1606 } 1607 1608 trace_kvm_dirty_ring_reaper("wakeup"); 1609 r->reaper_state = KVM_DIRTY_RING_REAPER_REAPING; 1610 1611 bql_lock(); 1612 kvm_dirty_ring_reap(s, NULL); 1613 bql_unlock(); 1614 1615 r->reaper_iteration++; 1616 } 1617 1618 g_assert_not_reached(); 1619 } 1620 1621 static void kvm_dirty_ring_reaper_init(KVMState *s) 1622 { 1623 struct KVMDirtyRingReaper *r = &s->reaper; 1624 1625 qemu_thread_create(&r->reaper_thr, "kvm-reaper", 1626 kvm_dirty_ring_reaper_thread, 1627 s, QEMU_THREAD_JOINABLE); 1628 } 1629 1630 static int kvm_dirty_ring_init(KVMState *s) 1631 { 1632 uint32_t ring_size = s->kvm_dirty_ring_size; 1633 uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn); 1634 unsigned int capability = KVM_CAP_DIRTY_LOG_RING; 1635 int ret; 1636 1637 s->kvm_dirty_ring_size = 0; 1638 s->kvm_dirty_ring_bytes = 0; 1639 1640 /* Bail if the dirty ring size isn't specified */ 1641 if (!ring_size) { 1642 return 0; 1643 } 1644 1645 /* 1646 * Read the max supported pages. Fall back to dirty logging mode 1647 * if the dirty ring isn't supported. 1648 */ 1649 ret = kvm_vm_check_extension(s, capability); 1650 if (ret <= 0) { 1651 capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL; 1652 ret = kvm_vm_check_extension(s, capability); 1653 } 1654 1655 if (ret <= 0) { 1656 warn_report("KVM dirty ring not available, using bitmap method"); 1657 return 0; 1658 } 1659 1660 if (ring_bytes > ret) { 1661 error_report("KVM dirty ring size %" PRIu32 " too big " 1662 "(maximum is %ld). Please use a smaller value.", 1663 ring_size, (long)ret / sizeof(struct kvm_dirty_gfn)); 1664 return -EINVAL; 1665 } 1666 1667 ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes); 1668 if (ret) { 1669 error_report("Enabling of KVM dirty ring failed: %s. " 1670 "Suggested minimum value is 1024.", strerror(-ret)); 1671 return -EIO; 1672 } 1673 1674 /* Enable the backup bitmap if it is supported */ 1675 ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP); 1676 if (ret > 0) { 1677 ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0); 1678 if (ret) { 1679 error_report("Enabling of KVM dirty ring's backup bitmap failed: " 1680 "%s. ", strerror(-ret)); 1681 return -EIO; 1682 } 1683 1684 s->kvm_dirty_ring_with_bitmap = true; 1685 } 1686 1687 s->kvm_dirty_ring_size = ring_size; 1688 s->kvm_dirty_ring_bytes = ring_bytes; 1689 1690 return 0; 1691 } 1692 1693 static void kvm_region_add(MemoryListener *listener, 1694 MemoryRegionSection *section) 1695 { 1696 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1697 KVMMemoryUpdate *update; 1698 1699 update = g_new0(KVMMemoryUpdate, 1); 1700 update->section = *section; 1701 1702 QSIMPLEQ_INSERT_TAIL(&kml->transaction_add, update, next); 1703 } 1704 1705 static void kvm_region_del(MemoryListener *listener, 1706 MemoryRegionSection *section) 1707 { 1708 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1709 KVMMemoryUpdate *update; 1710 1711 update = g_new0(KVMMemoryUpdate, 1); 1712 update->section = *section; 1713 1714 QSIMPLEQ_INSERT_TAIL(&kml->transaction_del, update, next); 1715 } 1716 1717 static void kvm_region_commit(MemoryListener *listener) 1718 { 1719 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, 1720 listener); 1721 KVMMemoryUpdate *u1, *u2; 1722 bool need_inhibit = false; 1723 1724 if (QSIMPLEQ_EMPTY(&kml->transaction_add) && 1725 QSIMPLEQ_EMPTY(&kml->transaction_del)) { 1726 return; 1727 } 1728 1729 /* 1730 * We have to be careful when regions to add overlap with ranges to remove. 1731 * We have to simulate atomic KVM memslot updates by making sure no ioctl() 1732 * is currently active. 1733 * 1734 * The lists are order by addresses, so it's easy to find overlaps. 1735 */ 1736 u1 = QSIMPLEQ_FIRST(&kml->transaction_del); 1737 u2 = QSIMPLEQ_FIRST(&kml->transaction_add); 1738 while (u1 && u2) { 1739 Range r1, r2; 1740 1741 range_init_nofail(&r1, u1->section.offset_within_address_space, 1742 int128_get64(u1->section.size)); 1743 range_init_nofail(&r2, u2->section.offset_within_address_space, 1744 int128_get64(u2->section.size)); 1745 1746 if (range_overlaps_range(&r1, &r2)) { 1747 need_inhibit = true; 1748 break; 1749 } 1750 if (range_lob(&r1) < range_lob(&r2)) { 1751 u1 = QSIMPLEQ_NEXT(u1, next); 1752 } else { 1753 u2 = QSIMPLEQ_NEXT(u2, next); 1754 } 1755 } 1756 1757 kvm_slots_lock(); 1758 if (need_inhibit) { 1759 accel_ioctl_inhibit_begin(); 1760 } 1761 1762 /* Remove all memslots before adding the new ones. */ 1763 while (!QSIMPLEQ_EMPTY(&kml->transaction_del)) { 1764 u1 = QSIMPLEQ_FIRST(&kml->transaction_del); 1765 QSIMPLEQ_REMOVE_HEAD(&kml->transaction_del, next); 1766 1767 kvm_set_phys_mem(kml, &u1->section, false); 1768 memory_region_unref(u1->section.mr); 1769 1770 g_free(u1); 1771 } 1772 while (!QSIMPLEQ_EMPTY(&kml->transaction_add)) { 1773 u1 = QSIMPLEQ_FIRST(&kml->transaction_add); 1774 QSIMPLEQ_REMOVE_HEAD(&kml->transaction_add, next); 1775 1776 memory_region_ref(u1->section.mr); 1777 kvm_set_phys_mem(kml, &u1->section, true); 1778 1779 g_free(u1); 1780 } 1781 1782 if (need_inhibit) { 1783 accel_ioctl_inhibit_end(); 1784 } 1785 kvm_slots_unlock(); 1786 } 1787 1788 static void kvm_log_sync(MemoryListener *listener, 1789 MemoryRegionSection *section) 1790 { 1791 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1792 1793 kvm_slots_lock(); 1794 kvm_physical_sync_dirty_bitmap(kml, section); 1795 kvm_slots_unlock(); 1796 } 1797 1798 static void kvm_log_sync_global(MemoryListener *l, bool last_stage) 1799 { 1800 KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener); 1801 KVMState *s = kvm_state; 1802 KVMSlot *mem; 1803 int i; 1804 1805 /* Flush all kernel dirty addresses into KVMSlot dirty bitmap */ 1806 kvm_dirty_ring_flush(); 1807 1808 kvm_slots_lock(); 1809 for (i = 0; i < kml->nr_slots_allocated; i++) { 1810 mem = &kml->slots[i]; 1811 if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1812 kvm_slot_sync_dirty_pages(mem); 1813 1814 if (s->kvm_dirty_ring_with_bitmap && last_stage && 1815 kvm_slot_get_dirty_log(s, mem)) { 1816 kvm_slot_sync_dirty_pages(mem); 1817 } 1818 1819 /* 1820 * This is not needed by KVM_GET_DIRTY_LOG because the 1821 * ioctl will unconditionally overwrite the whole region. 1822 * However kvm dirty ring has no such side effect. 1823 */ 1824 kvm_slot_reset_dirty_pages(mem); 1825 } 1826 } 1827 kvm_slots_unlock(); 1828 } 1829 1830 static void kvm_log_clear(MemoryListener *listener, 1831 MemoryRegionSection *section) 1832 { 1833 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1834 int r; 1835 1836 r = kvm_physical_log_clear(kml, section); 1837 if (r < 0) { 1838 error_report_once("%s: kvm log clear failed: mr=%s " 1839 "offset=%"HWADDR_PRIx" size=%"PRIx64, __func__, 1840 section->mr->name, section->offset_within_region, 1841 int128_get64(section->size)); 1842 abort(); 1843 } 1844 } 1845 1846 static void kvm_mem_ioeventfd_add(MemoryListener *listener, 1847 MemoryRegionSection *section, 1848 bool match_data, uint64_t data, 1849 EventNotifier *e) 1850 { 1851 int fd = event_notifier_get_fd(e); 1852 int r; 1853 1854 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 1855 data, true, int128_get64(section->size), 1856 match_data); 1857 if (r < 0) { 1858 fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", 1859 __func__, strerror(-r), -r); 1860 abort(); 1861 } 1862 } 1863 1864 static void kvm_mem_ioeventfd_del(MemoryListener *listener, 1865 MemoryRegionSection *section, 1866 bool match_data, uint64_t data, 1867 EventNotifier *e) 1868 { 1869 int fd = event_notifier_get_fd(e); 1870 int r; 1871 1872 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 1873 data, false, int128_get64(section->size), 1874 match_data); 1875 if (r < 0) { 1876 fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", 1877 __func__, strerror(-r), -r); 1878 abort(); 1879 } 1880 } 1881 1882 static void kvm_io_ioeventfd_add(MemoryListener *listener, 1883 MemoryRegionSection *section, 1884 bool match_data, uint64_t data, 1885 EventNotifier *e) 1886 { 1887 int fd = event_notifier_get_fd(e); 1888 int r; 1889 1890 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 1891 data, true, int128_get64(section->size), 1892 match_data); 1893 if (r < 0) { 1894 fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", 1895 __func__, strerror(-r), -r); 1896 abort(); 1897 } 1898 } 1899 1900 static void kvm_io_ioeventfd_del(MemoryListener *listener, 1901 MemoryRegionSection *section, 1902 bool match_data, uint64_t data, 1903 EventNotifier *e) 1904 1905 { 1906 int fd = event_notifier_get_fd(e); 1907 int r; 1908 1909 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 1910 data, false, int128_get64(section->size), 1911 match_data); 1912 if (r < 0) { 1913 fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", 1914 __func__, strerror(-r), -r); 1915 abort(); 1916 } 1917 } 1918 1919 void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, 1920 AddressSpace *as, int as_id, const char *name) 1921 { 1922 int i; 1923 1924 kml->as_id = as_id; 1925 1926 kvm_slots_grow(kml, KVM_MEMSLOTS_NR_ALLOC_DEFAULT); 1927 1928 QSIMPLEQ_INIT(&kml->transaction_add); 1929 QSIMPLEQ_INIT(&kml->transaction_del); 1930 1931 kml->listener.region_add = kvm_region_add; 1932 kml->listener.region_del = kvm_region_del; 1933 kml->listener.commit = kvm_region_commit; 1934 kml->listener.log_start = kvm_log_start; 1935 kml->listener.log_stop = kvm_log_stop; 1936 kml->listener.priority = MEMORY_LISTENER_PRIORITY_ACCEL; 1937 kml->listener.name = name; 1938 1939 if (s->kvm_dirty_ring_size) { 1940 kml->listener.log_sync_global = kvm_log_sync_global; 1941 } else { 1942 kml->listener.log_sync = kvm_log_sync; 1943 kml->listener.log_clear = kvm_log_clear; 1944 } 1945 1946 memory_listener_register(&kml->listener, as); 1947 1948 for (i = 0; i < s->nr_as; ++i) { 1949 if (!s->as[i].as) { 1950 s->as[i].as = as; 1951 s->as[i].ml = kml; 1952 break; 1953 } 1954 } 1955 } 1956 1957 static MemoryListener kvm_io_listener = { 1958 .name = "kvm-io", 1959 .coalesced_io_add = kvm_coalesce_pio_add, 1960 .coalesced_io_del = kvm_coalesce_pio_del, 1961 .eventfd_add = kvm_io_ioeventfd_add, 1962 .eventfd_del = kvm_io_ioeventfd_del, 1963 .priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND, 1964 }; 1965 1966 int kvm_set_irq(KVMState *s, int irq, int level) 1967 { 1968 struct kvm_irq_level event; 1969 int ret; 1970 1971 assert(kvm_async_interrupts_enabled()); 1972 1973 event.level = level; 1974 event.irq = irq; 1975 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event); 1976 if (ret < 0) { 1977 perror("kvm_set_irq"); 1978 abort(); 1979 } 1980 1981 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status; 1982 } 1983 1984 #ifdef KVM_CAP_IRQ_ROUTING 1985 typedef struct KVMMSIRoute { 1986 struct kvm_irq_routing_entry kroute; 1987 QTAILQ_ENTRY(KVMMSIRoute) entry; 1988 } KVMMSIRoute; 1989 1990 static void set_gsi(KVMState *s, unsigned int gsi) 1991 { 1992 set_bit(gsi, s->used_gsi_bitmap); 1993 } 1994 1995 static void clear_gsi(KVMState *s, unsigned int gsi) 1996 { 1997 clear_bit(gsi, s->used_gsi_bitmap); 1998 } 1999 2000 void kvm_init_irq_routing(KVMState *s) 2001 { 2002 int gsi_count; 2003 2004 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1; 2005 if (gsi_count > 0) { 2006 /* Round up so we can search ints using ffs */ 2007 s->used_gsi_bitmap = bitmap_new(gsi_count); 2008 s->gsi_count = gsi_count; 2009 } 2010 2011 s->irq_routes = g_malloc0(sizeof(*s->irq_routes)); 2012 s->nr_allocated_irq_routes = 0; 2013 2014 kvm_arch_init_irq_routing(s); 2015 } 2016 2017 void kvm_irqchip_commit_routes(KVMState *s) 2018 { 2019 int ret; 2020 2021 if (kvm_gsi_direct_mapping()) { 2022 return; 2023 } 2024 2025 if (!kvm_gsi_routing_enabled()) { 2026 return; 2027 } 2028 2029 s->irq_routes->flags = 0; 2030 trace_kvm_irqchip_commit_routes(); 2031 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes); 2032 assert(ret == 0); 2033 } 2034 2035 void kvm_add_routing_entry(KVMState *s, 2036 struct kvm_irq_routing_entry *entry) 2037 { 2038 struct kvm_irq_routing_entry *new; 2039 int n, size; 2040 2041 if (s->irq_routes->nr == s->nr_allocated_irq_routes) { 2042 n = s->nr_allocated_irq_routes * 2; 2043 if (n < 64) { 2044 n = 64; 2045 } 2046 size = sizeof(struct kvm_irq_routing); 2047 size += n * sizeof(*new); 2048 s->irq_routes = g_realloc(s->irq_routes, size); 2049 s->nr_allocated_irq_routes = n; 2050 } 2051 n = s->irq_routes->nr++; 2052 new = &s->irq_routes->entries[n]; 2053 2054 *new = *entry; 2055 2056 set_gsi(s, entry->gsi); 2057 } 2058 2059 static int kvm_update_routing_entry(KVMState *s, 2060 struct kvm_irq_routing_entry *new_entry) 2061 { 2062 struct kvm_irq_routing_entry *entry; 2063 int n; 2064 2065 for (n = 0; n < s->irq_routes->nr; n++) { 2066 entry = &s->irq_routes->entries[n]; 2067 if (entry->gsi != new_entry->gsi) { 2068 continue; 2069 } 2070 2071 if(!memcmp(entry, new_entry, sizeof *entry)) { 2072 return 0; 2073 } 2074 2075 *entry = *new_entry; 2076 2077 return 0; 2078 } 2079 2080 return -ESRCH; 2081 } 2082 2083 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin) 2084 { 2085 struct kvm_irq_routing_entry e = {}; 2086 2087 assert(pin < s->gsi_count); 2088 2089 e.gsi = irq; 2090 e.type = KVM_IRQ_ROUTING_IRQCHIP; 2091 e.flags = 0; 2092 e.u.irqchip.irqchip = irqchip; 2093 e.u.irqchip.pin = pin; 2094 kvm_add_routing_entry(s, &e); 2095 } 2096 2097 void kvm_irqchip_release_virq(KVMState *s, int virq) 2098 { 2099 struct kvm_irq_routing_entry *e; 2100 int i; 2101 2102 if (kvm_gsi_direct_mapping()) { 2103 return; 2104 } 2105 2106 for (i = 0; i < s->irq_routes->nr; i++) { 2107 e = &s->irq_routes->entries[i]; 2108 if (e->gsi == virq) { 2109 s->irq_routes->nr--; 2110 *e = s->irq_routes->entries[s->irq_routes->nr]; 2111 } 2112 } 2113 clear_gsi(s, virq); 2114 kvm_arch_release_virq_post(virq); 2115 trace_kvm_irqchip_release_virq(virq); 2116 } 2117 2118 void kvm_irqchip_add_change_notifier(Notifier *n) 2119 { 2120 notifier_list_add(&kvm_irqchip_change_notifiers, n); 2121 } 2122 2123 void kvm_irqchip_remove_change_notifier(Notifier *n) 2124 { 2125 notifier_remove(n); 2126 } 2127 2128 void kvm_irqchip_change_notify(void) 2129 { 2130 notifier_list_notify(&kvm_irqchip_change_notifiers, NULL); 2131 } 2132 2133 int kvm_irqchip_get_virq(KVMState *s) 2134 { 2135 int next_virq; 2136 2137 /* Return the lowest unused GSI in the bitmap */ 2138 next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count); 2139 if (next_virq >= s->gsi_count) { 2140 return -ENOSPC; 2141 } else { 2142 return next_virq; 2143 } 2144 } 2145 2146 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 2147 { 2148 struct kvm_msi msi; 2149 2150 msi.address_lo = (uint32_t)msg.address; 2151 msi.address_hi = msg.address >> 32; 2152 msi.data = le32_to_cpu(msg.data); 2153 msi.flags = 0; 2154 memset(msi.pad, 0, sizeof(msi.pad)); 2155 2156 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi); 2157 } 2158 2159 int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev) 2160 { 2161 struct kvm_irq_routing_entry kroute = {}; 2162 int virq; 2163 KVMState *s = c->s; 2164 MSIMessage msg = {0, 0}; 2165 2166 if (pci_available && dev) { 2167 msg = pci_get_msi_message(dev, vector); 2168 } 2169 2170 if (kvm_gsi_direct_mapping()) { 2171 return kvm_arch_msi_data_to_gsi(msg.data); 2172 } 2173 2174 if (!kvm_gsi_routing_enabled()) { 2175 return -ENOSYS; 2176 } 2177 2178 virq = kvm_irqchip_get_virq(s); 2179 if (virq < 0) { 2180 return virq; 2181 } 2182 2183 kroute.gsi = virq; 2184 kroute.type = KVM_IRQ_ROUTING_MSI; 2185 kroute.flags = 0; 2186 kroute.u.msi.address_lo = (uint32_t)msg.address; 2187 kroute.u.msi.address_hi = msg.address >> 32; 2188 kroute.u.msi.data = le32_to_cpu(msg.data); 2189 if (pci_available && kvm_msi_devid_required()) { 2190 kroute.flags = KVM_MSI_VALID_DEVID; 2191 kroute.u.msi.devid = pci_requester_id(dev); 2192 } 2193 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 2194 kvm_irqchip_release_virq(s, virq); 2195 return -EINVAL; 2196 } 2197 2198 if (s->irq_routes->nr < s->gsi_count) { 2199 trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A", 2200 vector, virq); 2201 2202 kvm_add_routing_entry(s, &kroute); 2203 kvm_arch_add_msi_route_post(&kroute, vector, dev); 2204 c->changes++; 2205 } else { 2206 kvm_irqchip_release_virq(s, virq); 2207 return -ENOSPC; 2208 } 2209 2210 return virq; 2211 } 2212 2213 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, 2214 PCIDevice *dev) 2215 { 2216 struct kvm_irq_routing_entry kroute = {}; 2217 2218 if (kvm_gsi_direct_mapping()) { 2219 return 0; 2220 } 2221 2222 if (!kvm_irqchip_in_kernel()) { 2223 return -ENOSYS; 2224 } 2225 2226 kroute.gsi = virq; 2227 kroute.type = KVM_IRQ_ROUTING_MSI; 2228 kroute.flags = 0; 2229 kroute.u.msi.address_lo = (uint32_t)msg.address; 2230 kroute.u.msi.address_hi = msg.address >> 32; 2231 kroute.u.msi.data = le32_to_cpu(msg.data); 2232 if (pci_available && kvm_msi_devid_required()) { 2233 kroute.flags = KVM_MSI_VALID_DEVID; 2234 kroute.u.msi.devid = pci_requester_id(dev); 2235 } 2236 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 2237 return -EINVAL; 2238 } 2239 2240 trace_kvm_irqchip_update_msi_route(virq); 2241 2242 return kvm_update_routing_entry(s, &kroute); 2243 } 2244 2245 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, 2246 EventNotifier *resample, int virq, 2247 bool assign) 2248 { 2249 int fd = event_notifier_get_fd(event); 2250 int rfd = resample ? event_notifier_get_fd(resample) : -1; 2251 2252 struct kvm_irqfd irqfd = { 2253 .fd = fd, 2254 .gsi = virq, 2255 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN, 2256 }; 2257 2258 if (rfd != -1) { 2259 assert(assign); 2260 if (kvm_irqchip_is_split()) { 2261 /* 2262 * When the slow irqchip (e.g. IOAPIC) is in the 2263 * userspace, KVM kernel resamplefd will not work because 2264 * the EOI of the interrupt will be delivered to userspace 2265 * instead, so the KVM kernel resamplefd kick will be 2266 * skipped. The userspace here mimics what the kernel 2267 * provides with resamplefd, remember the resamplefd and 2268 * kick it when we receive EOI of this IRQ. 2269 * 2270 * This is hackery because IOAPIC is mostly bypassed 2271 * (except EOI broadcasts) when irqfd is used. However 2272 * this can bring much performance back for split irqchip 2273 * with INTx IRQs (for VFIO, this gives 93% perf of the 2274 * full fast path, which is 46% perf boost comparing to 2275 * the INTx slow path). 2276 */ 2277 kvm_resample_fd_insert(virq, resample); 2278 } else { 2279 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE; 2280 irqfd.resamplefd = rfd; 2281 } 2282 } else if (!assign) { 2283 if (kvm_irqchip_is_split()) { 2284 kvm_resample_fd_remove(virq); 2285 } 2286 } 2287 2288 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd); 2289 } 2290 2291 #else /* !KVM_CAP_IRQ_ROUTING */ 2292 2293 void kvm_init_irq_routing(KVMState *s) 2294 { 2295 } 2296 2297 void kvm_irqchip_release_virq(KVMState *s, int virq) 2298 { 2299 } 2300 2301 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 2302 { 2303 abort(); 2304 } 2305 2306 int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev) 2307 { 2308 return -ENOSYS; 2309 } 2310 2311 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 2312 { 2313 return -ENOSYS; 2314 } 2315 2316 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 2317 { 2318 return -ENOSYS; 2319 } 2320 2321 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, 2322 EventNotifier *resample, int virq, 2323 bool assign) 2324 { 2325 abort(); 2326 } 2327 2328 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) 2329 { 2330 return -ENOSYS; 2331 } 2332 #endif /* !KVM_CAP_IRQ_ROUTING */ 2333 2334 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 2335 EventNotifier *rn, int virq) 2336 { 2337 return kvm_irqchip_assign_irqfd(s, n, rn, virq, true); 2338 } 2339 2340 int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 2341 int virq) 2342 { 2343 return kvm_irqchip_assign_irqfd(s, n, NULL, virq, false); 2344 } 2345 2346 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, 2347 EventNotifier *rn, qemu_irq irq) 2348 { 2349 gpointer key, gsi; 2350 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 2351 2352 if (!found) { 2353 return -ENXIO; 2354 } 2355 return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi)); 2356 } 2357 2358 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, 2359 qemu_irq irq) 2360 { 2361 gpointer key, gsi; 2362 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 2363 2364 if (!found) { 2365 return -ENXIO; 2366 } 2367 return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi)); 2368 } 2369 2370 void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi) 2371 { 2372 g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi)); 2373 } 2374 2375 static void kvm_irqchip_create(KVMState *s) 2376 { 2377 int ret; 2378 2379 assert(s->kernel_irqchip_split != ON_OFF_AUTO_AUTO); 2380 if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) { 2381 ; 2382 } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) { 2383 ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0); 2384 if (ret < 0) { 2385 fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret)); 2386 exit(1); 2387 } 2388 } else { 2389 return; 2390 } 2391 2392 if (kvm_check_extension(s, KVM_CAP_IRQFD) <= 0) { 2393 fprintf(stderr, "kvm: irqfd not implemented\n"); 2394 exit(1); 2395 } 2396 2397 /* First probe and see if there's a arch-specific hook to create the 2398 * in-kernel irqchip for us */ 2399 ret = kvm_arch_irqchip_create(s); 2400 if (ret == 0) { 2401 if (s->kernel_irqchip_split == ON_OFF_AUTO_ON) { 2402 error_report("Split IRQ chip mode not supported."); 2403 exit(1); 2404 } else { 2405 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP); 2406 } 2407 } 2408 if (ret < 0) { 2409 fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret)); 2410 exit(1); 2411 } 2412 2413 kvm_kernel_irqchip = true; 2414 /* If we have an in-kernel IRQ chip then we must have asynchronous 2415 * interrupt delivery (though the reverse is not necessarily true) 2416 */ 2417 kvm_async_interrupts_allowed = true; 2418 kvm_halt_in_kernel_allowed = true; 2419 2420 kvm_init_irq_routing(s); 2421 2422 s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal); 2423 } 2424 2425 /* Find number of supported CPUs using the recommended 2426 * procedure from the kernel API documentation to cope with 2427 * older kernels that may be missing capabilities. 2428 */ 2429 static int kvm_recommended_vcpus(KVMState *s) 2430 { 2431 int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS); 2432 return (ret) ? ret : 4; 2433 } 2434 2435 static int kvm_max_vcpus(KVMState *s) 2436 { 2437 int ret = kvm_vm_check_extension(s, KVM_CAP_MAX_VCPUS); 2438 return (ret) ? ret : kvm_recommended_vcpus(s); 2439 } 2440 2441 static int kvm_max_vcpu_id(KVMState *s) 2442 { 2443 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID); 2444 return (ret) ? ret : kvm_max_vcpus(s); 2445 } 2446 2447 bool kvm_vcpu_id_is_valid(int vcpu_id) 2448 { 2449 KVMState *s = KVM_STATE(current_accel()); 2450 return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s); 2451 } 2452 2453 bool kvm_dirty_ring_enabled(void) 2454 { 2455 return kvm_state && kvm_state->kvm_dirty_ring_size; 2456 } 2457 2458 static void query_stats_cb(StatsResultList **result, StatsTarget target, 2459 strList *names, strList *targets, Error **errp); 2460 static void query_stats_schemas_cb(StatsSchemaList **result, Error **errp); 2461 2462 uint32_t kvm_dirty_ring_size(void) 2463 { 2464 return kvm_state->kvm_dirty_ring_size; 2465 } 2466 2467 static int do_kvm_create_vm(MachineState *ms, int type) 2468 { 2469 KVMState *s; 2470 int ret; 2471 2472 s = KVM_STATE(ms->accelerator); 2473 2474 do { 2475 ret = kvm_ioctl(s, KVM_CREATE_VM, type); 2476 } while (ret == -EINTR); 2477 2478 if (ret < 0) { 2479 error_report("ioctl(KVM_CREATE_VM) failed: %s", strerror(-ret)); 2480 2481 #ifdef TARGET_S390X 2482 if (ret == -EINVAL) { 2483 error_printf("Host kernel setup problem detected." 2484 " Please verify:\n"); 2485 error_printf("- for kernels supporting the" 2486 " switch_amode or user_mode parameters, whether"); 2487 error_printf(" user space is running in primary address space\n"); 2488 error_printf("- for kernels supporting the vm.allocate_pgste" 2489 " sysctl, whether it is enabled\n"); 2490 } 2491 #elif defined(TARGET_PPC) 2492 if (ret == -EINVAL) { 2493 error_printf("PPC KVM module is not loaded. Try modprobe kvm_%s.\n", 2494 (type == 2) ? "pr" : "hv"); 2495 } 2496 #endif 2497 } 2498 2499 return ret; 2500 } 2501 2502 static int find_kvm_machine_type(MachineState *ms) 2503 { 2504 MachineClass *mc = MACHINE_GET_CLASS(ms); 2505 int type; 2506 2507 if (object_property_find(OBJECT(current_machine), "kvm-type")) { 2508 g_autofree char *kvm_type; 2509 kvm_type = object_property_get_str(OBJECT(current_machine), 2510 "kvm-type", 2511 &error_abort); 2512 type = mc->kvm_type(ms, kvm_type); 2513 } else if (mc->kvm_type) { 2514 type = mc->kvm_type(ms, NULL); 2515 } else { 2516 type = kvm_arch_get_default_type(ms); 2517 } 2518 return type; 2519 } 2520 2521 static int kvm_setup_dirty_ring(KVMState *s) 2522 { 2523 uint64_t dirty_log_manual_caps; 2524 int ret; 2525 2526 /* 2527 * Enable KVM dirty ring if supported, otherwise fall back to 2528 * dirty logging mode 2529 */ 2530 ret = kvm_dirty_ring_init(s); 2531 if (ret < 0) { 2532 return ret; 2533 } 2534 2535 /* 2536 * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 is not needed when dirty ring is 2537 * enabled. More importantly, KVM_DIRTY_LOG_INITIALLY_SET will assume no 2538 * page is wr-protected initially, which is against how kvm dirty ring is 2539 * usage - kvm dirty ring requires all pages are wr-protected at the very 2540 * beginning. Enabling this feature for dirty ring causes data corruption. 2541 * 2542 * TODO: Without KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and kvm clear dirty log, 2543 * we may expect a higher stall time when starting the migration. In the 2544 * future we can enable KVM_CLEAR_DIRTY_LOG to work with dirty ring too: 2545 * instead of clearing dirty bit, it can be a way to explicitly wr-protect 2546 * guest pages. 2547 */ 2548 if (!s->kvm_dirty_ring_size) { 2549 dirty_log_manual_caps = 2550 kvm_check_extension(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); 2551 dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | 2552 KVM_DIRTY_LOG_INITIALLY_SET); 2553 s->manual_dirty_log_protect = dirty_log_manual_caps; 2554 if (dirty_log_manual_caps) { 2555 ret = kvm_vm_enable_cap(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 0, 2556 dirty_log_manual_caps); 2557 if (ret) { 2558 warn_report("Trying to enable capability %"PRIu64" of " 2559 "KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 but failed. " 2560 "Falling back to the legacy mode. ", 2561 dirty_log_manual_caps); 2562 s->manual_dirty_log_protect = 0; 2563 } 2564 } 2565 } 2566 2567 return 0; 2568 } 2569 2570 static int kvm_init(MachineState *ms) 2571 { 2572 MachineClass *mc = MACHINE_GET_CLASS(ms); 2573 static const char upgrade_note[] = 2574 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n" 2575 "(see http://sourceforge.net/projects/kvm).\n"; 2576 const struct { 2577 const char *name; 2578 int num; 2579 } num_cpus[] = { 2580 { "SMP", ms->smp.cpus }, 2581 { "hotpluggable", ms->smp.max_cpus }, 2582 { /* end of list */ } 2583 }, *nc = num_cpus; 2584 int soft_vcpus_limit, hard_vcpus_limit; 2585 KVMState *s; 2586 const KVMCapabilityInfo *missing_cap; 2587 int ret; 2588 int type; 2589 2590 qemu_mutex_init(&kml_slots_lock); 2591 2592 s = KVM_STATE(ms->accelerator); 2593 2594 /* 2595 * On systems where the kernel can support different base page 2596 * sizes, host page size may be different from TARGET_PAGE_SIZE, 2597 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum 2598 * page size for the system though. 2599 */ 2600 assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size()); 2601 2602 s->sigmask_len = 8; 2603 accel_blocker_init(); 2604 2605 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG 2606 QTAILQ_INIT(&s->kvm_sw_breakpoints); 2607 #endif 2608 QLIST_INIT(&s->kvm_parked_vcpus); 2609 s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR); 2610 if (s->fd == -1) { 2611 error_report("Could not access KVM kernel module: %m"); 2612 ret = -errno; 2613 goto err; 2614 } 2615 2616 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0); 2617 if (ret < KVM_API_VERSION) { 2618 if (ret >= 0) { 2619 ret = -EINVAL; 2620 } 2621 error_report("kvm version too old"); 2622 goto err; 2623 } 2624 2625 if (ret > KVM_API_VERSION) { 2626 ret = -EINVAL; 2627 error_report("kvm version not supported"); 2628 goto err; 2629 } 2630 2631 kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT); 2632 s->nr_slots_max = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS); 2633 2634 /* If unspecified, use the default value */ 2635 if (!s->nr_slots_max) { 2636 s->nr_slots_max = KVM_MEMSLOTS_NR_MAX_DEFAULT; 2637 } 2638 2639 type = find_kvm_machine_type(ms); 2640 if (type < 0) { 2641 ret = -EINVAL; 2642 goto err; 2643 } 2644 2645 ret = do_kvm_create_vm(ms, type); 2646 if (ret < 0) { 2647 goto err; 2648 } 2649 2650 s->vmfd = ret; 2651 2652 s->nr_as = kvm_vm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE); 2653 if (s->nr_as <= 1) { 2654 s->nr_as = 1; 2655 } 2656 s->as = g_new0(struct KVMAs, s->nr_as); 2657 2658 /* check the vcpu limits */ 2659 soft_vcpus_limit = kvm_recommended_vcpus(s); 2660 hard_vcpus_limit = kvm_max_vcpus(s); 2661 2662 while (nc->name) { 2663 if (nc->num > soft_vcpus_limit) { 2664 warn_report("Number of %s cpus requested (%d) exceeds " 2665 "the recommended cpus supported by KVM (%d)", 2666 nc->name, nc->num, soft_vcpus_limit); 2667 2668 if (nc->num > hard_vcpus_limit) { 2669 error_report("Number of %s cpus requested (%d) exceeds " 2670 "the maximum cpus supported by KVM (%d)", 2671 nc->name, nc->num, hard_vcpus_limit); 2672 exit(1); 2673 } 2674 } 2675 nc++; 2676 } 2677 2678 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites); 2679 if (!missing_cap) { 2680 missing_cap = 2681 kvm_check_extension_list(s, kvm_arch_required_capabilities); 2682 } 2683 if (missing_cap) { 2684 ret = -EINVAL; 2685 error_report("kvm does not support %s", missing_cap->name); 2686 error_printf("%s", upgrade_note); 2687 goto err; 2688 } 2689 2690 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO); 2691 s->coalesced_pio = s->coalesced_mmio && 2692 kvm_check_extension(s, KVM_CAP_COALESCED_PIO); 2693 2694 ret = kvm_setup_dirty_ring(s); 2695 if (ret < 0) { 2696 goto err; 2697 } 2698 2699 #ifdef KVM_CAP_VCPU_EVENTS 2700 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS); 2701 #endif 2702 s->max_nested_state_len = kvm_check_extension(s, KVM_CAP_NESTED_STATE); 2703 2704 s->irq_set_ioctl = KVM_IRQ_LINE; 2705 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) { 2706 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS; 2707 } 2708 2709 kvm_readonly_mem_allowed = 2710 (kvm_vm_check_extension(s, KVM_CAP_READONLY_MEM) > 0); 2711 2712 kvm_resamplefds_allowed = 2713 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0); 2714 2715 kvm_vm_attributes_allowed = 2716 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0); 2717 2718 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG 2719 kvm_has_guest_debug = 2720 (kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG) > 0); 2721 #endif 2722 2723 kvm_sstep_flags = 0; 2724 if (kvm_has_guest_debug) { 2725 kvm_sstep_flags = SSTEP_ENABLE; 2726 2727 #if defined TARGET_KVM_HAVE_GUEST_DEBUG 2728 int guest_debug_flags = 2729 kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG2); 2730 2731 if (guest_debug_flags & KVM_GUESTDBG_BLOCKIRQ) { 2732 kvm_sstep_flags |= SSTEP_NOIRQ; 2733 } 2734 #endif 2735 } 2736 2737 kvm_state = s; 2738 2739 ret = kvm_arch_init(ms, s); 2740 if (ret < 0) { 2741 goto err; 2742 } 2743 2744 kvm_supported_memory_attributes = kvm_vm_check_extension(s, KVM_CAP_MEMORY_ATTRIBUTES); 2745 kvm_guest_memfd_supported = 2746 kvm_check_extension(s, KVM_CAP_GUEST_MEMFD) && 2747 kvm_check_extension(s, KVM_CAP_USER_MEMORY2) && 2748 (kvm_supported_memory_attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE); 2749 kvm_pre_fault_memory_supported = kvm_vm_check_extension(s, KVM_CAP_PRE_FAULT_MEMORY); 2750 2751 if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) { 2752 s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 2753 } 2754 2755 qemu_register_reset(kvm_unpoison_all, NULL); 2756 2757 if (s->kernel_irqchip_allowed) { 2758 kvm_irqchip_create(s); 2759 } 2760 2761 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add; 2762 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del; 2763 s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region; 2764 s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region; 2765 2766 kvm_memory_listener_register(s, &s->memory_listener, 2767 &address_space_memory, 0, "kvm-memory"); 2768 memory_listener_register(&kvm_io_listener, 2769 &address_space_io); 2770 2771 s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU); 2772 if (!s->sync_mmu) { 2773 ret = ram_block_discard_disable(true); 2774 assert(!ret); 2775 } 2776 2777 if (s->kvm_dirty_ring_size) { 2778 kvm_dirty_ring_reaper_init(s); 2779 } 2780 2781 if (kvm_check_extension(kvm_state, KVM_CAP_BINARY_STATS_FD)) { 2782 add_stats_callbacks(STATS_PROVIDER_KVM, query_stats_cb, 2783 query_stats_schemas_cb); 2784 } 2785 2786 return 0; 2787 2788 err: 2789 assert(ret < 0); 2790 if (s->vmfd >= 0) { 2791 close(s->vmfd); 2792 } 2793 if (s->fd != -1) { 2794 close(s->fd); 2795 } 2796 g_free(s->as); 2797 g_free(s->memory_listener.slots); 2798 2799 return ret; 2800 } 2801 2802 void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len) 2803 { 2804 s->sigmask_len = sigmask_len; 2805 } 2806 2807 static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction, 2808 int size, uint32_t count) 2809 { 2810 int i; 2811 uint8_t *ptr = data; 2812 2813 for (i = 0; i < count; i++) { 2814 address_space_rw(&address_space_io, port, attrs, 2815 ptr, size, 2816 direction == KVM_EXIT_IO_OUT); 2817 ptr += size; 2818 } 2819 } 2820 2821 static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run) 2822 { 2823 int i; 2824 2825 fprintf(stderr, "KVM internal error. Suberror: %d\n", 2826 run->internal.suberror); 2827 2828 for (i = 0; i < run->internal.ndata; ++i) { 2829 fprintf(stderr, "extra data[%d]: 0x%016"PRIx64"\n", 2830 i, (uint64_t)run->internal.data[i]); 2831 } 2832 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) { 2833 fprintf(stderr, "emulation failure\n"); 2834 if (!kvm_arch_stop_on_emulation_error(cpu)) { 2835 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2836 return EXCP_INTERRUPT; 2837 } 2838 } 2839 /* FIXME: Should trigger a qmp message to let management know 2840 * something went wrong. 2841 */ 2842 return -1; 2843 } 2844 2845 void kvm_flush_coalesced_mmio_buffer(void) 2846 { 2847 KVMState *s = kvm_state; 2848 2849 if (!s || s->coalesced_flush_in_progress) { 2850 return; 2851 } 2852 2853 s->coalesced_flush_in_progress = true; 2854 2855 if (s->coalesced_mmio_ring) { 2856 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring; 2857 while (ring->first != ring->last) { 2858 struct kvm_coalesced_mmio *ent; 2859 2860 ent = &ring->coalesced_mmio[ring->first]; 2861 2862 if (ent->pio == 1) { 2863 address_space_write(&address_space_io, ent->phys_addr, 2864 MEMTXATTRS_UNSPECIFIED, ent->data, 2865 ent->len); 2866 } else { 2867 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len); 2868 } 2869 smp_wmb(); 2870 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX; 2871 } 2872 } 2873 2874 s->coalesced_flush_in_progress = false; 2875 } 2876 2877 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) 2878 { 2879 if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) { 2880 Error *err = NULL; 2881 int ret = kvm_arch_get_registers(cpu, &err); 2882 if (ret) { 2883 if (err) { 2884 error_reportf_err(err, "Failed to synchronize CPU state: "); 2885 } else { 2886 error_report("Failed to get registers: %s", strerror(-ret)); 2887 } 2888 2889 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2890 vm_stop(RUN_STATE_INTERNAL_ERROR); 2891 } 2892 2893 cpu->vcpu_dirty = true; 2894 } 2895 } 2896 2897 void kvm_cpu_synchronize_state(CPUState *cpu) 2898 { 2899 if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) { 2900 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL); 2901 } 2902 } 2903 2904 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) 2905 { 2906 Error *err = NULL; 2907 int ret = kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE, &err); 2908 if (ret) { 2909 if (err) { 2910 error_reportf_err(err, "Restoring resisters after reset: "); 2911 } else { 2912 error_report("Failed to put registers after reset: %s", 2913 strerror(-ret)); 2914 } 2915 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2916 vm_stop(RUN_STATE_INTERNAL_ERROR); 2917 } 2918 2919 cpu->vcpu_dirty = false; 2920 } 2921 2922 void kvm_cpu_synchronize_post_reset(CPUState *cpu) 2923 { 2924 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); 2925 2926 if (cpu == first_cpu) { 2927 kvm_reset_parked_vcpus(kvm_state); 2928 } 2929 } 2930 2931 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) 2932 { 2933 Error *err = NULL; 2934 int ret = kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE, &err); 2935 if (ret) { 2936 if (err) { 2937 error_reportf_err(err, "Putting registers after init: "); 2938 } else { 2939 error_report("Failed to put registers after init: %s", 2940 strerror(-ret)); 2941 } 2942 exit(1); 2943 } 2944 2945 cpu->vcpu_dirty = false; 2946 } 2947 2948 void kvm_cpu_synchronize_post_init(CPUState *cpu) 2949 { 2950 if (!kvm_state->guest_state_protected) { 2951 /* 2952 * This runs before the machine_init_done notifiers, and is the last 2953 * opportunity to synchronize the state of confidential guests. 2954 */ 2955 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); 2956 } 2957 } 2958 2959 static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) 2960 { 2961 cpu->vcpu_dirty = true; 2962 } 2963 2964 void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu) 2965 { 2966 run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); 2967 } 2968 2969 #ifdef KVM_HAVE_MCE_INJECTION 2970 static __thread void *pending_sigbus_addr; 2971 static __thread int pending_sigbus_code; 2972 static __thread bool have_sigbus_pending; 2973 #endif 2974 2975 static void kvm_cpu_kick(CPUState *cpu) 2976 { 2977 qatomic_set(&cpu->kvm_run->immediate_exit, 1); 2978 } 2979 2980 static void kvm_cpu_kick_self(void) 2981 { 2982 if (kvm_immediate_exit) { 2983 kvm_cpu_kick(current_cpu); 2984 } else { 2985 qemu_cpu_kick_self(); 2986 } 2987 } 2988 2989 static void kvm_eat_signals(CPUState *cpu) 2990 { 2991 struct timespec ts = { 0, 0 }; 2992 siginfo_t siginfo; 2993 sigset_t waitset; 2994 sigset_t chkset; 2995 int r; 2996 2997 if (kvm_immediate_exit) { 2998 qatomic_set(&cpu->kvm_run->immediate_exit, 0); 2999 /* Write kvm_run->immediate_exit before the cpu->exit_request 3000 * write in kvm_cpu_exec. 3001 */ 3002 smp_wmb(); 3003 return; 3004 } 3005 3006 sigemptyset(&waitset); 3007 sigaddset(&waitset, SIG_IPI); 3008 3009 do { 3010 r = sigtimedwait(&waitset, &siginfo, &ts); 3011 if (r == -1 && !(errno == EAGAIN || errno == EINTR)) { 3012 perror("sigtimedwait"); 3013 exit(1); 3014 } 3015 3016 r = sigpending(&chkset); 3017 if (r == -1) { 3018 perror("sigpending"); 3019 exit(1); 3020 } 3021 } while (sigismember(&chkset, SIG_IPI)); 3022 } 3023 3024 int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private) 3025 { 3026 MemoryRegionSection section; 3027 ram_addr_t offset; 3028 MemoryRegion *mr; 3029 RAMBlock *rb; 3030 void *addr; 3031 int ret = -EINVAL; 3032 3033 trace_kvm_convert_memory(start, size, to_private ? "shared_to_private" : "private_to_shared"); 3034 3035 if (!QEMU_PTR_IS_ALIGNED(start, qemu_real_host_page_size()) || 3036 !QEMU_PTR_IS_ALIGNED(size, qemu_real_host_page_size())) { 3037 return ret; 3038 } 3039 3040 if (!size) { 3041 return ret; 3042 } 3043 3044 section = memory_region_find(get_system_memory(), start, size); 3045 mr = section.mr; 3046 if (!mr) { 3047 /* 3048 * Ignore converting non-assigned region to shared. 3049 * 3050 * TDX requires vMMIO region to be shared to inject #VE to guest. 3051 * OVMF issues conservatively MapGPA(shared) on 32bit PCI MMIO region, 3052 * and vIO-APIC 0xFEC00000 4K page. 3053 * OVMF assigns 32bit PCI MMIO region to 3054 * [top of low memory: typically 2GB=0xC000000, 0xFC00000) 3055 */ 3056 if (!to_private) { 3057 return 0; 3058 } 3059 return ret; 3060 } 3061 3062 if (!memory_region_has_guest_memfd(mr)) { 3063 /* 3064 * Because vMMIO region must be shared, guest TD may convert vMMIO 3065 * region to shared explicitly. Don't complain such case. See 3066 * memory_region_type() for checking if the region is MMIO region. 3067 */ 3068 if (!to_private && 3069 !memory_region_is_ram(mr) && 3070 !memory_region_is_ram_device(mr) && 3071 !memory_region_is_rom(mr) && 3072 !memory_region_is_romd(mr)) { 3073 ret = 0; 3074 } else { 3075 error_report("Convert non guest_memfd backed memory region " 3076 "(0x%"HWADDR_PRIx" ,+ 0x%"HWADDR_PRIx") to %s", 3077 start, size, to_private ? "private" : "shared"); 3078 } 3079 goto out_unref; 3080 } 3081 3082 if (to_private) { 3083 ret = kvm_set_memory_attributes_private(start, size); 3084 } else { 3085 ret = kvm_set_memory_attributes_shared(start, size); 3086 } 3087 if (ret) { 3088 goto out_unref; 3089 } 3090 3091 addr = memory_region_get_ram_ptr(mr) + section.offset_within_region; 3092 rb = qemu_ram_block_from_host(addr, false, &offset); 3093 3094 if (to_private) { 3095 if (rb->page_size != qemu_real_host_page_size()) { 3096 /* 3097 * shared memory is backed by hugetlb, which is supposed to be 3098 * pre-allocated and doesn't need to be discarded 3099 */ 3100 goto out_unref; 3101 } 3102 ret = ram_block_discard_range(rb, offset, size); 3103 } else { 3104 ret = ram_block_discard_guest_memfd_range(rb, offset, size); 3105 } 3106 3107 out_unref: 3108 memory_region_unref(mr); 3109 return ret; 3110 } 3111 3112 int kvm_cpu_exec(CPUState *cpu) 3113 { 3114 struct kvm_run *run = cpu->kvm_run; 3115 int ret, run_ret; 3116 3117 trace_kvm_cpu_exec(); 3118 3119 if (kvm_arch_process_async_events(cpu)) { 3120 qatomic_set(&cpu->exit_request, 0); 3121 return EXCP_HLT; 3122 } 3123 3124 bql_unlock(); 3125 cpu_exec_start(cpu); 3126 3127 do { 3128 MemTxAttrs attrs; 3129 3130 if (cpu->vcpu_dirty) { 3131 Error *err = NULL; 3132 ret = kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE, &err); 3133 if (ret) { 3134 if (err) { 3135 error_reportf_err(err, "Putting registers after init: "); 3136 } else { 3137 error_report("Failed to put registers after init: %s", 3138 strerror(-ret)); 3139 } 3140 ret = -1; 3141 break; 3142 } 3143 3144 cpu->vcpu_dirty = false; 3145 } 3146 3147 kvm_arch_pre_run(cpu, run); 3148 if (qatomic_read(&cpu->exit_request)) { 3149 trace_kvm_interrupt_exit_request(); 3150 /* 3151 * KVM requires us to reenter the kernel after IO exits to complete 3152 * instruction emulation. This self-signal will ensure that we 3153 * leave ASAP again. 3154 */ 3155 kvm_cpu_kick_self(); 3156 } 3157 3158 /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit. 3159 * Matching barrier in kvm_eat_signals. 3160 */ 3161 smp_rmb(); 3162 3163 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0); 3164 3165 attrs = kvm_arch_post_run(cpu, run); 3166 3167 #ifdef KVM_HAVE_MCE_INJECTION 3168 if (unlikely(have_sigbus_pending)) { 3169 bql_lock(); 3170 kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code, 3171 pending_sigbus_addr); 3172 have_sigbus_pending = false; 3173 bql_unlock(); 3174 } 3175 #endif 3176 3177 if (run_ret < 0) { 3178 if (run_ret == -EINTR || run_ret == -EAGAIN) { 3179 trace_kvm_io_window_exit(); 3180 kvm_eat_signals(cpu); 3181 ret = EXCP_INTERRUPT; 3182 break; 3183 } 3184 if (!(run_ret == -EFAULT && run->exit_reason == KVM_EXIT_MEMORY_FAULT)) { 3185 fprintf(stderr, "error: kvm run failed %s\n", 3186 strerror(-run_ret)); 3187 #ifdef TARGET_PPC 3188 if (run_ret == -EBUSY) { 3189 fprintf(stderr, 3190 "This is probably because your SMT is enabled.\n" 3191 "VCPU can only run on primary threads with all " 3192 "secondary threads offline.\n"); 3193 } 3194 #endif 3195 ret = -1; 3196 break; 3197 } 3198 } 3199 3200 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason); 3201 switch (run->exit_reason) { 3202 case KVM_EXIT_IO: 3203 /* Called outside BQL */ 3204 kvm_handle_io(run->io.port, attrs, 3205 (uint8_t *)run + run->io.data_offset, 3206 run->io.direction, 3207 run->io.size, 3208 run->io.count); 3209 ret = 0; 3210 break; 3211 case KVM_EXIT_MMIO: 3212 /* Called outside BQL */ 3213 address_space_rw(&address_space_memory, 3214 run->mmio.phys_addr, attrs, 3215 run->mmio.data, 3216 run->mmio.len, 3217 run->mmio.is_write); 3218 ret = 0; 3219 break; 3220 case KVM_EXIT_IRQ_WINDOW_OPEN: 3221 ret = EXCP_INTERRUPT; 3222 break; 3223 case KVM_EXIT_SHUTDOWN: 3224 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 3225 ret = EXCP_INTERRUPT; 3226 break; 3227 case KVM_EXIT_UNKNOWN: 3228 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n", 3229 (uint64_t)run->hw.hardware_exit_reason); 3230 ret = -1; 3231 break; 3232 case KVM_EXIT_INTERNAL_ERROR: 3233 ret = kvm_handle_internal_error(cpu, run); 3234 break; 3235 case KVM_EXIT_DIRTY_RING_FULL: 3236 /* 3237 * We shouldn't continue if the dirty ring of this vcpu is 3238 * still full. Got kicked by KVM_RESET_DIRTY_RINGS. 3239 */ 3240 trace_kvm_dirty_ring_full(cpu->cpu_index); 3241 bql_lock(); 3242 /* 3243 * We throttle vCPU by making it sleep once it exit from kernel 3244 * due to dirty ring full. In the dirtylimit scenario, reaping 3245 * all vCPUs after a single vCPU dirty ring get full result in 3246 * the miss of sleep, so just reap the ring-fulled vCPU. 3247 */ 3248 if (dirtylimit_in_service()) { 3249 kvm_dirty_ring_reap(kvm_state, cpu); 3250 } else { 3251 kvm_dirty_ring_reap(kvm_state, NULL); 3252 } 3253 bql_unlock(); 3254 dirtylimit_vcpu_execute(cpu); 3255 ret = 0; 3256 break; 3257 case KVM_EXIT_SYSTEM_EVENT: 3258 trace_kvm_run_exit_system_event(cpu->cpu_index, run->system_event.type); 3259 switch (run->system_event.type) { 3260 case KVM_SYSTEM_EVENT_SHUTDOWN: 3261 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); 3262 ret = EXCP_INTERRUPT; 3263 break; 3264 case KVM_SYSTEM_EVENT_RESET: 3265 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 3266 ret = EXCP_INTERRUPT; 3267 break; 3268 case KVM_SYSTEM_EVENT_CRASH: 3269 kvm_cpu_synchronize_state(cpu); 3270 bql_lock(); 3271 qemu_system_guest_panicked(cpu_get_crash_info(cpu)); 3272 bql_unlock(); 3273 ret = 0; 3274 break; 3275 default: 3276 ret = kvm_arch_handle_exit(cpu, run); 3277 break; 3278 } 3279 break; 3280 case KVM_EXIT_MEMORY_FAULT: 3281 trace_kvm_memory_fault(run->memory_fault.gpa, 3282 run->memory_fault.size, 3283 run->memory_fault.flags); 3284 if (run->memory_fault.flags & ~KVM_MEMORY_EXIT_FLAG_PRIVATE) { 3285 error_report("KVM_EXIT_MEMORY_FAULT: Unknown flag 0x%" PRIx64, 3286 (uint64_t)run->memory_fault.flags); 3287 ret = -1; 3288 break; 3289 } 3290 ret = kvm_convert_memory(run->memory_fault.gpa, run->memory_fault.size, 3291 run->memory_fault.flags & KVM_MEMORY_EXIT_FLAG_PRIVATE); 3292 break; 3293 default: 3294 ret = kvm_arch_handle_exit(cpu, run); 3295 break; 3296 } 3297 } while (ret == 0); 3298 3299 cpu_exec_end(cpu); 3300 bql_lock(); 3301 3302 if (ret < 0) { 3303 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 3304 vm_stop(RUN_STATE_INTERNAL_ERROR); 3305 } 3306 3307 qatomic_set(&cpu->exit_request, 0); 3308 return ret; 3309 } 3310 3311 int kvm_ioctl(KVMState *s, unsigned long type, ...) 3312 { 3313 int ret; 3314 void *arg; 3315 va_list ap; 3316 3317 va_start(ap, type); 3318 arg = va_arg(ap, void *); 3319 va_end(ap); 3320 3321 trace_kvm_ioctl(type, arg); 3322 ret = ioctl(s->fd, type, arg); 3323 if (ret == -1) { 3324 ret = -errno; 3325 } 3326 return ret; 3327 } 3328 3329 int kvm_vm_ioctl(KVMState *s, unsigned long type, ...) 3330 { 3331 int ret; 3332 void *arg; 3333 va_list ap; 3334 3335 va_start(ap, type); 3336 arg = va_arg(ap, void *); 3337 va_end(ap); 3338 3339 trace_kvm_vm_ioctl(type, arg); 3340 accel_ioctl_begin(); 3341 ret = ioctl(s->vmfd, type, arg); 3342 accel_ioctl_end(); 3343 if (ret == -1) { 3344 ret = -errno; 3345 } 3346 return ret; 3347 } 3348 3349 int kvm_vcpu_ioctl(CPUState *cpu, unsigned long type, ...) 3350 { 3351 int ret; 3352 void *arg; 3353 va_list ap; 3354 3355 va_start(ap, type); 3356 arg = va_arg(ap, void *); 3357 va_end(ap); 3358 3359 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg); 3360 accel_cpu_ioctl_begin(cpu); 3361 ret = ioctl(cpu->kvm_fd, type, arg); 3362 accel_cpu_ioctl_end(cpu); 3363 if (ret == -1) { 3364 ret = -errno; 3365 } 3366 return ret; 3367 } 3368 3369 int kvm_device_ioctl(int fd, unsigned long type, ...) 3370 { 3371 int ret; 3372 void *arg; 3373 va_list ap; 3374 3375 va_start(ap, type); 3376 arg = va_arg(ap, void *); 3377 va_end(ap); 3378 3379 trace_kvm_device_ioctl(fd, type, arg); 3380 accel_ioctl_begin(); 3381 ret = ioctl(fd, type, arg); 3382 accel_ioctl_end(); 3383 if (ret == -1) { 3384 ret = -errno; 3385 } 3386 return ret; 3387 } 3388 3389 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr) 3390 { 3391 int ret; 3392 struct kvm_device_attr attribute = { 3393 .group = group, 3394 .attr = attr, 3395 }; 3396 3397 if (!kvm_vm_attributes_allowed) { 3398 return 0; 3399 } 3400 3401 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute); 3402 /* kvm returns 0 on success for HAS_DEVICE_ATTR */ 3403 return ret ? 0 : 1; 3404 } 3405 3406 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr) 3407 { 3408 struct kvm_device_attr attribute = { 3409 .group = group, 3410 .attr = attr, 3411 .flags = 0, 3412 }; 3413 3414 return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1; 3415 } 3416 3417 int kvm_device_access(int fd, int group, uint64_t attr, 3418 void *val, bool write, Error **errp) 3419 { 3420 struct kvm_device_attr kvmattr; 3421 int err; 3422 3423 kvmattr.flags = 0; 3424 kvmattr.group = group; 3425 kvmattr.attr = attr; 3426 kvmattr.addr = (uintptr_t)val; 3427 3428 err = kvm_device_ioctl(fd, 3429 write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR, 3430 &kvmattr); 3431 if (err < 0) { 3432 error_setg_errno(errp, -err, 3433 "KVM_%s_DEVICE_ATTR failed: Group %d " 3434 "attr 0x%016" PRIx64, 3435 write ? "SET" : "GET", group, attr); 3436 } 3437 return err; 3438 } 3439 3440 bool kvm_has_sync_mmu(void) 3441 { 3442 return kvm_state->sync_mmu; 3443 } 3444 3445 int kvm_has_vcpu_events(void) 3446 { 3447 return kvm_state->vcpu_events; 3448 } 3449 3450 int kvm_max_nested_state_length(void) 3451 { 3452 return kvm_state->max_nested_state_len; 3453 } 3454 3455 int kvm_has_gsi_routing(void) 3456 { 3457 #ifdef KVM_CAP_IRQ_ROUTING 3458 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING); 3459 #else 3460 return false; 3461 #endif 3462 } 3463 3464 bool kvm_arm_supports_user_irq(void) 3465 { 3466 return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ); 3467 } 3468 3469 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG 3470 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu, vaddr pc) 3471 { 3472 struct kvm_sw_breakpoint *bp; 3473 3474 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) { 3475 if (bp->pc == pc) { 3476 return bp; 3477 } 3478 } 3479 return NULL; 3480 } 3481 3482 int kvm_sw_breakpoints_active(CPUState *cpu) 3483 { 3484 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints); 3485 } 3486 3487 struct kvm_set_guest_debug_data { 3488 struct kvm_guest_debug dbg; 3489 int err; 3490 }; 3491 3492 static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data) 3493 { 3494 struct kvm_set_guest_debug_data *dbg_data = 3495 (struct kvm_set_guest_debug_data *) data.host_ptr; 3496 3497 dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG, 3498 &dbg_data->dbg); 3499 } 3500 3501 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) 3502 { 3503 struct kvm_set_guest_debug_data data; 3504 3505 data.dbg.control = reinject_trap; 3506 3507 if (cpu->singlestep_enabled) { 3508 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP; 3509 3510 if (cpu->singlestep_enabled & SSTEP_NOIRQ) { 3511 data.dbg.control |= KVM_GUESTDBG_BLOCKIRQ; 3512 } 3513 } 3514 kvm_arch_update_guest_debug(cpu, &data.dbg); 3515 3516 run_on_cpu(cpu, kvm_invoke_set_guest_debug, 3517 RUN_ON_CPU_HOST_PTR(&data)); 3518 return data.err; 3519 } 3520 3521 bool kvm_supports_guest_debug(void) 3522 { 3523 /* probed during kvm_init() */ 3524 return kvm_has_guest_debug; 3525 } 3526 3527 int kvm_insert_breakpoint(CPUState *cpu, int type, vaddr addr, vaddr len) 3528 { 3529 struct kvm_sw_breakpoint *bp; 3530 int err; 3531 3532 if (type == GDB_BREAKPOINT_SW) { 3533 bp = kvm_find_sw_breakpoint(cpu, addr); 3534 if (bp) { 3535 bp->use_count++; 3536 return 0; 3537 } 3538 3539 bp = g_new(struct kvm_sw_breakpoint, 1); 3540 bp->pc = addr; 3541 bp->use_count = 1; 3542 err = kvm_arch_insert_sw_breakpoint(cpu, bp); 3543 if (err) { 3544 g_free(bp); 3545 return err; 3546 } 3547 3548 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 3549 } else { 3550 err = kvm_arch_insert_hw_breakpoint(addr, len, type); 3551 if (err) { 3552 return err; 3553 } 3554 } 3555 3556 CPU_FOREACH(cpu) { 3557 err = kvm_update_guest_debug(cpu, 0); 3558 if (err) { 3559 return err; 3560 } 3561 } 3562 return 0; 3563 } 3564 3565 int kvm_remove_breakpoint(CPUState *cpu, int type, vaddr addr, vaddr len) 3566 { 3567 struct kvm_sw_breakpoint *bp; 3568 int err; 3569 3570 if (type == GDB_BREAKPOINT_SW) { 3571 bp = kvm_find_sw_breakpoint(cpu, addr); 3572 if (!bp) { 3573 return -ENOENT; 3574 } 3575 3576 if (bp->use_count > 1) { 3577 bp->use_count--; 3578 return 0; 3579 } 3580 3581 err = kvm_arch_remove_sw_breakpoint(cpu, bp); 3582 if (err) { 3583 return err; 3584 } 3585 3586 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 3587 g_free(bp); 3588 } else { 3589 err = kvm_arch_remove_hw_breakpoint(addr, len, type); 3590 if (err) { 3591 return err; 3592 } 3593 } 3594 3595 CPU_FOREACH(cpu) { 3596 err = kvm_update_guest_debug(cpu, 0); 3597 if (err) { 3598 return err; 3599 } 3600 } 3601 return 0; 3602 } 3603 3604 void kvm_remove_all_breakpoints(CPUState *cpu) 3605 { 3606 struct kvm_sw_breakpoint *bp, *next; 3607 KVMState *s = cpu->kvm_state; 3608 CPUState *tmpcpu; 3609 3610 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) { 3611 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) { 3612 /* Try harder to find a CPU that currently sees the breakpoint. */ 3613 CPU_FOREACH(tmpcpu) { 3614 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) { 3615 break; 3616 } 3617 } 3618 } 3619 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry); 3620 g_free(bp); 3621 } 3622 kvm_arch_remove_all_hw_breakpoints(); 3623 3624 CPU_FOREACH(cpu) { 3625 kvm_update_guest_debug(cpu, 0); 3626 } 3627 } 3628 3629 #endif /* !TARGET_KVM_HAVE_GUEST_DEBUG */ 3630 3631 static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset) 3632 { 3633 KVMState *s = kvm_state; 3634 struct kvm_signal_mask *sigmask; 3635 int r; 3636 3637 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset)); 3638 3639 sigmask->len = s->sigmask_len; 3640 memcpy(sigmask->sigset, sigset, sizeof(*sigset)); 3641 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask); 3642 g_free(sigmask); 3643 3644 return r; 3645 } 3646 3647 static void kvm_ipi_signal(int sig) 3648 { 3649 if (current_cpu) { 3650 assert(kvm_immediate_exit); 3651 kvm_cpu_kick(current_cpu); 3652 } 3653 } 3654 3655 void kvm_init_cpu_signals(CPUState *cpu) 3656 { 3657 int r; 3658 sigset_t set; 3659 struct sigaction sigact; 3660 3661 memset(&sigact, 0, sizeof(sigact)); 3662 sigact.sa_handler = kvm_ipi_signal; 3663 sigaction(SIG_IPI, &sigact, NULL); 3664 3665 pthread_sigmask(SIG_BLOCK, NULL, &set); 3666 #if defined KVM_HAVE_MCE_INJECTION 3667 sigdelset(&set, SIGBUS); 3668 pthread_sigmask(SIG_SETMASK, &set, NULL); 3669 #endif 3670 sigdelset(&set, SIG_IPI); 3671 if (kvm_immediate_exit) { 3672 r = pthread_sigmask(SIG_SETMASK, &set, NULL); 3673 } else { 3674 r = kvm_set_signal_mask(cpu, &set); 3675 } 3676 if (r) { 3677 fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r)); 3678 exit(1); 3679 } 3680 } 3681 3682 /* Called asynchronously in VCPU thread. */ 3683 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr) 3684 { 3685 #ifdef KVM_HAVE_MCE_INJECTION 3686 if (have_sigbus_pending) { 3687 return 1; 3688 } 3689 have_sigbus_pending = true; 3690 pending_sigbus_addr = addr; 3691 pending_sigbus_code = code; 3692 qatomic_set(&cpu->exit_request, 1); 3693 return 0; 3694 #else 3695 return 1; 3696 #endif 3697 } 3698 3699 /* Called synchronously (via signalfd) in main thread. */ 3700 int kvm_on_sigbus(int code, void *addr) 3701 { 3702 #ifdef KVM_HAVE_MCE_INJECTION 3703 /* Action required MCE kills the process if SIGBUS is blocked. Because 3704 * that's what happens in the I/O thread, where we handle MCE via signalfd, 3705 * we can only get action optional here. 3706 */ 3707 assert(code != BUS_MCEERR_AR); 3708 kvm_arch_on_sigbus_vcpu(first_cpu, code, addr); 3709 return 0; 3710 #else 3711 return 1; 3712 #endif 3713 } 3714 3715 int kvm_create_device(KVMState *s, uint64_t type, bool test) 3716 { 3717 int ret; 3718 struct kvm_create_device create_dev; 3719 3720 create_dev.type = type; 3721 create_dev.fd = -1; 3722 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0; 3723 3724 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) { 3725 return -ENOTSUP; 3726 } 3727 3728 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev); 3729 if (ret) { 3730 return ret; 3731 } 3732 3733 return test ? 0 : create_dev.fd; 3734 } 3735 3736 bool kvm_device_supported(int vmfd, uint64_t type) 3737 { 3738 struct kvm_create_device create_dev = { 3739 .type = type, 3740 .fd = -1, 3741 .flags = KVM_CREATE_DEVICE_TEST, 3742 }; 3743 3744 if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) { 3745 return false; 3746 } 3747 3748 return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0); 3749 } 3750 3751 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source) 3752 { 3753 struct kvm_one_reg reg; 3754 int r; 3755 3756 reg.id = id; 3757 reg.addr = (uintptr_t) source; 3758 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 3759 if (r) { 3760 trace_kvm_failed_reg_set(id, strerror(-r)); 3761 } 3762 return r; 3763 } 3764 3765 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target) 3766 { 3767 struct kvm_one_reg reg; 3768 int r; 3769 3770 reg.id = id; 3771 reg.addr = (uintptr_t) target; 3772 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 3773 if (r) { 3774 trace_kvm_failed_reg_get(id, strerror(-r)); 3775 } 3776 return r; 3777 } 3778 3779 static bool kvm_accel_has_memory(MachineState *ms, AddressSpace *as, 3780 hwaddr start_addr, hwaddr size) 3781 { 3782 KVMState *kvm = KVM_STATE(ms->accelerator); 3783 int i; 3784 3785 for (i = 0; i < kvm->nr_as; ++i) { 3786 if (kvm->as[i].as == as && kvm->as[i].ml) { 3787 size = MIN(kvm_max_slot_size, size); 3788 return NULL != kvm_lookup_matching_slot(kvm->as[i].ml, 3789 start_addr, size); 3790 } 3791 } 3792 3793 return false; 3794 } 3795 3796 static void kvm_get_kvm_shadow_mem(Object *obj, Visitor *v, 3797 const char *name, void *opaque, 3798 Error **errp) 3799 { 3800 KVMState *s = KVM_STATE(obj); 3801 int64_t value = s->kvm_shadow_mem; 3802 3803 visit_type_int(v, name, &value, errp); 3804 } 3805 3806 static void kvm_set_kvm_shadow_mem(Object *obj, Visitor *v, 3807 const char *name, void *opaque, 3808 Error **errp) 3809 { 3810 KVMState *s = KVM_STATE(obj); 3811 int64_t value; 3812 3813 if (s->fd != -1) { 3814 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3815 return; 3816 } 3817 3818 if (!visit_type_int(v, name, &value, errp)) { 3819 return; 3820 } 3821 3822 s->kvm_shadow_mem = value; 3823 } 3824 3825 static void kvm_set_kernel_irqchip(Object *obj, Visitor *v, 3826 const char *name, void *opaque, 3827 Error **errp) 3828 { 3829 KVMState *s = KVM_STATE(obj); 3830 OnOffSplit mode; 3831 3832 if (s->fd != -1) { 3833 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3834 return; 3835 } 3836 3837 if (!visit_type_OnOffSplit(v, name, &mode, errp)) { 3838 return; 3839 } 3840 switch (mode) { 3841 case ON_OFF_SPLIT_ON: 3842 s->kernel_irqchip_allowed = true; 3843 s->kernel_irqchip_required = true; 3844 s->kernel_irqchip_split = ON_OFF_AUTO_OFF; 3845 break; 3846 case ON_OFF_SPLIT_OFF: 3847 s->kernel_irqchip_allowed = false; 3848 s->kernel_irqchip_required = false; 3849 s->kernel_irqchip_split = ON_OFF_AUTO_OFF; 3850 break; 3851 case ON_OFF_SPLIT_SPLIT: 3852 s->kernel_irqchip_allowed = true; 3853 s->kernel_irqchip_required = true; 3854 s->kernel_irqchip_split = ON_OFF_AUTO_ON; 3855 break; 3856 default: 3857 /* The value was checked in visit_type_OnOffSplit() above. If 3858 * we get here, then something is wrong in QEMU. 3859 */ 3860 abort(); 3861 } 3862 } 3863 3864 bool kvm_kernel_irqchip_allowed(void) 3865 { 3866 return kvm_state->kernel_irqchip_allowed; 3867 } 3868 3869 bool kvm_kernel_irqchip_required(void) 3870 { 3871 return kvm_state->kernel_irqchip_required; 3872 } 3873 3874 bool kvm_kernel_irqchip_split(void) 3875 { 3876 return kvm_state->kernel_irqchip_split == ON_OFF_AUTO_ON; 3877 } 3878 3879 static void kvm_get_dirty_ring_size(Object *obj, Visitor *v, 3880 const char *name, void *opaque, 3881 Error **errp) 3882 { 3883 KVMState *s = KVM_STATE(obj); 3884 uint32_t value = s->kvm_dirty_ring_size; 3885 3886 visit_type_uint32(v, name, &value, errp); 3887 } 3888 3889 static void kvm_set_dirty_ring_size(Object *obj, Visitor *v, 3890 const char *name, void *opaque, 3891 Error **errp) 3892 { 3893 KVMState *s = KVM_STATE(obj); 3894 uint32_t value; 3895 3896 if (s->fd != -1) { 3897 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3898 return; 3899 } 3900 3901 if (!visit_type_uint32(v, name, &value, errp)) { 3902 return; 3903 } 3904 if (value & (value - 1)) { 3905 error_setg(errp, "dirty-ring-size must be a power of two."); 3906 return; 3907 } 3908 3909 s->kvm_dirty_ring_size = value; 3910 } 3911 3912 static char *kvm_get_device(Object *obj, 3913 Error **errp G_GNUC_UNUSED) 3914 { 3915 KVMState *s = KVM_STATE(obj); 3916 3917 return g_strdup(s->device); 3918 } 3919 3920 static void kvm_set_device(Object *obj, 3921 const char *value, 3922 Error **errp G_GNUC_UNUSED) 3923 { 3924 KVMState *s = KVM_STATE(obj); 3925 3926 g_free(s->device); 3927 s->device = g_strdup(value); 3928 } 3929 3930 static void kvm_set_kvm_rapl(Object *obj, bool value, Error **errp) 3931 { 3932 KVMState *s = KVM_STATE(obj); 3933 s->msr_energy.enable = value; 3934 } 3935 3936 static void kvm_set_kvm_rapl_socket_path(Object *obj, 3937 const char *str, 3938 Error **errp) 3939 { 3940 KVMState *s = KVM_STATE(obj); 3941 g_free(s->msr_energy.socket_path); 3942 s->msr_energy.socket_path = g_strdup(str); 3943 } 3944 3945 static void kvm_accel_instance_init(Object *obj) 3946 { 3947 KVMState *s = KVM_STATE(obj); 3948 3949 s->fd = -1; 3950 s->vmfd = -1; 3951 s->kvm_shadow_mem = -1; 3952 s->kernel_irqchip_allowed = true; 3953 s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; 3954 /* KVM dirty ring is by default off */ 3955 s->kvm_dirty_ring_size = 0; 3956 s->kvm_dirty_ring_with_bitmap = false; 3957 s->kvm_eager_split_size = 0; 3958 s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; 3959 s->notify_window = 0; 3960 s->xen_version = 0; 3961 s->xen_gnttab_max_frames = 64; 3962 s->xen_evtchn_max_pirq = 256; 3963 s->device = NULL; 3964 s->msr_energy.enable = false; 3965 } 3966 3967 /** 3968 * kvm_gdbstub_sstep_flags(): 3969 * 3970 * Returns: SSTEP_* flags that KVM supports for guest debug. The 3971 * support is probed during kvm_init() 3972 */ 3973 static int kvm_gdbstub_sstep_flags(void) 3974 { 3975 return kvm_sstep_flags; 3976 } 3977 3978 static void kvm_accel_class_init(ObjectClass *oc, const void *data) 3979 { 3980 AccelClass *ac = ACCEL_CLASS(oc); 3981 ac->name = "KVM"; 3982 ac->init_machine = kvm_init; 3983 ac->has_memory = kvm_accel_has_memory; 3984 ac->allowed = &kvm_allowed; 3985 ac->gdbstub_supported_sstep_flags = kvm_gdbstub_sstep_flags; 3986 3987 object_class_property_add(oc, "kernel-irqchip", "on|off|split", 3988 NULL, kvm_set_kernel_irqchip, 3989 NULL, NULL); 3990 object_class_property_set_description(oc, "kernel-irqchip", 3991 "Configure KVM in-kernel irqchip"); 3992 3993 object_class_property_add(oc, "kvm-shadow-mem", "int", 3994 kvm_get_kvm_shadow_mem, kvm_set_kvm_shadow_mem, 3995 NULL, NULL); 3996 object_class_property_set_description(oc, "kvm-shadow-mem", 3997 "KVM shadow MMU size"); 3998 3999 object_class_property_add(oc, "dirty-ring-size", "uint32", 4000 kvm_get_dirty_ring_size, kvm_set_dirty_ring_size, 4001 NULL, NULL); 4002 object_class_property_set_description(oc, "dirty-ring-size", 4003 "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); 4004 4005 object_class_property_add_str(oc, "device", kvm_get_device, kvm_set_device); 4006 object_class_property_set_description(oc, "device", 4007 "Path to the device node to use (default: /dev/kvm)"); 4008 4009 object_class_property_add_bool(oc, "rapl", 4010 NULL, 4011 kvm_set_kvm_rapl); 4012 object_class_property_set_description(oc, "rapl", 4013 "Allow energy related MSRs for RAPL interface in Guest"); 4014 4015 object_class_property_add_str(oc, "rapl-helper-socket", NULL, 4016 kvm_set_kvm_rapl_socket_path); 4017 object_class_property_set_description(oc, "rapl-helper-socket", 4018 "Socket Path for comminucating with the Virtual MSR helper daemon"); 4019 4020 kvm_arch_accel_class_init(oc); 4021 } 4022 4023 static const TypeInfo kvm_accel_type = { 4024 .name = TYPE_KVM_ACCEL, 4025 .parent = TYPE_ACCEL, 4026 .instance_init = kvm_accel_instance_init, 4027 .class_init = kvm_accel_class_init, 4028 .instance_size = sizeof(KVMState), 4029 }; 4030 4031 static void kvm_type_init(void) 4032 { 4033 type_register_static(&kvm_accel_type); 4034 } 4035 4036 type_init(kvm_type_init); 4037 4038 typedef struct StatsArgs { 4039 union StatsResultsType { 4040 StatsResultList **stats; 4041 StatsSchemaList **schema; 4042 } result; 4043 strList *names; 4044 Error **errp; 4045 } StatsArgs; 4046 4047 static StatsList *add_kvmstat_entry(struct kvm_stats_desc *pdesc, 4048 uint64_t *stats_data, 4049 StatsList *stats_list, 4050 Error **errp) 4051 { 4052 4053 Stats *stats; 4054 uint64List *val_list = NULL; 4055 4056 /* Only add stats that we understand. */ 4057 switch (pdesc->flags & KVM_STATS_TYPE_MASK) { 4058 case KVM_STATS_TYPE_CUMULATIVE: 4059 case KVM_STATS_TYPE_INSTANT: 4060 case KVM_STATS_TYPE_PEAK: 4061 case KVM_STATS_TYPE_LINEAR_HIST: 4062 case KVM_STATS_TYPE_LOG_HIST: 4063 break; 4064 default: 4065 return stats_list; 4066 } 4067 4068 switch (pdesc->flags & KVM_STATS_UNIT_MASK) { 4069 case KVM_STATS_UNIT_NONE: 4070 case KVM_STATS_UNIT_BYTES: 4071 case KVM_STATS_UNIT_CYCLES: 4072 case KVM_STATS_UNIT_SECONDS: 4073 case KVM_STATS_UNIT_BOOLEAN: 4074 break; 4075 default: 4076 return stats_list; 4077 } 4078 4079 switch (pdesc->flags & KVM_STATS_BASE_MASK) { 4080 case KVM_STATS_BASE_POW10: 4081 case KVM_STATS_BASE_POW2: 4082 break; 4083 default: 4084 return stats_list; 4085 } 4086 4087 /* Alloc and populate data list */ 4088 stats = g_new0(Stats, 1); 4089 stats->name = g_strdup(pdesc->name); 4090 stats->value = g_new0(StatsValue, 1); 4091 4092 if ((pdesc->flags & KVM_STATS_UNIT_MASK) == KVM_STATS_UNIT_BOOLEAN) { 4093 stats->value->u.boolean = *stats_data; 4094 stats->value->type = QTYPE_QBOOL; 4095 } else if (pdesc->size == 1) { 4096 stats->value->u.scalar = *stats_data; 4097 stats->value->type = QTYPE_QNUM; 4098 } else { 4099 int i; 4100 for (i = 0; i < pdesc->size; i++) { 4101 QAPI_LIST_PREPEND(val_list, stats_data[i]); 4102 } 4103 stats->value->u.list = val_list; 4104 stats->value->type = QTYPE_QLIST; 4105 } 4106 4107 QAPI_LIST_PREPEND(stats_list, stats); 4108 return stats_list; 4109 } 4110 4111 static StatsSchemaValueList *add_kvmschema_entry(struct kvm_stats_desc *pdesc, 4112 StatsSchemaValueList *list, 4113 Error **errp) 4114 { 4115 StatsSchemaValueList *schema_entry = g_new0(StatsSchemaValueList, 1); 4116 schema_entry->value = g_new0(StatsSchemaValue, 1); 4117 4118 switch (pdesc->flags & KVM_STATS_TYPE_MASK) { 4119 case KVM_STATS_TYPE_CUMULATIVE: 4120 schema_entry->value->type = STATS_TYPE_CUMULATIVE; 4121 break; 4122 case KVM_STATS_TYPE_INSTANT: 4123 schema_entry->value->type = STATS_TYPE_INSTANT; 4124 break; 4125 case KVM_STATS_TYPE_PEAK: 4126 schema_entry->value->type = STATS_TYPE_PEAK; 4127 break; 4128 case KVM_STATS_TYPE_LINEAR_HIST: 4129 schema_entry->value->type = STATS_TYPE_LINEAR_HISTOGRAM; 4130 schema_entry->value->bucket_size = pdesc->bucket_size; 4131 schema_entry->value->has_bucket_size = true; 4132 break; 4133 case KVM_STATS_TYPE_LOG_HIST: 4134 schema_entry->value->type = STATS_TYPE_LOG2_HISTOGRAM; 4135 break; 4136 default: 4137 goto exit; 4138 } 4139 4140 switch (pdesc->flags & KVM_STATS_UNIT_MASK) { 4141 case KVM_STATS_UNIT_NONE: 4142 break; 4143 case KVM_STATS_UNIT_BOOLEAN: 4144 schema_entry->value->has_unit = true; 4145 schema_entry->value->unit = STATS_UNIT_BOOLEAN; 4146 break; 4147 case KVM_STATS_UNIT_BYTES: 4148 schema_entry->value->has_unit = true; 4149 schema_entry->value->unit = STATS_UNIT_BYTES; 4150 break; 4151 case KVM_STATS_UNIT_CYCLES: 4152 schema_entry->value->has_unit = true; 4153 schema_entry->value->unit = STATS_UNIT_CYCLES; 4154 break; 4155 case KVM_STATS_UNIT_SECONDS: 4156 schema_entry->value->has_unit = true; 4157 schema_entry->value->unit = STATS_UNIT_SECONDS; 4158 break; 4159 default: 4160 goto exit; 4161 } 4162 4163 schema_entry->value->exponent = pdesc->exponent; 4164 if (pdesc->exponent) { 4165 switch (pdesc->flags & KVM_STATS_BASE_MASK) { 4166 case KVM_STATS_BASE_POW10: 4167 schema_entry->value->has_base = true; 4168 schema_entry->value->base = 10; 4169 break; 4170 case KVM_STATS_BASE_POW2: 4171 schema_entry->value->has_base = true; 4172 schema_entry->value->base = 2; 4173 break; 4174 default: 4175 goto exit; 4176 } 4177 } 4178 4179 schema_entry->value->name = g_strdup(pdesc->name); 4180 schema_entry->next = list; 4181 return schema_entry; 4182 exit: 4183 g_free(schema_entry->value); 4184 g_free(schema_entry); 4185 return list; 4186 } 4187 4188 /* Cached stats descriptors */ 4189 typedef struct StatsDescriptors { 4190 const char *ident; /* cache key, currently the StatsTarget */ 4191 struct kvm_stats_desc *kvm_stats_desc; 4192 struct kvm_stats_header kvm_stats_header; 4193 QTAILQ_ENTRY(StatsDescriptors) next; 4194 } StatsDescriptors; 4195 4196 static QTAILQ_HEAD(, StatsDescriptors) stats_descriptors = 4197 QTAILQ_HEAD_INITIALIZER(stats_descriptors); 4198 4199 /* 4200 * Return the descriptors for 'target', that either have already been read 4201 * or are retrieved from 'stats_fd'. 4202 */ 4203 static StatsDescriptors *find_stats_descriptors(StatsTarget target, int stats_fd, 4204 Error **errp) 4205 { 4206 StatsDescriptors *descriptors; 4207 const char *ident; 4208 struct kvm_stats_desc *kvm_stats_desc; 4209 struct kvm_stats_header *kvm_stats_header; 4210 size_t size_desc; 4211 ssize_t ret; 4212 4213 ident = StatsTarget_str(target); 4214 QTAILQ_FOREACH(descriptors, &stats_descriptors, next) { 4215 if (g_str_equal(descriptors->ident, ident)) { 4216 return descriptors; 4217 } 4218 } 4219 4220 descriptors = g_new0(StatsDescriptors, 1); 4221 4222 /* Read stats header */ 4223 kvm_stats_header = &descriptors->kvm_stats_header; 4224 ret = pread(stats_fd, kvm_stats_header, sizeof(*kvm_stats_header), 0); 4225 if (ret != sizeof(*kvm_stats_header)) { 4226 error_setg(errp, "KVM stats: failed to read stats header: " 4227 "expected %zu actual %zu", 4228 sizeof(*kvm_stats_header), ret); 4229 g_free(descriptors); 4230 return NULL; 4231 } 4232 size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; 4233 4234 /* Read stats descriptors */ 4235 kvm_stats_desc = g_malloc0_n(kvm_stats_header->num_desc, size_desc); 4236 ret = pread(stats_fd, kvm_stats_desc, 4237 size_desc * kvm_stats_header->num_desc, 4238 kvm_stats_header->desc_offset); 4239 4240 if (ret != size_desc * kvm_stats_header->num_desc) { 4241 error_setg(errp, "KVM stats: failed to read stats descriptors: " 4242 "expected %zu actual %zu", 4243 size_desc * kvm_stats_header->num_desc, ret); 4244 g_free(descriptors); 4245 g_free(kvm_stats_desc); 4246 return NULL; 4247 } 4248 descriptors->kvm_stats_desc = kvm_stats_desc; 4249 descriptors->ident = ident; 4250 QTAILQ_INSERT_TAIL(&stats_descriptors, descriptors, next); 4251 return descriptors; 4252 } 4253 4254 static void query_stats(StatsResultList **result, StatsTarget target, 4255 strList *names, int stats_fd, CPUState *cpu, 4256 Error **errp) 4257 { 4258 struct kvm_stats_desc *kvm_stats_desc; 4259 struct kvm_stats_header *kvm_stats_header; 4260 StatsDescriptors *descriptors; 4261 g_autofree uint64_t *stats_data = NULL; 4262 struct kvm_stats_desc *pdesc; 4263 StatsList *stats_list = NULL; 4264 size_t size_desc, size_data = 0; 4265 ssize_t ret; 4266 int i; 4267 4268 descriptors = find_stats_descriptors(target, stats_fd, errp); 4269 if (!descriptors) { 4270 return; 4271 } 4272 4273 kvm_stats_header = &descriptors->kvm_stats_header; 4274 kvm_stats_desc = descriptors->kvm_stats_desc; 4275 size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; 4276 4277 /* Tally the total data size; read schema data */ 4278 for (i = 0; i < kvm_stats_header->num_desc; ++i) { 4279 pdesc = (void *)kvm_stats_desc + i * size_desc; 4280 size_data += pdesc->size * sizeof(*stats_data); 4281 } 4282 4283 stats_data = g_malloc0(size_data); 4284 ret = pread(stats_fd, stats_data, size_data, kvm_stats_header->data_offset); 4285 4286 if (ret != size_data) { 4287 error_setg(errp, "KVM stats: failed to read data: " 4288 "expected %zu actual %zu", size_data, ret); 4289 return; 4290 } 4291 4292 for (i = 0; i < kvm_stats_header->num_desc; ++i) { 4293 uint64_t *stats; 4294 pdesc = (void *)kvm_stats_desc + i * size_desc; 4295 4296 /* Add entry to the list */ 4297 stats = (void *)stats_data + pdesc->offset; 4298 if (!apply_str_list_filter(pdesc->name, names)) { 4299 continue; 4300 } 4301 stats_list = add_kvmstat_entry(pdesc, stats, stats_list, errp); 4302 } 4303 4304 if (!stats_list) { 4305 return; 4306 } 4307 4308 switch (target) { 4309 case STATS_TARGET_VM: 4310 add_stats_entry(result, STATS_PROVIDER_KVM, NULL, stats_list); 4311 break; 4312 case STATS_TARGET_VCPU: 4313 add_stats_entry(result, STATS_PROVIDER_KVM, 4314 cpu->parent_obj.canonical_path, 4315 stats_list); 4316 break; 4317 default: 4318 g_assert_not_reached(); 4319 } 4320 } 4321 4322 static void query_stats_schema(StatsSchemaList **result, StatsTarget target, 4323 int stats_fd, Error **errp) 4324 { 4325 struct kvm_stats_desc *kvm_stats_desc; 4326 struct kvm_stats_header *kvm_stats_header; 4327 StatsDescriptors *descriptors; 4328 struct kvm_stats_desc *pdesc; 4329 StatsSchemaValueList *stats_list = NULL; 4330 size_t size_desc; 4331 int i; 4332 4333 descriptors = find_stats_descriptors(target, stats_fd, errp); 4334 if (!descriptors) { 4335 return; 4336 } 4337 4338 kvm_stats_header = &descriptors->kvm_stats_header; 4339 kvm_stats_desc = descriptors->kvm_stats_desc; 4340 size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; 4341 4342 /* Tally the total data size; read schema data */ 4343 for (i = 0; i < kvm_stats_header->num_desc; ++i) { 4344 pdesc = (void *)kvm_stats_desc + i * size_desc; 4345 stats_list = add_kvmschema_entry(pdesc, stats_list, errp); 4346 } 4347 4348 add_stats_schema(result, STATS_PROVIDER_KVM, target, stats_list); 4349 } 4350 4351 static void query_stats_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args) 4352 { 4353 int stats_fd = cpu->kvm_vcpu_stats_fd; 4354 Error *local_err = NULL; 4355 4356 if (stats_fd == -1) { 4357 error_setg_errno(&local_err, errno, "KVM stats: ioctl failed"); 4358 error_propagate(kvm_stats_args->errp, local_err); 4359 return; 4360 } 4361 query_stats(kvm_stats_args->result.stats, STATS_TARGET_VCPU, 4362 kvm_stats_args->names, stats_fd, cpu, 4363 kvm_stats_args->errp); 4364 } 4365 4366 static void query_stats_schema_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args) 4367 { 4368 int stats_fd = cpu->kvm_vcpu_stats_fd; 4369 Error *local_err = NULL; 4370 4371 if (stats_fd == -1) { 4372 error_setg_errno(&local_err, errno, "KVM stats: ioctl failed"); 4373 error_propagate(kvm_stats_args->errp, local_err); 4374 return; 4375 } 4376 query_stats_schema(kvm_stats_args->result.schema, STATS_TARGET_VCPU, stats_fd, 4377 kvm_stats_args->errp); 4378 } 4379 4380 static void query_stats_cb(StatsResultList **result, StatsTarget target, 4381 strList *names, strList *targets, Error **errp) 4382 { 4383 KVMState *s = kvm_state; 4384 CPUState *cpu; 4385 int stats_fd; 4386 4387 switch (target) { 4388 case STATS_TARGET_VM: 4389 { 4390 stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL); 4391 if (stats_fd == -1) { 4392 error_setg_errno(errp, errno, "KVM stats: ioctl failed"); 4393 return; 4394 } 4395 query_stats(result, target, names, stats_fd, NULL, errp); 4396 close(stats_fd); 4397 break; 4398 } 4399 case STATS_TARGET_VCPU: 4400 { 4401 StatsArgs stats_args; 4402 stats_args.result.stats = result; 4403 stats_args.names = names; 4404 stats_args.errp = errp; 4405 CPU_FOREACH(cpu) { 4406 if (!apply_str_list_filter(cpu->parent_obj.canonical_path, targets)) { 4407 continue; 4408 } 4409 query_stats_vcpu(cpu, &stats_args); 4410 } 4411 break; 4412 } 4413 default: 4414 break; 4415 } 4416 } 4417 4418 void query_stats_schemas_cb(StatsSchemaList **result, Error **errp) 4419 { 4420 StatsArgs stats_args; 4421 KVMState *s = kvm_state; 4422 int stats_fd; 4423 4424 stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL); 4425 if (stats_fd == -1) { 4426 error_setg_errno(errp, errno, "KVM stats: ioctl failed"); 4427 return; 4428 } 4429 query_stats_schema(result, STATS_TARGET_VM, stats_fd, errp); 4430 close(stats_fd); 4431 4432 if (first_cpu) { 4433 stats_args.result.schema = result; 4434 stats_args.errp = errp; 4435 query_stats_schema_vcpu(first_cpu, &stats_args); 4436 } 4437 } 4438 4439 void kvm_mark_guest_state_protected(void) 4440 { 4441 kvm_state->guest_state_protected = true; 4442 } 4443 4444 int kvm_create_guest_memfd(uint64_t size, uint64_t flags, Error **errp) 4445 { 4446 int fd; 4447 struct kvm_create_guest_memfd guest_memfd = { 4448 .size = size, 4449 .flags = flags, 4450 }; 4451 4452 if (!kvm_guest_memfd_supported) { 4453 error_setg(errp, "KVM does not support guest_memfd"); 4454 return -1; 4455 } 4456 4457 fd = kvm_vm_ioctl(kvm_state, KVM_CREATE_GUEST_MEMFD, &guest_memfd); 4458 if (fd < 0) { 4459 error_setg_errno(errp, errno, "Error creating KVM guest_memfd"); 4460 return -1; 4461 } 4462 4463 return fd; 4464 } 4465