Lines Matching +full:- +full:- +full:disable +full:- +full:numa
10 * See the COPYING file in the top-level directory.
16 #include "qemu/error-report.h"
18 #include "system/numa.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "hw/virtio/virtio-mem.h"
30 #include "hw/qdev-properties.h"
87 * to disable THP in case we fail to properly probe the THP size, or if the
104 /* No THP -> no restrictions. */ in virtio_mem_thp_size()
117 /* Sanity-check the value and fallback to something reasonable. */ in virtio_mem_thp_size()
185 * pages on-demand with stale content when plugging new blocks. in virtio_mem_is_busy()
206 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); in virtio_mem_for_each_unplugged_range()
207 while (first_zero_bit < vmem->bitmap_size) { in virtio_mem_for_each_unplugged_range()
208 offset = first_zero_bit * vmem->block_size; in virtio_mem_for_each_unplugged_range()
209 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_range()
210 first_zero_bit + 1) - 1; in virtio_mem_for_each_unplugged_range()
211 size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; in virtio_mem_for_each_unplugged_range()
217 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_range()
230 first_bit = find_first_bit(vmem->bitmap, vmem->bitmap_size); in virtio_mem_for_each_plugged_range()
231 while (first_bit < vmem->bitmap_size) { in virtio_mem_for_each_plugged_range()
232 offset = first_bit * vmem->block_size; in virtio_mem_for_each_plugged_range()
233 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_range()
234 first_bit + 1) - 1; in virtio_mem_for_each_plugged_range()
235 size = (last_bit - first_bit + 1) * vmem->block_size; in virtio_mem_for_each_plugged_range()
241 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_range()
255 uint64_t start = MAX(s->offset_within_region, offset); in virtio_mem_intersect_memory_section()
256 uint64_t end = MIN(s->offset_within_region + int128_get64(s->size), in virtio_mem_intersect_memory_section()
263 s->offset_within_address_space += start - s->offset_within_region; in virtio_mem_intersect_memory_section()
264 s->offset_within_region = start; in virtio_mem_intersect_memory_section()
265 s->size = int128_make64(end - start); in virtio_mem_intersect_memory_section()
280 first_bit = s->offset_within_region / vmem->block_size; in virtio_mem_for_each_plugged_section()
281 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); in virtio_mem_for_each_plugged_section()
282 while (first_bit < vmem->bitmap_size) { in virtio_mem_for_each_plugged_section()
285 offset = first_bit * vmem->block_size; in virtio_mem_for_each_plugged_section()
286 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_section()
287 first_bit + 1) - 1; in virtio_mem_for_each_plugged_section()
288 size = (last_bit - first_bit + 1) * vmem->block_size; in virtio_mem_for_each_plugged_section()
297 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_section()
312 first_bit = s->offset_within_region / vmem->block_size; in virtio_mem_for_each_unplugged_section()
313 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); in virtio_mem_for_each_unplugged_section()
314 while (first_bit < vmem->bitmap_size) { in virtio_mem_for_each_unplugged_section()
317 offset = first_bit * vmem->block_size; in virtio_mem_for_each_unplugged_section()
318 last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_section()
319 first_bit + 1) - 1; in virtio_mem_for_each_unplugged_section()
320 size = (last_bit - first_bit + 1) * vmem->block_size; in virtio_mem_for_each_unplugged_section()
329 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_section()
339 return rdl->notify_populate(rdl, s); in virtio_mem_notify_populate_cb()
346 rdl->notify_discard(rdl, s); in virtio_mem_notify_discard_cb()
355 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_notify_unplug()
356 MemoryRegionSection tmp = *rdl->section; in virtio_mem_notify_unplug()
361 rdl->notify_discard(rdl, &tmp); in virtio_mem_notify_unplug()
371 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_notify_plug()
372 MemoryRegionSection tmp = *rdl->section; in virtio_mem_notify_plug()
377 ret = rdl->notify_populate(rdl, &tmp); in virtio_mem_notify_plug()
384 /* Notify all already-notified listeners. */ in virtio_mem_notify_plug()
385 QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { in virtio_mem_notify_plug()
386 MemoryRegionSection tmp = *rdl2->section; in virtio_mem_notify_plug()
394 rdl2->notify_discard(rdl2, &tmp); in virtio_mem_notify_plug()
404 if (!vmem->size) { in virtio_mem_notify_unplug_all()
408 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_notify_unplug_all()
409 if (rdl->double_discard_supported) { in virtio_mem_notify_unplug_all()
410 rdl->notify_discard(rdl, rdl->section); in virtio_mem_notify_unplug_all()
412 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_notify_unplug_all()
421 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_is_range_plugged()
422 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; in virtio_mem_is_range_plugged()
426 found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); in virtio_mem_is_range_plugged()
433 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_is_range_unplugged()
434 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; in virtio_mem_is_range_unplugged()
438 found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); in virtio_mem_is_range_unplugged()
445 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_set_range_plugged()
446 const unsigned long nbits = size / vmem->block_size; in virtio_mem_set_range_plugged()
448 bitmap_set(vmem->bitmap, bit, nbits); in virtio_mem_set_range_plugged()
454 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_set_range_unplugged()
455 const unsigned long nbits = size / vmem->block_size; in virtio_mem_set_range_unplugged()
457 bitmap_clear(vmem->bitmap, bit, nbits); in virtio_mem_set_range_unplugged()
464 VirtQueue *vq = vmem->vq; in virtio_mem_send_response()
466 trace_virtio_mem_send_response(le16_to_cpu(resp->type)); in virtio_mem_send_response()
467 iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); in virtio_mem_send_response()
487 if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { in virtio_mem_valid_range()
493 if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { in virtio_mem_valid_range()
496 if (gpa + size > vmem->addr + vmem->usable_region_size) { in virtio_mem_valid_range()
504 const uint64_t memslot_offset = idx * vmem->memslot_size; in virtio_mem_activate_memslot()
506 assert(vmem->memslots); in virtio_mem_activate_memslot()
513 if (memory_region_is_mapped(&vmem->memslots[idx])) { in virtio_mem_activate_memslot()
516 memory_region_add_subregion(vmem->mr, memslot_offset, &vmem->memslots[idx]); in virtio_mem_activate_memslot()
521 assert(vmem->memslots); in virtio_mem_deactivate_memslot()
523 if (!memory_region_is_mapped(&vmem->memslots[idx])) { in virtio_mem_deactivate_memslot()
526 memory_region_del_subregion(vmem->mr, &vmem->memslots[idx]); in virtio_mem_deactivate_memslot()
532 const unsigned int start_idx = offset / vmem->memslot_size; in virtio_mem_activate_memslots_to_plug()
533 const unsigned int end_idx = (offset + size + vmem->memslot_size - 1) / in virtio_mem_activate_memslots_to_plug()
534 vmem->memslot_size; in virtio_mem_activate_memslots_to_plug()
537 assert(vmem->dynamic_memslots); in virtio_mem_activate_memslots_to_plug()
551 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_deactivate_unplugged_memslots()
552 const unsigned int start_idx = offset / vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
553 const unsigned int end_idx = (offset + size + vmem->memslot_size - 1) / in virtio_mem_deactivate_unplugged_memslots()
554 vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
557 assert(vmem->dynamic_memslots); in virtio_mem_deactivate_unplugged_memslots()
562 const uint64_t memslot_offset = idx * vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
563 uint64_t memslot_size = vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
566 if (idx == vmem->nb_memslots - 1) { in virtio_mem_deactivate_unplugged_memslots()
567 memslot_size = region_size - memslot_offset; in virtio_mem_deactivate_unplugged_memslots()
576 const uint64_t gpa = vmem->addr + memslot_offset; in virtio_mem_deactivate_unplugged_memslots()
591 const uint64_t offset = start_gpa - vmem->addr; in virtio_mem_set_block_state()
592 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_set_block_state()
596 return -EBUSY; in virtio_mem_set_block_state()
601 return -EBUSY; in virtio_mem_set_block_state()
606 if (vmem->dynamic_memslots) { in virtio_mem_set_block_state()
612 if (vmem->prealloc) { in virtio_mem_set_block_state()
613 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; in virtio_mem_set_block_state()
614 int fd = memory_region_get_fd(&vmem->memdev->mr); in virtio_mem_set_block_state()
630 ret = -EBUSY; in virtio_mem_set_block_state()
644 if (vmem->dynamic_memslots) { in virtio_mem_set_block_state()
648 if (ret && vmem->dynamic_memslots) { in virtio_mem_set_block_state()
654 ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); in virtio_mem_set_block_state()
655 return -EBUSY; in virtio_mem_set_block_state()
665 const uint64_t size = nb_blocks * vmem->block_size; in virtio_mem_state_change_request()
672 if (plug && (vmem->size + size > vmem->requested_size)) { in virtio_mem_state_change_request()
687 vmem->size += size; in virtio_mem_state_change_request()
689 vmem->size -= size; in virtio_mem_state_change_request()
691 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); in virtio_mem_state_change_request()
698 const uint64_t gpa = le64_to_cpu(req->u.plug.addr); in virtio_mem_plug_request()
699 const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); in virtio_mem_plug_request()
710 const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); in virtio_mem_unplug_request()
711 const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); in virtio_mem_unplug_request()
723 uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), in virtio_mem_resize_usable_region()
727 newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); in virtio_mem_resize_usable_region()
733 if (newsize < vmem->usable_region_size && !can_shrink) { in virtio_mem_resize_usable_region()
737 trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); in virtio_mem_resize_usable_region()
738 vmem->usable_region_size = newsize; in virtio_mem_resize_usable_region()
743 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_unplug_all()
744 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_unplug_all()
746 if (vmem->size) { in virtio_mem_unplug_all()
748 return -EBUSY; in virtio_mem_unplug_all()
751 return -EBUSY; in virtio_mem_unplug_all()
755 bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); in virtio_mem_unplug_all()
756 vmem->size = 0; in virtio_mem_unplug_all()
757 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); in virtio_mem_unplug_all()
760 if (vmem->dynamic_memslots) { in virtio_mem_unplug_all()
766 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); in virtio_mem_unplug_all()
784 const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); in virtio_mem_state_request()
785 const uint64_t gpa = le64_to_cpu(req->u.state.addr); in virtio_mem_state_request()
786 const uint64_t size = nb_blocks * vmem->block_size; in virtio_mem_state_request()
822 if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { in virtio_mem_handle_request()
823 virtio_error(vdev, "virtio-mem protocol violation: invalid request" in virtio_mem_handle_request()
830 if (iov_size(elem->in_sg, elem->in_num) < in virtio_mem_handle_request()
832 virtio_error(vdev, "virtio-mem protocol violation: not enough space" in virtio_mem_handle_request()
834 iov_size(elem->in_sg, elem->in_num)); in virtio_mem_handle_request()
855 virtio_error(vdev, "virtio-mem protocol violation: unknown request" in virtio_mem_handle_request()
871 config->block_size = cpu_to_le64(vmem->block_size); in virtio_mem_get_config()
872 config->node_id = cpu_to_le16(vmem->node); in virtio_mem_get_config()
873 config->requested_size = cpu_to_le64(vmem->requested_size); in virtio_mem_get_config()
874 config->plugged_size = cpu_to_le64(vmem->size); in virtio_mem_get_config()
875 config->addr = cpu_to_le64(vmem->addr); in virtio_mem_get_config()
876 config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); in virtio_mem_get_config()
877 config->usable_region_size = cpu_to_le64(vmem->usable_region_size); in virtio_mem_get_config()
886 if (ms->numa_state && acpi_builtin()) { in virtio_mem_get_features()
889 assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); in virtio_mem_get_features()
890 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { in virtio_mem_get_features()
903 return -EFAULT; in virtio_mem_validate_features()
910 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_prepare_mr()
912 assert(!vmem->mr && vmem->dynamic_memslots); in virtio_mem_prepare_mr()
913 vmem->mr = g_new0(MemoryRegion, 1); in virtio_mem_prepare_mr()
914 memory_region_init(vmem->mr, OBJECT(vmem), "virtio-mem", in virtio_mem_prepare_mr()
916 vmem->mr->align = memory_region_get_alignment(&vmem->memdev->mr); in virtio_mem_prepare_mr()
921 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_prepare_memslots()
924 g_assert(!vmem->memslots && vmem->nb_memslots && vmem->dynamic_memslots); in virtio_mem_prepare_memslots()
925 vmem->memslots = g_new0(MemoryRegion, vmem->nb_memslots); in virtio_mem_prepare_memslots()
928 for (idx = 0; idx < vmem->nb_memslots; idx++) { in virtio_mem_prepare_memslots()
929 const uint64_t memslot_offset = idx * vmem->memslot_size; in virtio_mem_prepare_memslots()
930 uint64_t memslot_size = vmem->memslot_size; in virtio_mem_prepare_memslots()
934 if (idx == vmem->nb_memslots - 1) { in virtio_mem_prepare_memslots()
935 memslot_size = region_size - memslot_offset; in virtio_mem_prepare_memslots()
938 snprintf(name, sizeof(name), "memslot-%u", idx); in virtio_mem_prepare_memslots()
939 memory_region_init_alias(&vmem->memslots[idx], OBJECT(vmem), name, in virtio_mem_prepare_memslots()
940 &vmem->memdev->mr, memslot_offset, in virtio_mem_prepare_memslots()
947 memory_region_set_unmergeable(&vmem->memslots[idx], true); in virtio_mem_prepare_memslots()
954 int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; in virtio_mem_device_realize()
962 if (!vmem->memdev) { in virtio_mem_device_realize()
965 } else if (host_memory_backend_is_mapped(vmem->memdev)) { in virtio_mem_device_realize()
968 object_get_canonical_path_component(OBJECT(vmem->memdev))); in virtio_mem_device_realize()
970 } else if (!memory_region_is_ram(&vmem->memdev->mr) || in virtio_mem_device_realize()
971 memory_region_is_rom(&vmem->memdev->mr) || in virtio_mem_device_realize()
972 !vmem->memdev->mr.ram_block) { in virtio_mem_device_realize()
976 } else if (vmem->memdev->prealloc) { in virtio_mem_device_realize()
979 " virtio-mem device. ", VIRTIO_MEM_MEMDEV_PROP, in virtio_mem_device_realize()
980 object_get_canonical_path_component(OBJECT(vmem->memdev))); in virtio_mem_device_realize()
984 if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || in virtio_mem_device_realize()
985 (!nb_numa_nodes && vmem->node)) { in virtio_mem_device_realize()
987 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, in virtio_mem_device_realize()
988 vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); in virtio_mem_device_realize()
997 rb = vmem->memdev->mr.ram_block; in virtio_mem_device_realize()
1001 switch (vmem->unplugged_inaccessible) { in virtio_mem_device_realize()
1004 vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; in virtio_mem_device_realize()
1006 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; in virtio_mem_device_realize()
1020 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; in virtio_mem_device_realize()
1023 if (vmem->dynamic_memslots && in virtio_mem_device_realize()
1024 vmem->unplugged_inaccessible != ON_OFF_AUTO_ON) { in virtio_mem_device_realize()
1036 if (!vmem->block_size) { in virtio_mem_device_realize()
1037 vmem->block_size = virtio_mem_default_block_size(rb); in virtio_mem_device_realize()
1040 if (vmem->block_size < page_size) { in virtio_mem_device_realize()
1044 } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { in virtio_mem_device_realize()
1049 if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { in virtio_mem_device_realize()
1052 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); in virtio_mem_device_realize()
1054 } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { in virtio_mem_device_realize()
1057 vmem->block_size); in virtio_mem_device_realize()
1059 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), in virtio_mem_device_realize()
1060 vmem->block_size)) { in virtio_mem_device_realize()
1063 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); in virtio_mem_device_realize()
1074 * QEMU or migrated using the file content. "x-ignore-shared" will be in virtio_mem_device_realize()
1085 error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); in virtio_mem_device_realize()
1091 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); in virtio_mem_device_realize()
1093 vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / in virtio_mem_device_realize()
1094 vmem->block_size; in virtio_mem_device_realize()
1095 vmem->bitmap = bitmap_new(vmem->bitmap_size); in virtio_mem_device_realize()
1098 vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); in virtio_mem_device_realize()
1101 * With "dynamic-memslots=off" (old behavior) we always map the whole in virtio_mem_device_realize()
1104 if (vmem->dynamic_memslots) { in virtio_mem_device_realize()
1105 if (!vmem->mr) { in virtio_mem_device_realize()
1108 if (vmem->nb_memslots <= 1) { in virtio_mem_device_realize()
1109 vmem->nb_memslots = 1; in virtio_mem_device_realize()
1110 vmem->memslot_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_device_realize()
1112 if (!vmem->memslots) { in virtio_mem_device_realize()
1116 assert(!vmem->mr && !vmem->nb_memslots && !vmem->memslots); in virtio_mem_device_realize()
1119 host_memory_backend_set_mapped(vmem->memdev, true); in virtio_mem_device_realize()
1120 vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); in virtio_mem_device_realize()
1121 if (vmem->early_migration) { in virtio_mem_device_realize()
1128 * it is safe for the guest -- during system resets that call in virtio_mem_device_realize()
1144 vmem->system_reset = VIRTIO_MEM_SYSTEM_RESET(obj); in virtio_mem_device_realize()
1145 vmem->system_reset->vmem = vmem; in virtio_mem_device_realize()
1152 memory_region_set_ram_discard_manager(&vmem->memdev->mr, in virtio_mem_device_realize()
1165 memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); in virtio_mem_device_unrealize()
1167 qemu_unregister_resettable(OBJECT(vmem->system_reset)); in virtio_mem_device_unrealize()
1168 object_unref(OBJECT(vmem->system_reset)); in virtio_mem_device_unrealize()
1170 if (vmem->early_migration) { in virtio_mem_device_unrealize()
1174 vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); in virtio_mem_device_unrealize()
1175 host_memory_backend_set_mapped(vmem->memdev, false); in virtio_mem_device_unrealize()
1178 g_free(vmem->bitmap); in virtio_mem_device_unrealize()
1185 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_discard_range_cb()
1187 return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; in virtio_mem_discard_range_cb()
1213 if (vmem->dynamic_memslots) { in virtio_mem_post_load_bitmap()
1226 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_post_load_bitmap()
1227 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_post_load_bitmap()
1241 if (!vmem->early_migration) { in virtio_mem_post_load()
1252 if (migrate_ram_is_ignored(vmem->memdev->mr.ram_block)) { in virtio_mem_post_load()
1256 if (vmem->prealloc && !vmem->early_migration) { in virtio_mem_post_load()
1270 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; in virtio_mem_prealloc_range_cb()
1271 int fd = memory_region_get_fd(&vmem->memdev->mr); in virtio_mem_prealloc_range_cb()
1276 return -ENOMEM; in virtio_mem_prealloc_range_cb()
1284 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_post_load_early()
1287 if (!vmem->prealloc) { in virtio_mem_post_load_early()
1316 * However, we run after POSTCOPY_INCOMING_ADVISE -- but before actual in virtio_mem_post_load_early()
1326 return -EBUSY; in virtio_mem_post_load_early()
1346 VirtIOMEM *vmem = tmp->parent; in virtio_mem_mig_sanity_checks_pre_save()
1348 tmp->addr = vmem->addr; in virtio_mem_mig_sanity_checks_pre_save()
1349 tmp->region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_mig_sanity_checks_pre_save()
1350 tmp->block_size = vmem->block_size; in virtio_mem_mig_sanity_checks_pre_save()
1351 tmp->node = vmem->node; in virtio_mem_mig_sanity_checks_pre_save()
1358 VirtIOMEM *vmem = tmp->parent; in virtio_mem_mig_sanity_checks_post_load()
1359 const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_mig_sanity_checks_post_load()
1361 if (tmp->addr != vmem->addr) { in virtio_mem_mig_sanity_checks_post_load()
1363 VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); in virtio_mem_mig_sanity_checks_post_load()
1364 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1370 if (tmp->region_size != new_region_size) { in virtio_mem_mig_sanity_checks_post_load()
1372 PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, in virtio_mem_mig_sanity_checks_post_load()
1374 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1376 if (tmp->block_size != vmem->block_size) { in virtio_mem_mig_sanity_checks_post_load()
1378 VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, in virtio_mem_mig_sanity_checks_post_load()
1379 vmem->block_size); in virtio_mem_mig_sanity_checks_post_load()
1380 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1382 if (tmp->node != vmem->node) { in virtio_mem_mig_sanity_checks_post_load()
1384 VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); in virtio_mem_mig_sanity_checks_post_load()
1385 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1391 .name = "virtio-mem-device/sanity-checks",
1408 return !vmem->early_migration; in virtio_mem_vmstate_field_exists()
1412 .name = "virtio-mem-device",
1439 * vmstate_virtio_mem_device instead -- see virtio_mem_vmstate_field_exists().
1442 .name = "virtio-mem-device-early",
1457 .name = "virtio-mem",
1469 vi->memaddr = vmem->addr; in virtio_mem_fill_device_info()
1470 vi->node = vmem->node; in virtio_mem_fill_device_info()
1471 vi->requested_size = vmem->requested_size; in virtio_mem_fill_device_info()
1472 vi->size = vmem->size; in virtio_mem_fill_device_info()
1473 vi->max_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_fill_device_info()
1474 vi->block_size = vmem->block_size; in virtio_mem_fill_device_info()
1475 vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); in virtio_mem_fill_device_info()
1480 if (!vmem->memdev) { in virtio_mem_get_memory_region()
1483 } else if (vmem->dynamic_memslots) { in virtio_mem_get_memory_region()
1484 if (!vmem->mr) { in virtio_mem_get_memory_region()
1487 return vmem->mr; in virtio_mem_get_memory_region()
1490 return &vmem->memdev->mr; in virtio_mem_get_memory_region()
1499 if (!vmem->dynamic_memslots) { in virtio_mem_decide_memslots()
1504 assert(!vmem->nb_memslots); in virtio_mem_decide_memslots()
1507 if (limit <= 1 || !vmem->memdev || !vmem->memdev->mr.ram_block) { in virtio_mem_decide_memslots()
1508 vmem->nb_memslots = 1; in virtio_mem_decide_memslots()
1512 rb = vmem->memdev->mr.ram_block; in virtio_mem_decide_memslots()
1513 region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_decide_memslots()
1519 if (!vmem->block_size) { in virtio_mem_decide_memslots()
1520 vmem->block_size = virtio_mem_default_block_size(rb); in virtio_mem_decide_memslots()
1523 if (vmem->block_size < qemu_ram_pagesize(rb) || in virtio_mem_decide_memslots()
1524 !QEMU_IS_ALIGNED(region_size, vmem->block_size)) { in virtio_mem_decide_memslots()
1525 vmem->nb_memslots = 1; in virtio_mem_decide_memslots()
1533 memslot_size = QEMU_ALIGN_UP(region_size / limit, vmem->block_size); in virtio_mem_decide_memslots()
1534 min_memslot_size = MAX(vmem->block_size, VIRTIO_MEM_MIN_MEMSLOT_SIZE); in virtio_mem_decide_memslots()
1539 vmem->memslot_size = memslot_size; in virtio_mem_decide_memslots()
1541 vmem->nb_memslots = memslots; in virtio_mem_decide_memslots()
1546 if (!vmem->dynamic_memslots) { in virtio_mem_get_memslots()
1552 g_assert(vmem->nb_memslots); in virtio_mem_get_memslots()
1553 return vmem->nb_memslots; in virtio_mem_get_memslots()
1559 notifier_list_add(&vmem->size_change_notifiers, notifier); in virtio_mem_add_size_change_notifier()
1572 uint64_t value = vmem->size; in virtio_mem_get_size()
1582 uint64_t value = vmem->requested_size; in virtio_mem_get_requested_size()
1602 if (DEVICE(obj)->realized) { in virtio_mem_set_requested_size()
1603 if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { in virtio_mem_set_requested_size()
1606 vmem->block_size); in virtio_mem_set_requested_size()
1608 } else if (value > memory_region_size(&vmem->memdev->mr)) { in virtio_mem_set_requested_size()
1611 memory_region_size(&vmem->memdev->mr)); in virtio_mem_set_requested_size()
1615 if (value != vmem->requested_size) { in virtio_mem_set_requested_size()
1617 vmem->requested_size = value; in virtio_mem_set_requested_size()
1625 vmem->requested_size = value; in virtio_mem_set_requested_size()
1633 uint64_t value = vmem->block_size; in virtio_mem_get_block_size()
1640 if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { in virtio_mem_get_block_size()
1641 value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); in virtio_mem_get_block_size()
1656 if (DEVICE(obj)->realized) { in virtio_mem_set_block_size()
1673 vmem->block_size = value; in virtio_mem_set_block_size()
1680 notifier_list_init(&vmem->size_change_notifiers); in virtio_mem_instance_init()
1681 QLIST_INIT(&vmem->rdl_list); in virtio_mem_instance_init()
1702 g_free(vmem->memslots); in virtio_mem_instance_finalize()
1703 vmem->memslots = NULL; in virtio_mem_instance_finalize()
1704 g_free(vmem->mr); in virtio_mem_instance_finalize()
1705 vmem->mr = NULL; in virtio_mem_instance_finalize()
1729 g_assert(mr == &vmem->memdev->mr); in virtio_mem_rdm_get_min_granularity()
1730 return vmem->block_size; in virtio_mem_rdm_get_min_granularity()
1737 uint64_t start_gpa = vmem->addr + s->offset_within_region; in virtio_mem_rdm_is_populated()
1738 uint64_t end_gpa = start_gpa + int128_get64(s->size); in virtio_mem_rdm_is_populated()
1740 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_is_populated()
1742 start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); in virtio_mem_rdm_is_populated()
1743 end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); in virtio_mem_rdm_is_populated()
1745 if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { in virtio_mem_rdm_is_populated()
1749 return virtio_mem_is_range_plugged(vmem, start_gpa, end_gpa - start_gpa); in virtio_mem_rdm_is_populated()
1761 return ((ReplayRamPopulate)data->fn)(s, data->opaque); in virtio_mem_rdm_replay_populated_cb()
1775 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_replay_populated()
1785 ((ReplayRamDiscard)data->fn)(s, data->opaque); in virtio_mem_rdm_replay_discarded_cb()
1800 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_replay_discarded()
1812 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_register_listener()
1813 rdl->section = memory_region_section_new_copy(s); in virtio_mem_rdm_register_listener()
1815 QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); in virtio_mem_rdm_register_listener()
1816 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_rdm_register_listener()
1820 strerror(-ret)); in virtio_mem_rdm_register_listener()
1829 g_assert(rdl->section->mr == &vmem->memdev->mr); in virtio_mem_rdm_unregister_listener()
1830 if (vmem->size) { in virtio_mem_rdm_unregister_listener()
1831 if (rdl->double_discard_supported) { in virtio_mem_rdm_unregister_listener()
1832 rdl->notify_discard(rdl, rdl->section); in virtio_mem_rdm_unregister_listener()
1834 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_rdm_unregister_listener()
1839 memory_region_section_free_copy(rdl->section); in virtio_mem_rdm_unregister_listener()
1840 rdl->section = NULL; in virtio_mem_rdm_unregister_listener()
1846 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_OFF) { in virtio_mem_unplug_request_check()
1851 error_setg(errp, "virtio-mem device cannot get unplugged while" in virtio_mem_unplug_request_check()
1856 if (vmem->size) { in virtio_mem_unplug_request_check()
1857 error_setg(errp, "virtio-mem device cannot get unplugged while some" in virtio_mem_unplug_request_check()
1861 if (vmem->requested_size) { in virtio_mem_unplug_request_check()
1862 error_setg(errp, "virtio-mem device cannot get unplugged while" in virtio_mem_unplug_request_check()
1876 dc->vmsd = &vmstate_virtio_mem; in virtio_mem_class_init()
1878 set_bit(DEVICE_CATEGORY_MISC, dc->categories); in virtio_mem_class_init()
1879 vdc->realize = virtio_mem_device_realize; in virtio_mem_class_init()
1880 vdc->unrealize = virtio_mem_device_unrealize; in virtio_mem_class_init()
1881 vdc->get_config = virtio_mem_get_config; in virtio_mem_class_init()
1882 vdc->get_features = virtio_mem_get_features; in virtio_mem_class_init()
1883 vdc->validate_features = virtio_mem_validate_features; in virtio_mem_class_init()
1884 vdc->vmsd = &vmstate_virtio_mem_device; in virtio_mem_class_init()
1886 vmc->fill_device_info = virtio_mem_fill_device_info; in virtio_mem_class_init()
1887 vmc->get_memory_region = virtio_mem_get_memory_region; in virtio_mem_class_init()
1888 vmc->decide_memslots = virtio_mem_decide_memslots; in virtio_mem_class_init()
1889 vmc->get_memslots = virtio_mem_get_memslots; in virtio_mem_class_init()
1890 vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; in virtio_mem_class_init()
1891 vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; in virtio_mem_class_init()
1892 vmc->unplug_request_check = virtio_mem_unplug_request_check; in virtio_mem_class_init()
1894 rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; in virtio_mem_class_init()
1895 rdmc->is_populated = virtio_mem_rdm_is_populated; in virtio_mem_class_init()
1896 rdmc->replay_populated = virtio_mem_rdm_replay_populated; in virtio_mem_class_init()
1897 rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; in virtio_mem_class_init()
1898 rdmc->register_listener = virtio_mem_rdm_register_listener; in virtio_mem_class_init()
1899 rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; in virtio_mem_class_init()
1937 return &vmem_reset->reset_state; in virtio_mem_system_reset_get_state()
1943 VirtIOMEM *vmem = vmem_reset->vmem; in virtio_mem_system_reset_hold()
1946 * When waking up from standby/suspend-to-ram, do not unplug any memory. in virtio_mem_system_reset_hold()
1965 rc->get_state = virtio_mem_system_reset_get_state; in virtio_mem_system_reset_class_init()
1966 rc->phases.hold = virtio_mem_system_reset_hold; in virtio_mem_system_reset_class_init()