Lines Matching +full:wakeup +full:- +full:source
4 * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates
10 * See the COPYING file in the top-level directory.
16 * source to the destination before all the data has been copied.
23 #include "qemu-file.h"
25 #include "postcopy-ram.h"
31 #include "qemu/error-report.h"
39 #include "qemu/mmap-alloc.h"
93 qemu_event_init(&mis->thread_sync_event, false); in postcopy_thread_create()
95 qemu_event_wait(&mis->thread_sync_event); in postcopy_thread_create()
96 qemu_event_destroy(&mis->thread_sync_event); in postcopy_thread_create()
136 g_free(ctx->page_fault_vcpu_time); in destroy_blocktime_context()
137 g_free(ctx->vcpu_addr); in destroy_blocktime_context()
138 g_free(ctx->vcpu_blocktime); in destroy_blocktime_context()
152 unsigned int smp_cpus = ms->smp.cpus; in blocktime_context_new()
154 ctx->page_fault_vcpu_time = g_new0(uint32_t, smp_cpus); in blocktime_context_new()
155 ctx->vcpu_addr = g_new0(uintptr_t, smp_cpus); in blocktime_context_new()
156 ctx->vcpu_blocktime = g_new0(uint32_t, smp_cpus); in blocktime_context_new()
158 ctx->exit_notifier.notify = migration_exit_cb; in blocktime_context_new()
159 ctx->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); in blocktime_context_new()
160 qemu_add_exit_notifier(&ctx->exit_notifier); in blocktime_context_new()
170 for (i = ms->smp.cpus - 1; i >= 0; i--) { in get_vcpu_blocktime_list()
171 QAPI_LIST_PREPEND(list, ctx->vcpu_blocktime[i]); in get_vcpu_blocktime_list()
180 * unless postcopy-blocktime capability was set.
187 PostcopyBlocktimeContext *bc = mis->blocktime_ctx; in fill_destination_postcopy_migration_info()
193 info->has_postcopy_blocktime = true; in fill_destination_postcopy_migration_info()
194 info->postcopy_blocktime = bc->total_blocktime; in fill_destination_postcopy_migration_info()
195 info->has_postcopy_vcpu_blocktime = true; in fill_destination_postcopy_migration_info()
196 info->postcopy_vcpu_blocktime = get_vcpu_blocktime_list(bc); in fill_destination_postcopy_migration_info()
202 PostcopyBlocktimeContext *bc = mis->blocktime_ctx; in get_postcopy_total_blocktime()
208 return bc->total_blocktime; in get_postcopy_total_blocktime()
217 * __NR_userfaultfd - should be checked before
228 if (ufd == -1) { in receive_ufd_features()
306 if (!mis->blocktime_ctx) { in ufd_check_and_apply()
307 mis->blocktime_ctx = blocktime_context_new(); in ufd_check_and_apply()
355 if (rb->fd >= 0) { in test_ramblock_postcopiable()
356 fs = qemu_fd_getfs(rb->fd); in test_ramblock_postcopiable()
376 int ufd = -1; in postcopy_ram_supported_by_host()
390 if (ufd == -1) { in postcopy_ram_supported_by_host()
439 MAP_ANONYMOUS, -1, 0); in postcopy_ram_supported_by_host()
477 if (ufd != -1) { in postcopy_ram_supported_by_host()
485 * must be done right at the start prior to pre-copy.
498 * resize RAM blocks when syncing RAM block sizes from the source during in init_range()
501 rb->postcopy_length = length; in init_range()
506 * - we're going to get the copy from the source anyway. in init_range()
510 return -1; in init_range()
525 ram_addr_t length = rb->postcopy_length; in cleanup_range()
544 if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) { in cleanup_range()
547 return -1; in cleanup_range()
554 * Initialise postcopy-ram, setting the RAM to a state where we can go into
561 return -1; in postcopy_ram_incoming_init()
571 if (mis->postcopy_tmp_pages) { in postcopy_temp_pages_cleanup()
572 for (i = 0; i < mis->postcopy_channels; i++) { in postcopy_temp_pages_cleanup()
573 if (mis->postcopy_tmp_pages[i].tmp_huge_page) { in postcopy_temp_pages_cleanup()
574 munmap(mis->postcopy_tmp_pages[i].tmp_huge_page, in postcopy_temp_pages_cleanup()
575 mis->largest_page_size); in postcopy_temp_pages_cleanup()
576 mis->postcopy_tmp_pages[i].tmp_huge_page = NULL; in postcopy_temp_pages_cleanup()
579 g_free(mis->postcopy_tmp_pages); in postcopy_temp_pages_cleanup()
580 mis->postcopy_tmp_pages = NULL; in postcopy_temp_pages_cleanup()
583 if (mis->postcopy_tmp_zero_page) { in postcopy_temp_pages_cleanup()
584 munmap(mis->postcopy_tmp_zero_page, mis->largest_page_size); in postcopy_temp_pages_cleanup()
585 mis->postcopy_tmp_zero_page = NULL; in postcopy_temp_pages_cleanup()
596 if (mis->preempt_thread_status == PREEMPT_THREAD_CREATED) { in postcopy_ram_incoming_cleanup()
598 mis->preempt_thread_status = PREEMPT_THREAD_QUIT; in postcopy_ram_incoming_cleanup()
611 WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) { in postcopy_ram_incoming_cleanup()
612 if (qatomic_read(&mis->page_requested_count)) { in postcopy_ram_incoming_cleanup()
618 qemu_cond_wait(&mis->page_request_cond, in postcopy_ram_incoming_cleanup()
619 &mis->page_request_mutex); in postcopy_ram_incoming_cleanup()
623 if (mis->postcopy_qemufile_dst) { in postcopy_ram_incoming_cleanup()
624 qemu_file_shutdown(mis->postcopy_qemufile_dst); in postcopy_ram_incoming_cleanup()
626 qemu_thread_join(&mis->postcopy_prio_thread); in postcopy_ram_incoming_cleanup()
627 mis->preempt_thread_status = PREEMPT_THREAD_NONE; in postcopy_ram_incoming_cleanup()
630 if (mis->have_fault_thread) { in postcopy_ram_incoming_cleanup()
634 qatomic_set(&mis->fault_thread_quit, 1); in postcopy_ram_incoming_cleanup()
637 qemu_thread_join(&mis->fault_thread); in postcopy_ram_incoming_cleanup()
641 return -1; in postcopy_ram_incoming_cleanup()
645 return -1; in postcopy_ram_incoming_cleanup()
649 close(mis->userfault_fd); in postcopy_ram_incoming_cleanup()
650 close(mis->userfault_event_fd); in postcopy_ram_incoming_cleanup()
651 mis->have_fault_thread = false; in postcopy_ram_incoming_cleanup()
681 ram_addr_t length = rb->postcopy_length; in nhp_range()
702 return -1; in postcopy_ram_prepare_discard()
725 reg_struct.range.len = rb->postcopy_length; in ram_block_enable_notify()
729 if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, ®_struct)) { in ram_block_enable_notify()
731 return -1; in ram_block_enable_notify()
735 return -1; in ram_block_enable_notify()
750 return uffd_wakeup(pcfd->fd, in postcopy_wake_shared()
789 trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb), in postcopy_request_shared_page()
792 trace_postcopy_request_shared_page_present(pcfd->idstr, in postcopy_request_shared_page()
805 if (cpu_iter->thread_id == pid) { in get_mem_fault_cpu_index()
806 trace_get_mem_fault_cpu_index(cpu_iter->cpu_index, pid); in get_mem_fault_cpu_index()
807 return cpu_iter->cpu_index; in get_mem_fault_cpu_index()
810 trace_get_mem_fault_cpu_index(-1, pid); in get_mem_fault_cpu_index()
811 return -1; in get_mem_fault_cpu_index()
816 int64_t start_time_offset = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - in get_low_time_offset()
817 dc->start_time; in get_low_time_offset()
834 PostcopyBlocktimeContext *dc = mis->blocktime_ctx; in mark_postcopy_blocktime_begin()
846 if (dc->vcpu_addr[cpu] == 0) { in mark_postcopy_blocktime_begin()
847 qatomic_inc(&dc->smp_cpus_down); in mark_postcopy_blocktime_begin()
850 qatomic_xchg(&dc->last_begin, low_time_offset); in mark_postcopy_blocktime_begin()
851 qatomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset); in mark_postcopy_blocktime_begin()
852 qatomic_xchg(&dc->vcpu_addr[cpu], addr); in mark_postcopy_blocktime_begin()
861 qatomic_xchg(&dc->vcpu_addr[cpu], 0); in mark_postcopy_blocktime_begin()
862 qatomic_xchg(&dc->page_fault_vcpu_time[cpu], 0); in mark_postcopy_blocktime_begin()
863 qatomic_dec(&dc->smp_cpus_down); in mark_postcopy_blocktime_begin()
865 trace_mark_postcopy_blocktime_begin(addr, dc, dc->page_fault_vcpu_time[cpu], in mark_postcopy_blocktime_begin()
877 * -----***********------------xxx***************------------------------> CPU1
880 * ------------****************xxx---------------------------------------> CPU2
883 * ------------------------****xxx********-------------------------------> CPU3
886 * S2,E1 - doesn't match condition due to sequence S1,S2,E1 doesn't include CPU3
887 * S3,S1,E2 - sequence includes all CPUs, in this case overlap will be S1,E2 -
889 * S1 - here is last_begin
891 * * - means blocktime per vCPU
892 * x - means overlapped blocktime (total blocktime)
899 PostcopyBlocktimeContext *dc = mis->blocktime_ctx; in mark_postcopy_blocktime_end()
901 unsigned int smp_cpus = ms->smp.cpus; in mark_postcopy_blocktime_end()
918 read_vcpu_time = qatomic_fetch_add(&dc->page_fault_vcpu_time[i], 0); in mark_postcopy_blocktime_end()
919 if (qatomic_fetch_add(&dc->vcpu_addr[i], 0) != addr || in mark_postcopy_blocktime_end()
923 qatomic_xchg(&dc->vcpu_addr[i], 0); in mark_postcopy_blocktime_end()
924 vcpu_blocktime = low_time_offset - read_vcpu_time; in mark_postcopy_blocktime_end()
930 qatomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) { in mark_postcopy_blocktime_end()
934 dc->vcpu_blocktime[i] += vcpu_blocktime; in mark_postcopy_blocktime_end()
937 qatomic_sub(&dc->smp_cpus_down, affected_cpu); in mark_postcopy_blocktime_end()
939 dc->total_blocktime += low_time_offset - qatomic_fetch_add( in mark_postcopy_blocktime_end()
940 &dc->last_begin, 0); in mark_postcopy_blocktime_end()
942 trace_mark_postcopy_blocktime_end(addr, dc, dc->total_blocktime, in mark_postcopy_blocktime_end()
949 qemu_sem_wait(&mis->postcopy_pause_sem_fault); in postcopy_pause_fault_thread()
966 mis->last_rb = NULL; /* last RAMBlock we sent part of */ in postcopy_ram_fault_thread()
967 qemu_event_set(&mis->thread_sync_event); in postcopy_ram_fault_thread()
970 size_t pfd_len = 2 + mis->postcopy_remote_fds->len; in postcopy_ram_fault_thread()
974 pfd[0].fd = mis->userfault_fd; in postcopy_ram_fault_thread()
976 pfd[1].fd = mis->userfault_event_fd; in postcopy_ram_fault_thread()
979 for (index = 0; index < mis->postcopy_remote_fds->len; index++) { in postcopy_ram_fault_thread()
980 struct PostCopyFD *pcfd = &g_array_index(mis->postcopy_remote_fds, in postcopy_ram_fault_thread()
982 pfd[2 + index].fd = pcfd->fd; in postcopy_ram_fault_thread()
984 trace_postcopy_ram_fault_thread_fds_extra(2 + index, pcfd->idstr, in postcopy_ram_fault_thread()
985 pcfd->fd); in postcopy_ram_fault_thread()
998 poll_result = poll(pfd, pfd_len, -1 /* Wait forever */); in postcopy_ram_fault_thread()
999 if (poll_result == -1) { in postcopy_ram_fault_thread()
1004 if (!mis->to_src_file) { in postcopy_ram_fault_thread()
1017 if (read(mis->userfault_event_fd, &tmp64, 8) != 8) { in postcopy_ram_fault_thread()
1022 if (qatomic_read(&mis->fault_thread_quit)) { in postcopy_ram_fault_thread()
1029 poll_result--; in postcopy_ram_fault_thread()
1030 ret = read(mis->userfault_fd, &msg, sizeof(msg)); in postcopy_ram_fault_thread()
1077 * Send the request to the source - we want to request one in postcopy_ram_fault_thread()
1094 &g_array_index(mis->postcopy_remote_fds, in postcopy_ram_fault_thread()
1095 struct PostCopyFD, index - 2); in postcopy_ram_fault_thread()
1097 poll_result--; in postcopy_ram_fault_thread()
1100 __func__, index, pcfd->fd); in postcopy_ram_fault_thread()
1105 ret = read(pcfd->fd, &msg, sizeof(msg)); in postcopy_ram_fault_thread()
1136 ret = pcfd->handler(pcfd, &msg); in postcopy_ram_fault_thread()
1139 __func__, index, pcfd->idstr); in postcopy_ram_fault_thread()
1159 mis->postcopy_channels = RAM_CHANNEL_MAX; in postcopy_temp_pages_setup()
1162 mis->postcopy_channels = 1; in postcopy_temp_pages_setup()
1165 channels = mis->postcopy_channels; in postcopy_temp_pages_setup()
1166 mis->postcopy_tmp_pages = g_malloc0_n(sizeof(PostcopyTmpPage), channels); in postcopy_temp_pages_setup()
1169 tmp_page = &mis->postcopy_tmp_pages[i]; in postcopy_temp_pages_setup()
1170 temp_page = mmap(NULL, mis->largest_page_size, PROT_READ | PROT_WRITE, in postcopy_temp_pages_setup()
1171 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); in postcopy_temp_pages_setup()
1177 return -err; in postcopy_temp_pages_setup()
1179 tmp_page->tmp_huge_page = temp_page; in postcopy_temp_pages_setup()
1187 mis->postcopy_tmp_zero_page = mmap(NULL, mis->largest_page_size, in postcopy_temp_pages_setup()
1189 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); in postcopy_temp_pages_setup()
1190 if (mis->postcopy_tmp_zero_page == MAP_FAILED) { in postcopy_temp_pages_setup()
1192 mis->postcopy_tmp_zero_page = NULL; in postcopy_temp_pages_setup()
1195 return -err; in postcopy_temp_pages_setup()
1198 memset(mis->postcopy_tmp_zero_page, '\0', mis->largest_page_size); in postcopy_temp_pages_setup()
1208 mis->userfault_fd = uffd_open(O_CLOEXEC | O_NONBLOCK); in postcopy_ram_incoming_setup()
1209 if (mis->userfault_fd == -1) { in postcopy_ram_incoming_setup()
1212 return -1; in postcopy_ram_incoming_setup()
1219 if (!ufd_check_and_apply(mis->userfault_fd, mis, &local_err)) { in postcopy_ram_incoming_setup()
1221 return -1; in postcopy_ram_incoming_setup()
1224 /* Now an eventfd we use to tell the fault-thread to quit */ in postcopy_ram_incoming_setup()
1225 mis->userfault_event_fd = eventfd(0, EFD_CLOEXEC); in postcopy_ram_incoming_setup()
1226 if (mis->userfault_event_fd == -1) { in postcopy_ram_incoming_setup()
1229 close(mis->userfault_fd); in postcopy_ram_incoming_setup()
1230 return -1; in postcopy_ram_incoming_setup()
1233 postcopy_thread_create(mis, &mis->fault_thread, in postcopy_ram_incoming_setup()
1236 mis->have_fault_thread = true; in postcopy_ram_incoming_setup()
1241 return -1; in postcopy_ram_incoming_setup()
1245 /* Error dumped in the sub-function */ in postcopy_ram_incoming_setup()
1246 return -1; in postcopy_ram_incoming_setup()
1254 postcopy_thread_create(mis, &mis->postcopy_prio_thread, in postcopy_ram_incoming_setup()
1257 mis->preempt_thread_status = PREEMPT_THREAD_CREATED; in postcopy_ram_incoming_setup()
1268 int userfault_fd = mis->userfault_fd; in qemu_ufd_copy_ioctl()
1278 qemu_mutex_lock(&mis->page_request_mutex); in qemu_ufd_copy_ioctl()
1285 if (g_tree_lookup(mis->page_requested, host_addr)) { in qemu_ufd_copy_ioctl()
1286 g_tree_remove(mis->page_requested, host_addr); in qemu_ufd_copy_ioctl()
1287 int left_pages = qatomic_dec_fetch(&mis->page_requested_count); in qemu_ufd_copy_ioctl()
1289 trace_postcopy_page_req_del(host_addr, mis->page_requested_count); in qemu_ufd_copy_ioctl()
1292 if (mis->preempt_thread_status == PREEMPT_THREAD_QUIT && in qemu_ufd_copy_ioctl()
1299 qemu_cond_signal(&mis->page_request_cond); in qemu_ufd_copy_ioctl()
1302 qemu_mutex_unlock(&mis->page_request_mutex); in qemu_ufd_copy_ioctl()
1312 GArray *pcrfds = mis->postcopy_remote_fds; in postcopy_notify_shared_wake()
1314 for (i = 0; i < pcrfds->len; i++) { in postcopy_notify_shared_wake()
1316 int ret = cur->waker(cur, rb, offset); in postcopy_notify_shared_wake()
1372 return postcopy_place_page(mis, host, mis->postcopy_tmp_zero_page, rb); in postcopy_place_page_zero()
1391 return -1; in postcopy_ram_incoming_init()
1435 /* ------------------------------------------------------------------------- */
1438 tmp_page->target_pages = 0; in postcopy_temp_page_reset()
1439 tmp_page->host_addr = NULL; in postcopy_temp_page_reset()
1442 * of the non-zero small page within this huge page. in postcopy_temp_page_reset()
1444 tmp_page->all_zero = true; in postcopy_temp_page_reset()
1452 * Wakeup the fault_thread. It's an eventfd that should currently in postcopy_fault_thread_notify()
1455 if (write(mis->userfault_event_fd, &tmp64, 8) != 8) { in postcopy_fault_thread_notify()
1501 qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, in postcopy_discard_send_range()
1521 qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, in postcopy_discard_send_finish()
1558 mis->postcopy_remote_fds = g_array_append_val(mis->postcopy_remote_fds, in postcopy_register_shared_ufd()
1568 GArray *pcrfds = mis->postcopy_remote_fds; in postcopy_unregister_shared_ufd()
1574 for (i = 0; i < pcrfds->len; i++) { in postcopy_unregister_shared_ufd()
1576 if (cur->fd == pcfd->fd) { in postcopy_unregister_shared_ufd()
1577 mis->postcopy_remote_fds = g_array_remove_index(pcrfds, i); in postcopy_unregister_shared_ufd()
1590 mis->postcopy_qemufile_dst = file; in postcopy_preempt_new_channel()
1591 qemu_sem_post(&mis->postcopy_qemufile_dst_done); in postcopy_preempt_new_channel()
1608 s->postcopy_qemufile_src = qemu_file_new_output(ioc); in postcopy_preempt_send_channel_done()
1616 qemu_sem_post(&s->postcopy_qemufile_src_sem); in postcopy_preempt_send_channel_done()
1643 tioc = migration_tls_client_create(ioc, s->hostname, &local_err); in postcopy_preempt_send_channel_new()
1648 qio_channel_set_name(QIO_CHANNEL(tioc), "migration-tls-preempt"); in postcopy_preempt_send_channel_new()
1663 * channel established, -1 for error.
1677 if (!s->preempt_pre_7_2) { in postcopy_preempt_establish_channel()
1685 qemu_sem_wait(&s->postcopy_qemufile_src_sem); in postcopy_preempt_establish_channel()
1687 return s->postcopy_qemufile_src ? 0 : -1; in postcopy_preempt_establish_channel()
1699 qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex); in postcopy_pause_ram_fast_load()
1700 qemu_sem_wait(&mis->postcopy_pause_sem_fast_load); in postcopy_pause_ram_fast_load()
1701 qemu_mutex_lock(&mis->postcopy_prio_thread_mutex); in postcopy_pause_ram_fast_load()
1707 return mis->preempt_thread_status != PREEMPT_THREAD_QUIT; in preempt_thread_should_run()
1719 qemu_event_set(&mis->thread_sync_event); in postcopy_preempt_thread()
1725 qemu_sem_wait(&mis->postcopy_qemufile_dst_done); in postcopy_preempt_thread()
1728 qemu_mutex_lock(&mis->postcopy_prio_thread_mutex); in postcopy_preempt_thread()
1730 ret = ram_load_postcopy(mis->postcopy_qemufile_dst, in postcopy_preempt_thread()
1740 qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex); in postcopy_preempt_thread()