113     VFIOMultifd *multifd = migration->multifd;  in vfio_load_state_buffer_insert()  local
116     vfio_state_buffers_assert_init(&multifd->load_bufs);  in vfio_load_state_buffer_insert()
117     if (packet->idx >= vfio_state_buffers_size_get(&multifd->load_bufs)) {  in vfio_load_state_buffer_insert()
118         vfio_state_buffers_size_set(&multifd->load_bufs, packet->idx + 1);  in vfio_load_state_buffer_insert()
121     lb = vfio_state_buffers_at(&multifd->load_bufs, packet->idx);  in vfio_load_state_buffer_insert()
128     assert(packet->idx >= multifd->load_buf_idx);  in vfio_load_state_buffer_insert()
142     VFIOMultifd *multifd = migration->multifd;  in vfio_multifd_load_state_buffer()  local
152     assert(multifd);  in vfio_multifd_load_state_buffer()
183     WITH_QEMU_LOCK_GUARD(&multifd->load_bufs_mutex) {  in vfio_multifd_load_state_buffer()
186             multifd->load_buf_idx_last = packet->idx;  in vfio_multifd_load_state_buffer()
194         qemu_cond_signal(&multifd->load_bufs_buffer_ready_cond);  in vfio_multifd_load_state_buffer()
204     VFIOMultifd *multifd = migration->multifd;  in vfio_load_bufs_thread_load_config()  local
211     assert(multifd->load_buf_idx == multifd->load_buf_idx_last);  in vfio_load_bufs_thread_load_config()
212     lb = vfio_state_buffers_at(&multifd->load_bufs, multifd->load_buf_idx);  in vfio_load_bufs_thread_load_config()
251 static VFIOStateBuffer *vfio_load_state_buffer_get(VFIOMultifd *multifd)  in vfio_load_state_buffer_get()  argument
256     bufs_len = vfio_state_buffers_size_get(&multifd->load_bufs);  in vfio_load_state_buffer_get()
257     if (multifd->load_buf_idx >= bufs_len) {  in vfio_load_state_buffer_get()
258         assert(multifd->load_buf_idx == bufs_len);  in vfio_load_state_buffer_get()
262     lb = vfio_state_buffers_at(&multifd->load_bufs,  in vfio_load_state_buffer_get()
263                                multifd->load_buf_idx);  in vfio_load_state_buffer_get()
276     VFIOMultifd *multifd = migration->multifd;  in vfio_load_state_buffer_write()  local
286                                                    multifd->load_buf_idx);  in vfio_load_state_buffer_write()
300         qemu_mutex_unlock(&multifd->load_bufs_mutex);  in vfio_load_state_buffer_write()
303         qemu_mutex_lock(&multifd->load_bufs_mutex);  in vfio_load_state_buffer_write()
308                        vbasedev->name, multifd->load_buf_idx, errno_save);  in vfio_load_state_buffer_write()
318                                                  multifd->load_buf_idx);  in vfio_load_state_buffer_write()
323 static bool vfio_load_bufs_thread_want_exit(VFIOMultifd *multifd,  in vfio_load_bufs_thread_want_exit()  argument
326     return multifd->load_bufs_thread_want_exit || qatomic_read(should_quit);  in vfio_load_bufs_thread_want_exit()
345     VFIOMultifd *multifd = migration->multifd;  in vfio_load_bufs_thread()  local
350     assert(multifd);  in vfio_load_bufs_thread()
351     QEMU_LOCK_GUARD(&multifd->load_bufs_mutex);  in vfio_load_bufs_thread()
353     assert(multifd->load_bufs_thread_running);  in vfio_load_bufs_thread()
362         if (vfio_load_bufs_thread_want_exit(multifd, should_quit)) {  in vfio_load_bufs_thread()
367         assert(multifd->load_buf_idx <= multifd->load_buf_idx_last);  in vfio_load_bufs_thread()
369         lb = vfio_load_state_buffer_get(multifd);  in vfio_load_bufs_thread()
372                                                         multifd->load_buf_idx);  in vfio_load_bufs_thread()
373             qemu_cond_wait(&multifd->load_bufs_buffer_ready_cond,  in vfio_load_bufs_thread()
374                            &multifd->load_bufs_mutex);  in vfio_load_bufs_thread()
378         if (multifd->load_buf_idx == multifd->load_buf_idx_last) {  in vfio_load_bufs_thread()
382         if (multifd->load_buf_idx == 0) {  in vfio_load_bufs_thread()
390         if (multifd->load_buf_idx == multifd->load_buf_idx_last - 1) {  in vfio_load_bufs_thread()
394         multifd->load_buf_idx++;  in vfio_load_bufs_thread()
408     multifd->load_bufs_thread_running = false;  in vfio_load_bufs_thread()
409     qemu_cond_signal(&multifd->load_bufs_thread_finished_cond);  in vfio_load_bufs_thread()
418     VFIOMultifd *multifd = g_new(VFIOMultifd, 1);  in vfio_multifd_new()  local
420     vfio_state_buffers_init(&multifd->load_bufs);  in vfio_multifd_new()
422     qemu_mutex_init(&multifd->load_bufs_mutex);  in vfio_multifd_new()
424     multifd->load_buf_idx = 0;  in vfio_multifd_new()
425     multifd->load_buf_idx_last = UINT32_MAX;  in vfio_multifd_new()
426     qemu_cond_init(&multifd->load_bufs_buffer_ready_cond);  in vfio_multifd_new()
428     multifd->load_bufs_thread_running = false;  in vfio_multifd_new()
429     multifd->load_bufs_thread_want_exit = false;  in vfio_multifd_new()
430     qemu_cond_init(&multifd->load_bufs_thread_finished_cond);  in vfio_multifd_new()
432     return multifd;  in vfio_multifd_new()
442 static void vfio_load_cleanup_load_bufs_thread(VFIOMultifd *multifd)  in vfio_load_cleanup_load_bufs_thread()  argument
446     WITH_QEMU_LOCK_GUARD(&multifd->load_bufs_mutex) {  in vfio_load_cleanup_load_bufs_thread()
447         while (multifd->load_bufs_thread_running) {  in vfio_load_cleanup_load_bufs_thread()
448             multifd->load_bufs_thread_want_exit = true;  in vfio_load_cleanup_load_bufs_thread()
450             qemu_cond_signal(&multifd->load_bufs_buffer_ready_cond);  in vfio_load_cleanup_load_bufs_thread()
451             qemu_cond_wait(&multifd->load_bufs_thread_finished_cond,  in vfio_load_cleanup_load_bufs_thread()
452                            &multifd->load_bufs_mutex);  in vfio_load_cleanup_load_bufs_thread()
458 static void vfio_multifd_free(VFIOMultifd *multifd)  in vfio_multifd_free()  argument
460     vfio_load_cleanup_load_bufs_thread(multifd);  in vfio_multifd_free()
462     qemu_cond_destroy(&multifd->load_bufs_thread_finished_cond);  in vfio_multifd_free()
463     vfio_state_buffers_destroy(&multifd->load_bufs);  in vfio_multifd_free()
464     qemu_cond_destroy(&multifd->load_bufs_buffer_ready_cond);  in vfio_multifd_free()
465     qemu_mutex_destroy(&multifd->load_bufs_mutex);  in vfio_multifd_free()
467     g_free(multifd);  in vfio_multifd_free()
474     g_clear_pointer(&migration->multifd, vfio_multifd_free);  in vfio_multifd_cleanup()
518         assert(!migration->multifd);  in vfio_multifd_setup()
519         migration->multifd = vfio_multifd_new();  in vfio_multifd_setup()
670     VFIOMultifd *multifd = migration->multifd;  in vfio_multifd_switchover_start()  local
672     assert(multifd);  in vfio_multifd_switchover_start()
676     WITH_QEMU_LOCK_GUARD(&multifd->load_bufs_mutex) {  in vfio_multifd_switchover_start()
677         assert(!multifd->load_bufs_thread_running);  in vfio_multifd_switchover_start()
678         multifd->load_bufs_thread_running = true;  in vfio_multifd_switchover_start()