1 /* 2 * QEMU live migration 3 * 4 * Copyright IBM, Corp. 2008 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu-common.h" 17 #include "qemu/error-report.h" 18 #include "qemu/main-loop.h" 19 #include "migration/migration.h" 20 #include "migration/qemu-file.h" 21 #include "sysemu/sysemu.h" 22 #include "block/block.h" 23 #include "qapi/qmp/qerror.h" 24 #include "qemu/sockets.h" 25 #include "qemu/rcu.h" 26 #include "migration/block.h" 27 #include "qemu/thread.h" 28 #include "qmp-commands.h" 29 #include "trace.h" 30 #include "qapi/util.h" 31 #include "qapi-event.h" 32 #include "qom/cpu.h" 33 34 #define MAX_THROTTLE (32 << 20) /* Migration transfer speed throttling */ 35 36 /* Amount of time to allocate to each "chunk" of bandwidth-throttled 37 * data. */ 38 #define BUFFER_DELAY 100 39 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) 40 41 /* Default compression thread count */ 42 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8 43 /* Default decompression thread count, usually decompression is at 44 * least 4 times as fast as compression.*/ 45 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2 46 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */ 47 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1 48 /* Define default autoconverge cpu throttle migration parameters */ 49 #define DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL 20 50 #define DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT 10 51 52 /* Migration XBZRLE default cache size */ 53 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024) 54 55 static NotifierList migration_state_notifiers = 56 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); 57 58 static bool deferred_incoming; 59 60 /* When we add fault tolerance, we could have several 61 migrations at once. For now we don't need to add 62 dynamic creation of migration */ 63 64 /* For outgoing */ 65 MigrationState *migrate_get_current(void) 66 { 67 static MigrationState current_migration = { 68 .state = MIGRATION_STATUS_NONE, 69 .bandwidth_limit = MAX_THROTTLE, 70 .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE, 71 .mbps = -1, 72 .parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = 73 DEFAULT_MIGRATE_COMPRESS_LEVEL, 74 .parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 75 DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT, 76 .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 77 DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT, 78 .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] = 79 DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL, 80 .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] = 81 DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT, 82 }; 83 84 return ¤t_migration; 85 } 86 87 /* For incoming */ 88 static MigrationIncomingState *mis_current; 89 90 MigrationIncomingState *migration_incoming_get_current(void) 91 { 92 return mis_current; 93 } 94 95 MigrationIncomingState *migration_incoming_state_new(QEMUFile* f) 96 { 97 mis_current = g_new0(MigrationIncomingState, 1); 98 mis_current->file = f; 99 QLIST_INIT(&mis_current->loadvm_handlers); 100 101 return mis_current; 102 } 103 104 void migration_incoming_state_destroy(void) 105 { 106 loadvm_free_handlers(mis_current); 107 g_free(mis_current); 108 mis_current = NULL; 109 } 110 111 112 typedef struct { 113 bool optional; 114 uint32_t size; 115 uint8_t runstate[100]; 116 RunState state; 117 bool received; 118 } GlobalState; 119 120 static GlobalState global_state; 121 122 int global_state_store(void) 123 { 124 if (!runstate_store((char *)global_state.runstate, 125 sizeof(global_state.runstate))) { 126 error_report("runstate name too big: %s", global_state.runstate); 127 trace_migrate_state_too_big(); 128 return -EINVAL; 129 } 130 return 0; 131 } 132 133 void global_state_store_running(void) 134 { 135 const char *state = RunState_lookup[RUN_STATE_RUNNING]; 136 strncpy((char *)global_state.runstate, 137 state, sizeof(global_state.runstate)); 138 } 139 140 static bool global_state_received(void) 141 { 142 return global_state.received; 143 } 144 145 static RunState global_state_get_runstate(void) 146 { 147 return global_state.state; 148 } 149 150 void global_state_set_optional(void) 151 { 152 global_state.optional = true; 153 } 154 155 static bool global_state_needed(void *opaque) 156 { 157 GlobalState *s = opaque; 158 char *runstate = (char *)s->runstate; 159 160 /* If it is not optional, it is mandatory */ 161 162 if (s->optional == false) { 163 return true; 164 } 165 166 /* If state is running or paused, it is not needed */ 167 168 if (strcmp(runstate, "running") == 0 || 169 strcmp(runstate, "paused") == 0) { 170 return false; 171 } 172 173 /* for any other state it is needed */ 174 return true; 175 } 176 177 static int global_state_post_load(void *opaque, int version_id) 178 { 179 GlobalState *s = opaque; 180 Error *local_err = NULL; 181 int r; 182 char *runstate = (char *)s->runstate; 183 184 s->received = true; 185 trace_migrate_global_state_post_load(runstate); 186 187 r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE_MAX, 188 -1, &local_err); 189 190 if (r == -1) { 191 if (local_err) { 192 error_report_err(local_err); 193 } 194 return -EINVAL; 195 } 196 s->state = r; 197 198 return 0; 199 } 200 201 static void global_state_pre_save(void *opaque) 202 { 203 GlobalState *s = opaque; 204 205 trace_migrate_global_state_pre_save((char *)s->runstate); 206 s->size = strlen((char *)s->runstate) + 1; 207 } 208 209 static const VMStateDescription vmstate_globalstate = { 210 .name = "globalstate", 211 .version_id = 1, 212 .minimum_version_id = 1, 213 .post_load = global_state_post_load, 214 .pre_save = global_state_pre_save, 215 .needed = global_state_needed, 216 .fields = (VMStateField[]) { 217 VMSTATE_UINT32(size, GlobalState), 218 VMSTATE_BUFFER(runstate, GlobalState), 219 VMSTATE_END_OF_LIST() 220 }, 221 }; 222 223 void register_global_state(void) 224 { 225 /* We would use it independently that we receive it */ 226 strcpy((char *)&global_state.runstate, ""); 227 global_state.received = false; 228 vmstate_register(NULL, 0, &vmstate_globalstate, &global_state); 229 } 230 231 static void migrate_generate_event(int new_state) 232 { 233 if (migrate_use_events()) { 234 qapi_event_send_migration(new_state, &error_abort); 235 } 236 } 237 238 /* 239 * Called on -incoming with a defer: uri. 240 * The migration can be started later after any parameters have been 241 * changed. 242 */ 243 static void deferred_incoming_migration(Error **errp) 244 { 245 if (deferred_incoming) { 246 error_setg(errp, "Incoming migration already deferred"); 247 } 248 deferred_incoming = true; 249 } 250 251 void qemu_start_incoming_migration(const char *uri, Error **errp) 252 { 253 const char *p; 254 255 qapi_event_send_migration(MIGRATION_STATUS_SETUP, &error_abort); 256 if (!strcmp(uri, "defer")) { 257 deferred_incoming_migration(errp); 258 } else if (strstart(uri, "tcp:", &p)) { 259 tcp_start_incoming_migration(p, errp); 260 #ifdef CONFIG_RDMA 261 } else if (strstart(uri, "rdma:", &p)) { 262 rdma_start_incoming_migration(p, errp); 263 #endif 264 #if !defined(WIN32) 265 } else if (strstart(uri, "exec:", &p)) { 266 exec_start_incoming_migration(p, errp); 267 } else if (strstart(uri, "unix:", &p)) { 268 unix_start_incoming_migration(p, errp); 269 } else if (strstart(uri, "fd:", &p)) { 270 fd_start_incoming_migration(p, errp); 271 #endif 272 } else { 273 error_setg(errp, "unknown migration protocol: %s", uri); 274 } 275 } 276 277 static void process_incoming_migration_co(void *opaque) 278 { 279 QEMUFile *f = opaque; 280 Error *local_err = NULL; 281 int ret; 282 283 migration_incoming_state_new(f); 284 migrate_generate_event(MIGRATION_STATUS_ACTIVE); 285 ret = qemu_loadvm_state(f); 286 287 qemu_fclose(f); 288 free_xbzrle_decoded_buf(); 289 migration_incoming_state_destroy(); 290 291 if (ret < 0) { 292 migrate_generate_event(MIGRATION_STATUS_FAILED); 293 error_report("load of migration failed: %s", strerror(-ret)); 294 migrate_decompress_threads_join(); 295 exit(EXIT_FAILURE); 296 } 297 migrate_generate_event(MIGRATION_STATUS_COMPLETED); 298 qemu_announce_self(); 299 300 /* Make sure all file formats flush their mutable metadata */ 301 bdrv_invalidate_cache_all(&local_err); 302 if (local_err) { 303 error_report_err(local_err); 304 migrate_decompress_threads_join(); 305 exit(EXIT_FAILURE); 306 } 307 308 /* If global state section was not received or we are in running 309 state, we need to obey autostart. Any other state is set with 310 runstate_set. */ 311 312 if (!global_state_received() || 313 global_state_get_runstate() == RUN_STATE_RUNNING) { 314 if (autostart) { 315 vm_start(); 316 } else { 317 runstate_set(RUN_STATE_PAUSED); 318 } 319 } else { 320 runstate_set(global_state_get_runstate()); 321 } 322 migrate_decompress_threads_join(); 323 } 324 325 void process_incoming_migration(QEMUFile *f) 326 { 327 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co); 328 int fd = qemu_get_fd(f); 329 330 assert(fd != -1); 331 migrate_decompress_threads_create(); 332 qemu_set_nonblock(fd); 333 qemu_coroutine_enter(co, f); 334 } 335 336 /* amount of nanoseconds we are willing to wait for migration to be down. 337 * the choice of nanoseconds is because it is the maximum resolution that 338 * get_clock() can achieve. It is an internal measure. All user-visible 339 * units must be in seconds */ 340 static uint64_t max_downtime = 300000000; 341 342 uint64_t migrate_max_downtime(void) 343 { 344 return max_downtime; 345 } 346 347 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp) 348 { 349 MigrationCapabilityStatusList *head = NULL; 350 MigrationCapabilityStatusList *caps; 351 MigrationState *s = migrate_get_current(); 352 int i; 353 354 caps = NULL; /* silence compiler warning */ 355 for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) { 356 if (head == NULL) { 357 head = g_malloc0(sizeof(*caps)); 358 caps = head; 359 } else { 360 caps->next = g_malloc0(sizeof(*caps)); 361 caps = caps->next; 362 } 363 caps->value = 364 g_malloc(sizeof(*caps->value)); 365 caps->value->capability = i; 366 caps->value->state = s->enabled_capabilities[i]; 367 } 368 369 return head; 370 } 371 372 MigrationParameters *qmp_query_migrate_parameters(Error **errp) 373 { 374 MigrationParameters *params; 375 MigrationState *s = migrate_get_current(); 376 377 params = g_malloc0(sizeof(*params)); 378 params->compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 379 params->compress_threads = 380 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 381 params->decompress_threads = 382 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 383 params->x_cpu_throttle_initial = 384 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL]; 385 params->x_cpu_throttle_increment = 386 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT]; 387 388 return params; 389 } 390 391 static void get_xbzrle_cache_stats(MigrationInfo *info) 392 { 393 if (migrate_use_xbzrle()) { 394 info->has_xbzrle_cache = true; 395 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache)); 396 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size(); 397 info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred(); 398 info->xbzrle_cache->pages = xbzrle_mig_pages_transferred(); 399 info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss(); 400 info->xbzrle_cache->cache_miss_rate = xbzrle_mig_cache_miss_rate(); 401 info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow(); 402 } 403 } 404 405 MigrationInfo *qmp_query_migrate(Error **errp) 406 { 407 MigrationInfo *info = g_malloc0(sizeof(*info)); 408 MigrationState *s = migrate_get_current(); 409 410 switch (s->state) { 411 case MIGRATION_STATUS_NONE: 412 /* no migration has happened ever */ 413 break; 414 case MIGRATION_STATUS_SETUP: 415 info->has_status = true; 416 info->has_total_time = false; 417 break; 418 case MIGRATION_STATUS_ACTIVE: 419 case MIGRATION_STATUS_CANCELLING: 420 info->has_status = true; 421 info->has_total_time = true; 422 info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) 423 - s->total_time; 424 info->has_expected_downtime = true; 425 info->expected_downtime = s->expected_downtime; 426 info->has_setup_time = true; 427 info->setup_time = s->setup_time; 428 429 info->has_ram = true; 430 info->ram = g_malloc0(sizeof(*info->ram)); 431 info->ram->transferred = ram_bytes_transferred(); 432 info->ram->remaining = ram_bytes_remaining(); 433 info->ram->total = ram_bytes_total(); 434 info->ram->duplicate = dup_mig_pages_transferred(); 435 info->ram->skipped = skipped_mig_pages_transferred(); 436 info->ram->normal = norm_mig_pages_transferred(); 437 info->ram->normal_bytes = norm_mig_bytes_transferred(); 438 info->ram->dirty_pages_rate = s->dirty_pages_rate; 439 info->ram->mbps = s->mbps; 440 info->ram->dirty_sync_count = s->dirty_sync_count; 441 442 if (blk_mig_active()) { 443 info->has_disk = true; 444 info->disk = g_malloc0(sizeof(*info->disk)); 445 info->disk->transferred = blk_mig_bytes_transferred(); 446 info->disk->remaining = blk_mig_bytes_remaining(); 447 info->disk->total = blk_mig_bytes_total(); 448 } 449 450 if (cpu_throttle_active()) { 451 info->has_x_cpu_throttle_percentage = true; 452 info->x_cpu_throttle_percentage = cpu_throttle_get_percentage(); 453 } 454 455 get_xbzrle_cache_stats(info); 456 break; 457 case MIGRATION_STATUS_COMPLETED: 458 get_xbzrle_cache_stats(info); 459 460 info->has_status = true; 461 info->has_total_time = true; 462 info->total_time = s->total_time; 463 info->has_downtime = true; 464 info->downtime = s->downtime; 465 info->has_setup_time = true; 466 info->setup_time = s->setup_time; 467 468 info->has_ram = true; 469 info->ram = g_malloc0(sizeof(*info->ram)); 470 info->ram->transferred = ram_bytes_transferred(); 471 info->ram->remaining = 0; 472 info->ram->total = ram_bytes_total(); 473 info->ram->duplicate = dup_mig_pages_transferred(); 474 info->ram->skipped = skipped_mig_pages_transferred(); 475 info->ram->normal = norm_mig_pages_transferred(); 476 info->ram->normal_bytes = norm_mig_bytes_transferred(); 477 info->ram->mbps = s->mbps; 478 info->ram->dirty_sync_count = s->dirty_sync_count; 479 break; 480 case MIGRATION_STATUS_FAILED: 481 info->has_status = true; 482 break; 483 case MIGRATION_STATUS_CANCELLED: 484 info->has_status = true; 485 break; 486 } 487 info->status = s->state; 488 489 return info; 490 } 491 492 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, 493 Error **errp) 494 { 495 MigrationState *s = migrate_get_current(); 496 MigrationCapabilityStatusList *cap; 497 498 if (s->state == MIGRATION_STATUS_ACTIVE || 499 s->state == MIGRATION_STATUS_SETUP) { 500 error_setg(errp, QERR_MIGRATION_ACTIVE); 501 return; 502 } 503 504 for (cap = params; cap; cap = cap->next) { 505 s->enabled_capabilities[cap->value->capability] = cap->value->state; 506 } 507 } 508 509 void qmp_migrate_set_parameters(bool has_compress_level, 510 int64_t compress_level, 511 bool has_compress_threads, 512 int64_t compress_threads, 513 bool has_decompress_threads, 514 int64_t decompress_threads, 515 bool has_x_cpu_throttle_initial, 516 int64_t x_cpu_throttle_initial, 517 bool has_x_cpu_throttle_increment, 518 int64_t x_cpu_throttle_increment, Error **errp) 519 { 520 MigrationState *s = migrate_get_current(); 521 522 if (has_compress_level && (compress_level < 0 || compress_level > 9)) { 523 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level", 524 "is invalid, it should be in the range of 0 to 9"); 525 return; 526 } 527 if (has_compress_threads && 528 (compress_threads < 1 || compress_threads > 255)) { 529 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 530 "compress_threads", 531 "is invalid, it should be in the range of 1 to 255"); 532 return; 533 } 534 if (has_decompress_threads && 535 (decompress_threads < 1 || decompress_threads > 255)) { 536 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 537 "decompress_threads", 538 "is invalid, it should be in the range of 1 to 255"); 539 return; 540 } 541 if (has_x_cpu_throttle_initial && 542 (x_cpu_throttle_initial < 1 || x_cpu_throttle_initial > 99)) { 543 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 544 "x_cpu_throttle_initial", 545 "an integer in the range of 1 to 99"); 546 } 547 if (has_x_cpu_throttle_increment && 548 (x_cpu_throttle_increment < 1 || x_cpu_throttle_increment > 99)) { 549 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 550 "x_cpu_throttle_increment", 551 "an integer in the range of 1 to 99"); 552 } 553 554 if (has_compress_level) { 555 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 556 } 557 if (has_compress_threads) { 558 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = compress_threads; 559 } 560 if (has_decompress_threads) { 561 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 562 decompress_threads; 563 } 564 if (has_x_cpu_throttle_initial) { 565 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] = 566 x_cpu_throttle_initial; 567 } 568 569 if (has_x_cpu_throttle_increment) { 570 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] = 571 x_cpu_throttle_increment; 572 } 573 } 574 575 /* shared migration helpers */ 576 577 static void migrate_set_state(MigrationState *s, int old_state, int new_state) 578 { 579 if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) { 580 trace_migrate_set_state(new_state); 581 migrate_generate_event(new_state); 582 } 583 } 584 585 static void migrate_fd_cleanup(void *opaque) 586 { 587 MigrationState *s = opaque; 588 589 qemu_bh_delete(s->cleanup_bh); 590 s->cleanup_bh = NULL; 591 592 if (s->file) { 593 trace_migrate_fd_cleanup(); 594 qemu_mutex_unlock_iothread(); 595 qemu_thread_join(&s->thread); 596 qemu_mutex_lock_iothread(); 597 598 migrate_compress_threads_join(); 599 qemu_fclose(s->file); 600 s->file = NULL; 601 } 602 603 assert(s->state != MIGRATION_STATUS_ACTIVE); 604 605 if (s->state != MIGRATION_STATUS_COMPLETED) { 606 qemu_savevm_state_cancel(); 607 if (s->state == MIGRATION_STATUS_CANCELLING) { 608 migrate_set_state(s, MIGRATION_STATUS_CANCELLING, 609 MIGRATION_STATUS_CANCELLED); 610 } 611 } 612 613 notifier_list_notify(&migration_state_notifiers, s); 614 } 615 616 void migrate_fd_error(MigrationState *s) 617 { 618 trace_migrate_fd_error(); 619 assert(s->file == NULL); 620 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 621 notifier_list_notify(&migration_state_notifiers, s); 622 } 623 624 static void migrate_fd_cancel(MigrationState *s) 625 { 626 int old_state ; 627 QEMUFile *f = migrate_get_current()->file; 628 trace_migrate_fd_cancel(); 629 630 do { 631 old_state = s->state; 632 if (old_state != MIGRATION_STATUS_SETUP && 633 old_state != MIGRATION_STATUS_ACTIVE) { 634 break; 635 } 636 migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING); 637 } while (s->state != MIGRATION_STATUS_CANCELLING); 638 639 /* 640 * If we're unlucky the migration code might be stuck somewhere in a 641 * send/write while the network has failed and is waiting to timeout; 642 * if we've got shutdown(2) available then we can force it to quit. 643 * The outgoing qemu file gets closed in migrate_fd_cleanup that is 644 * called in a bh, so there is no race against this cancel. 645 */ 646 if (s->state == MIGRATION_STATUS_CANCELLING && f) { 647 qemu_file_shutdown(f); 648 } 649 } 650 651 void add_migration_state_change_notifier(Notifier *notify) 652 { 653 notifier_list_add(&migration_state_notifiers, notify); 654 } 655 656 void remove_migration_state_change_notifier(Notifier *notify) 657 { 658 notifier_remove(notify); 659 } 660 661 bool migration_in_setup(MigrationState *s) 662 { 663 return s->state == MIGRATION_STATUS_SETUP; 664 } 665 666 bool migration_has_finished(MigrationState *s) 667 { 668 return s->state == MIGRATION_STATUS_COMPLETED; 669 } 670 671 bool migration_has_failed(MigrationState *s) 672 { 673 return (s->state == MIGRATION_STATUS_CANCELLED || 674 s->state == MIGRATION_STATUS_FAILED); 675 } 676 677 static MigrationState *migrate_init(const MigrationParams *params) 678 { 679 MigrationState *s = migrate_get_current(); 680 int64_t bandwidth_limit = s->bandwidth_limit; 681 bool enabled_capabilities[MIGRATION_CAPABILITY_MAX]; 682 int64_t xbzrle_cache_size = s->xbzrle_cache_size; 683 int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 684 int compress_thread_count = 685 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 686 int decompress_thread_count = 687 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 688 int x_cpu_throttle_initial = 689 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL]; 690 int x_cpu_throttle_increment = 691 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT]; 692 693 memcpy(enabled_capabilities, s->enabled_capabilities, 694 sizeof(enabled_capabilities)); 695 696 memset(s, 0, sizeof(*s)); 697 s->params = *params; 698 memcpy(s->enabled_capabilities, enabled_capabilities, 699 sizeof(enabled_capabilities)); 700 s->xbzrle_cache_size = xbzrle_cache_size; 701 702 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 703 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 704 compress_thread_count; 705 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 706 decompress_thread_count; 707 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] = 708 x_cpu_throttle_initial; 709 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] = 710 x_cpu_throttle_increment; 711 s->bandwidth_limit = bandwidth_limit; 712 migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); 713 714 s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 715 return s; 716 } 717 718 static GSList *migration_blockers; 719 720 void migrate_add_blocker(Error *reason) 721 { 722 migration_blockers = g_slist_prepend(migration_blockers, reason); 723 } 724 725 void migrate_del_blocker(Error *reason) 726 { 727 migration_blockers = g_slist_remove(migration_blockers, reason); 728 } 729 730 void qmp_migrate_incoming(const char *uri, Error **errp) 731 { 732 Error *local_err = NULL; 733 static bool once = true; 734 735 if (!deferred_incoming) { 736 error_setg(errp, "For use with '-incoming defer'"); 737 return; 738 } 739 if (!once) { 740 error_setg(errp, "The incoming migration has already been started"); 741 } 742 743 qemu_start_incoming_migration(uri, &local_err); 744 745 if (local_err) { 746 error_propagate(errp, local_err); 747 return; 748 } 749 750 once = false; 751 } 752 753 void qmp_migrate(const char *uri, bool has_blk, bool blk, 754 bool has_inc, bool inc, bool has_detach, bool detach, 755 Error **errp) 756 { 757 Error *local_err = NULL; 758 MigrationState *s = migrate_get_current(); 759 MigrationParams params; 760 const char *p; 761 762 params.blk = has_blk && blk; 763 params.shared = has_inc && inc; 764 765 if (s->state == MIGRATION_STATUS_ACTIVE || 766 s->state == MIGRATION_STATUS_SETUP || 767 s->state == MIGRATION_STATUS_CANCELLING) { 768 error_setg(errp, QERR_MIGRATION_ACTIVE); 769 return; 770 } 771 if (runstate_check(RUN_STATE_INMIGRATE)) { 772 error_setg(errp, "Guest is waiting for an incoming migration"); 773 return; 774 } 775 776 if (qemu_savevm_state_blocked(errp)) { 777 return; 778 } 779 780 if (migration_blockers) { 781 *errp = error_copy(migration_blockers->data); 782 return; 783 } 784 785 /* We are starting a new migration, so we want to start in a clean 786 state. This change is only needed if previous migration 787 failed/was cancelled. We don't use migrate_set_state() because 788 we are setting the initial state, not changing it. */ 789 s->state = MIGRATION_STATUS_NONE; 790 791 s = migrate_init(¶ms); 792 793 if (strstart(uri, "tcp:", &p)) { 794 tcp_start_outgoing_migration(s, p, &local_err); 795 #ifdef CONFIG_RDMA 796 } else if (strstart(uri, "rdma:", &p)) { 797 rdma_start_outgoing_migration(s, p, &local_err); 798 #endif 799 #if !defined(WIN32) 800 } else if (strstart(uri, "exec:", &p)) { 801 exec_start_outgoing_migration(s, p, &local_err); 802 } else if (strstart(uri, "unix:", &p)) { 803 unix_start_outgoing_migration(s, p, &local_err); 804 } else if (strstart(uri, "fd:", &p)) { 805 fd_start_outgoing_migration(s, p, &local_err); 806 #endif 807 } else { 808 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri", 809 "a valid migration protocol"); 810 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 811 return; 812 } 813 814 if (local_err) { 815 migrate_fd_error(s); 816 error_propagate(errp, local_err); 817 return; 818 } 819 } 820 821 void qmp_migrate_cancel(Error **errp) 822 { 823 migrate_fd_cancel(migrate_get_current()); 824 } 825 826 void qmp_migrate_set_cache_size(int64_t value, Error **errp) 827 { 828 MigrationState *s = migrate_get_current(); 829 int64_t new_size; 830 831 /* Check for truncation */ 832 if (value != (size_t)value) { 833 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 834 "exceeding address space"); 835 return; 836 } 837 838 /* Cache should not be larger than guest ram size */ 839 if (value > ram_bytes_total()) { 840 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 841 "exceeds guest ram size "); 842 return; 843 } 844 845 new_size = xbzrle_cache_resize(value); 846 if (new_size < 0) { 847 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 848 "is smaller than page size"); 849 return; 850 } 851 852 s->xbzrle_cache_size = new_size; 853 } 854 855 int64_t qmp_query_migrate_cache_size(Error **errp) 856 { 857 return migrate_xbzrle_cache_size(); 858 } 859 860 void qmp_migrate_set_speed(int64_t value, Error **errp) 861 { 862 MigrationState *s; 863 864 if (value < 0) { 865 value = 0; 866 } 867 if (value > SIZE_MAX) { 868 value = SIZE_MAX; 869 } 870 871 s = migrate_get_current(); 872 s->bandwidth_limit = value; 873 if (s->file) { 874 qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO); 875 } 876 } 877 878 void qmp_migrate_set_downtime(double value, Error **errp) 879 { 880 value *= 1e9; 881 value = MAX(0, MIN(UINT64_MAX, value)); 882 max_downtime = (uint64_t)value; 883 } 884 885 bool migrate_auto_converge(void) 886 { 887 MigrationState *s; 888 889 s = migrate_get_current(); 890 891 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE]; 892 } 893 894 bool migrate_zero_blocks(void) 895 { 896 MigrationState *s; 897 898 s = migrate_get_current(); 899 900 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS]; 901 } 902 903 bool migrate_use_compression(void) 904 { 905 MigrationState *s; 906 907 s = migrate_get_current(); 908 909 return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS]; 910 } 911 912 int migrate_compress_level(void) 913 { 914 MigrationState *s; 915 916 s = migrate_get_current(); 917 918 return s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 919 } 920 921 int migrate_compress_threads(void) 922 { 923 MigrationState *s; 924 925 s = migrate_get_current(); 926 927 return s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 928 } 929 930 int migrate_decompress_threads(void) 931 { 932 MigrationState *s; 933 934 s = migrate_get_current(); 935 936 return s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 937 } 938 939 bool migrate_use_events(void) 940 { 941 MigrationState *s; 942 943 s = migrate_get_current(); 944 945 return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS]; 946 } 947 948 int migrate_use_xbzrle(void) 949 { 950 MigrationState *s; 951 952 s = migrate_get_current(); 953 954 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE]; 955 } 956 957 int64_t migrate_xbzrle_cache_size(void) 958 { 959 MigrationState *s; 960 961 s = migrate_get_current(); 962 963 return s->xbzrle_cache_size; 964 } 965 966 /** 967 * migration_completion: Used by migration_thread when there's not much left. 968 * The caller 'breaks' the loop when this returns. 969 * 970 * @s: Current migration state 971 * @*old_vm_running: Pointer to old_vm_running flag 972 * @*start_time: Pointer to time to update 973 */ 974 static void migration_completion(MigrationState *s, bool *old_vm_running, 975 int64_t *start_time) 976 { 977 int ret; 978 979 qemu_mutex_lock_iothread(); 980 *start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 981 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); 982 *old_vm_running = runstate_is_running(); 983 984 ret = global_state_store(); 985 if (!ret) { 986 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); 987 if (ret >= 0) { 988 qemu_file_set_rate_limit(s->file, INT64_MAX); 989 qemu_savevm_state_complete(s->file); 990 } 991 } 992 qemu_mutex_unlock_iothread(); 993 994 if (ret < 0) { 995 goto fail; 996 } 997 998 if (qemu_file_get_error(s->file)) { 999 trace_migration_completion_file_err(); 1000 goto fail; 1001 } 1002 1003 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_COMPLETED); 1004 return; 1005 1006 fail: 1007 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_FAILED); 1008 } 1009 1010 /* migration thread support */ 1011 1012 static void *migration_thread(void *opaque) 1013 { 1014 MigrationState *s = opaque; 1015 int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 1016 int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); 1017 int64_t initial_bytes = 0; 1018 int64_t max_size = 0; 1019 int64_t start_time = initial_time; 1020 bool old_vm_running = false; 1021 1022 rcu_register_thread(); 1023 1024 qemu_savevm_state_header(s->file); 1025 qemu_savevm_state_begin(s->file, &s->params); 1026 1027 s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; 1028 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); 1029 1030 while (s->state == MIGRATION_STATUS_ACTIVE) { 1031 int64_t current_time; 1032 uint64_t pending_size; 1033 1034 if (!qemu_file_rate_limit(s->file)) { 1035 pending_size = qemu_savevm_state_pending(s->file, max_size); 1036 trace_migrate_pending(pending_size, max_size); 1037 if (pending_size && pending_size >= max_size) { 1038 qemu_savevm_state_iterate(s->file); 1039 } else { 1040 trace_migration_thread_low_pending(pending_size); 1041 migration_completion(s, &old_vm_running, &start_time); 1042 break; 1043 } 1044 } 1045 1046 if (qemu_file_get_error(s->file)) { 1047 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 1048 MIGRATION_STATUS_FAILED); 1049 break; 1050 } 1051 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 1052 if (current_time >= initial_time + BUFFER_DELAY) { 1053 uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes; 1054 uint64_t time_spent = current_time - initial_time; 1055 double bandwidth = transferred_bytes / time_spent; 1056 max_size = bandwidth * migrate_max_downtime() / 1000000; 1057 1058 s->mbps = time_spent ? (((double) transferred_bytes * 8.0) / 1059 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1; 1060 1061 trace_migrate_transferred(transferred_bytes, time_spent, 1062 bandwidth, max_size); 1063 /* if we haven't sent anything, we don't want to recalculate 1064 10000 is a small enough number for our purposes */ 1065 if (s->dirty_bytes_rate && transferred_bytes > 10000) { 1066 s->expected_downtime = s->dirty_bytes_rate / bandwidth; 1067 } 1068 1069 qemu_file_reset_rate_limit(s->file); 1070 initial_time = current_time; 1071 initial_bytes = qemu_ftell(s->file); 1072 } 1073 if (qemu_file_rate_limit(s->file)) { 1074 /* usleep expects microseconds */ 1075 g_usleep((initial_time + BUFFER_DELAY - current_time)*1000); 1076 } 1077 } 1078 1079 /* If we enabled cpu throttling for auto-converge, turn it off. */ 1080 cpu_throttle_stop(); 1081 1082 qemu_mutex_lock_iothread(); 1083 if (s->state == MIGRATION_STATUS_COMPLETED) { 1084 int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 1085 uint64_t transferred_bytes = qemu_ftell(s->file); 1086 s->total_time = end_time - s->total_time; 1087 s->downtime = end_time - start_time; 1088 if (s->total_time) { 1089 s->mbps = (((double) transferred_bytes * 8.0) / 1090 ((double) s->total_time)) / 1000; 1091 } 1092 runstate_set(RUN_STATE_POSTMIGRATE); 1093 } else { 1094 if (old_vm_running) { 1095 vm_start(); 1096 } 1097 } 1098 qemu_bh_schedule(s->cleanup_bh); 1099 qemu_mutex_unlock_iothread(); 1100 1101 rcu_unregister_thread(); 1102 return NULL; 1103 } 1104 1105 void migrate_fd_connect(MigrationState *s) 1106 { 1107 /* This is a best 1st approximation. ns to ms */ 1108 s->expected_downtime = max_downtime/1000000; 1109 s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s); 1110 1111 qemu_file_set_rate_limit(s->file, 1112 s->bandwidth_limit / XFER_LIMIT_RATIO); 1113 1114 /* Notify before starting migration thread */ 1115 notifier_list_notify(&migration_state_notifiers, s); 1116 1117 migrate_compress_threads_create(); 1118 qemu_thread_create(&s->thread, "migration", migration_thread, s, 1119 QEMU_THREAD_JOINABLE); 1120 } 1121