1 /* 2 * Migration support for VFIO devices 3 * 4 * Copyright NVIDIA, Inc. 2020 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2. See 7 * the COPYING file in the top-level directory. 8 */ 9 10 #include "qemu/osdep.h" 11 #include "qemu/main-loop.h" 12 #include "qemu/cutils.h" 13 #include "qemu/units.h" 14 #include "qemu/error-report.h" 15 #include <linux/vfio.h> 16 #include <sys/ioctl.h> 17 18 #include "system/runstate.h" 19 #include "hw/vfio/vfio-common.h" 20 #include "migration/misc.h" 21 #include "migration/savevm.h" 22 #include "migration/vmstate.h" 23 #include "migration/qemu-file.h" 24 #include "migration/register.h" 25 #include "migration/blocker.h" 26 #include "migration-multifd.h" 27 #include "qapi/error.h" 28 #include "qapi/qapi-events-vfio.h" 29 #include "exec/ramlist.h" 30 #include "exec/ram_addr.h" 31 #include "pci.h" 32 #include "trace.h" 33 #include "hw/hw.h" 34 35 /* 36 * This is an arbitrary size based on migration of mlx5 devices, where typically 37 * total device migration size is on the order of 100s of MB. Testing with 38 * larger values, e.g. 128MB and 1GB, did not show a performance improvement. 39 */ 40 #define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1 * MiB) 41 42 static unsigned long bytes_transferred; 43 44 static const char *mig_state_to_str(enum vfio_device_mig_state state) 45 { 46 switch (state) { 47 case VFIO_DEVICE_STATE_ERROR: 48 return "ERROR"; 49 case VFIO_DEVICE_STATE_STOP: 50 return "STOP"; 51 case VFIO_DEVICE_STATE_RUNNING: 52 return "RUNNING"; 53 case VFIO_DEVICE_STATE_STOP_COPY: 54 return "STOP_COPY"; 55 case VFIO_DEVICE_STATE_RESUMING: 56 return "RESUMING"; 57 case VFIO_DEVICE_STATE_RUNNING_P2P: 58 return "RUNNING_P2P"; 59 case VFIO_DEVICE_STATE_PRE_COPY: 60 return "PRE_COPY"; 61 case VFIO_DEVICE_STATE_PRE_COPY_P2P: 62 return "PRE_COPY_P2P"; 63 default: 64 return "UNKNOWN STATE"; 65 } 66 } 67 68 static QapiVfioMigrationState 69 mig_state_to_qapi_state(enum vfio_device_mig_state state) 70 { 71 switch (state) { 72 case VFIO_DEVICE_STATE_STOP: 73 return QAPI_VFIO_MIGRATION_STATE_STOP; 74 case VFIO_DEVICE_STATE_RUNNING: 75 return QAPI_VFIO_MIGRATION_STATE_RUNNING; 76 case VFIO_DEVICE_STATE_STOP_COPY: 77 return QAPI_VFIO_MIGRATION_STATE_STOP_COPY; 78 case VFIO_DEVICE_STATE_RESUMING: 79 return QAPI_VFIO_MIGRATION_STATE_RESUMING; 80 case VFIO_DEVICE_STATE_RUNNING_P2P: 81 return QAPI_VFIO_MIGRATION_STATE_RUNNING_P2P; 82 case VFIO_DEVICE_STATE_PRE_COPY: 83 return QAPI_VFIO_MIGRATION_STATE_PRE_COPY; 84 case VFIO_DEVICE_STATE_PRE_COPY_P2P: 85 return QAPI_VFIO_MIGRATION_STATE_PRE_COPY_P2P; 86 default: 87 g_assert_not_reached(); 88 } 89 } 90 91 static void vfio_migration_send_event(VFIODevice *vbasedev) 92 { 93 VFIOMigration *migration = vbasedev->migration; 94 DeviceState *dev = vbasedev->dev; 95 g_autofree char *qom_path = NULL; 96 Object *obj; 97 98 if (!vbasedev->migration_events) { 99 return; 100 } 101 102 g_assert(vbasedev->ops->vfio_get_object); 103 obj = vbasedev->ops->vfio_get_object(vbasedev); 104 g_assert(obj); 105 qom_path = object_get_canonical_path(obj); 106 107 qapi_event_send_vfio_migration( 108 dev->id, qom_path, mig_state_to_qapi_state(migration->device_state)); 109 } 110 111 static void vfio_migration_set_device_state(VFIODevice *vbasedev, 112 enum vfio_device_mig_state state) 113 { 114 VFIOMigration *migration = vbasedev->migration; 115 116 trace_vfio_migration_set_device_state(vbasedev->name, 117 mig_state_to_str(state)); 118 119 migration->device_state = state; 120 vfio_migration_send_event(vbasedev); 121 } 122 123 int vfio_migration_set_state(VFIODevice *vbasedev, 124 enum vfio_device_mig_state new_state, 125 enum vfio_device_mig_state recover_state, 126 Error **errp) 127 { 128 VFIOMigration *migration = vbasedev->migration; 129 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 130 sizeof(struct vfio_device_feature_mig_state), 131 sizeof(uint64_t))] = {}; 132 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 133 struct vfio_device_feature_mig_state *mig_state = 134 (struct vfio_device_feature_mig_state *)feature->data; 135 int ret; 136 g_autofree char *error_prefix = 137 g_strdup_printf("%s: Failed setting device state to %s.", 138 vbasedev->name, mig_state_to_str(new_state)); 139 140 trace_vfio_migration_set_state(vbasedev->name, mig_state_to_str(new_state), 141 mig_state_to_str(recover_state)); 142 143 if (new_state == migration->device_state) { 144 return 0; 145 } 146 147 feature->argsz = sizeof(buf); 148 feature->flags = 149 VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE; 150 mig_state->device_state = new_state; 151 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 152 /* Try to set the device in some good state */ 153 ret = -errno; 154 155 if (recover_state == VFIO_DEVICE_STATE_ERROR) { 156 error_setg_errno(errp, errno, 157 "%s Recover state is ERROR. Resetting device", 158 error_prefix); 159 160 goto reset_device; 161 } 162 163 error_setg_errno(errp, errno, 164 "%s Setting device in recover state %s", 165 error_prefix, mig_state_to_str(recover_state)); 166 167 mig_state->device_state = recover_state; 168 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 169 ret = -errno; 170 /* 171 * If setting the device in recover state fails, report 172 * the error here and propagate the first error. 173 */ 174 error_report( 175 "%s: Failed setting device in recover state, err: %s. Resetting device", 176 vbasedev->name, strerror(errno)); 177 178 goto reset_device; 179 } 180 181 vfio_migration_set_device_state(vbasedev, recover_state); 182 183 return ret; 184 } 185 186 vfio_migration_set_device_state(vbasedev, new_state); 187 if (mig_state->data_fd != -1) { 188 if (migration->data_fd != -1) { 189 /* 190 * This can happen if the device is asynchronously reset and 191 * terminates a data transfer. 192 */ 193 error_setg(errp, "%s: data_fd out of sync", vbasedev->name); 194 close(mig_state->data_fd); 195 196 return -EBADF; 197 } 198 199 migration->data_fd = mig_state->data_fd; 200 } 201 202 return 0; 203 204 reset_device: 205 if (ioctl(vbasedev->fd, VFIO_DEVICE_RESET)) { 206 hw_error("%s: Failed resetting device, err: %s", vbasedev->name, 207 strerror(errno)); 208 } 209 210 vfio_migration_set_device_state(vbasedev, VFIO_DEVICE_STATE_RUNNING); 211 212 return ret; 213 } 214 215 /* 216 * Some device state transitions require resetting the device if they fail. 217 * This function sets the device in new_state and resets the device if that 218 * fails. Reset is done by using ERROR as the recover state. 219 */ 220 static int 221 vfio_migration_set_state_or_reset(VFIODevice *vbasedev, 222 enum vfio_device_mig_state new_state, 223 Error **errp) 224 { 225 return vfio_migration_set_state(vbasedev, new_state, 226 VFIO_DEVICE_STATE_ERROR, errp); 227 } 228 229 static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, 230 uint64_t data_size) 231 { 232 VFIOMigration *migration = vbasedev->migration; 233 int ret; 234 235 ret = qemu_file_get_to_fd(f, migration->data_fd, data_size); 236 trace_vfio_load_state_device_data(vbasedev->name, data_size, ret); 237 238 return ret; 239 } 240 241 int vfio_save_device_config_state(QEMUFile *f, void *opaque, Error **errp) 242 { 243 VFIODevice *vbasedev = opaque; 244 int ret; 245 246 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE); 247 248 if (vbasedev->ops && vbasedev->ops->vfio_save_config) { 249 ret = vbasedev->ops->vfio_save_config(vbasedev, f, errp); 250 if (ret) { 251 return ret; 252 } 253 } 254 255 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); 256 257 trace_vfio_save_device_config_state(vbasedev->name); 258 259 ret = qemu_file_get_error(f); 260 if (ret < 0) { 261 error_setg_errno(errp, -ret, "Failed to save state"); 262 } 263 return ret; 264 } 265 266 int vfio_load_device_config_state(QEMUFile *f, void *opaque) 267 { 268 VFIODevice *vbasedev = opaque; 269 uint64_t data; 270 271 trace_vfio_load_device_config_state_start(vbasedev->name); 272 273 if (vbasedev->ops && vbasedev->ops->vfio_load_config) { 274 int ret; 275 276 ret = vbasedev->ops->vfio_load_config(vbasedev, f); 277 if (ret) { 278 error_report("%s: Failed to load device config space", 279 vbasedev->name); 280 return ret; 281 } 282 } 283 284 data = qemu_get_be64(f); 285 if (data != VFIO_MIG_FLAG_END_OF_STATE) { 286 error_report("%s: Failed loading device config space, " 287 "end flag incorrect 0x%"PRIx64, vbasedev->name, data); 288 return -EINVAL; 289 } 290 291 trace_vfio_load_device_config_state_end(vbasedev->name); 292 return qemu_file_get_error(f); 293 } 294 295 static void vfio_migration_cleanup(VFIODevice *vbasedev) 296 { 297 VFIOMigration *migration = vbasedev->migration; 298 299 close(migration->data_fd); 300 migration->data_fd = -1; 301 } 302 303 static int vfio_query_stop_copy_size(VFIODevice *vbasedev, 304 uint64_t *stop_copy_size) 305 { 306 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 307 sizeof(struct vfio_device_feature_mig_data_size), 308 sizeof(uint64_t))] = {}; 309 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 310 struct vfio_device_feature_mig_data_size *mig_data_size = 311 (struct vfio_device_feature_mig_data_size *)feature->data; 312 313 feature->argsz = sizeof(buf); 314 feature->flags = 315 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIG_DATA_SIZE; 316 317 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 318 return -errno; 319 } 320 321 *stop_copy_size = mig_data_size->stop_copy_length; 322 323 return 0; 324 } 325 326 static int vfio_query_precopy_size(VFIOMigration *migration) 327 { 328 struct vfio_precopy_info precopy = { 329 .argsz = sizeof(precopy), 330 }; 331 332 migration->precopy_init_size = 0; 333 migration->precopy_dirty_size = 0; 334 335 if (ioctl(migration->data_fd, VFIO_MIG_GET_PRECOPY_INFO, &precopy)) { 336 return -errno; 337 } 338 339 migration->precopy_init_size = precopy.initial_bytes; 340 migration->precopy_dirty_size = precopy.dirty_bytes; 341 342 return 0; 343 } 344 345 /* Returns the size of saved data on success and -errno on error */ 346 static ssize_t vfio_save_block(QEMUFile *f, VFIOMigration *migration) 347 { 348 ssize_t data_size; 349 350 data_size = read(migration->data_fd, migration->data_buffer, 351 migration->data_buffer_size); 352 if (data_size < 0) { 353 /* 354 * Pre-copy emptied all the device state for now. For more information, 355 * please refer to the Linux kernel VFIO uAPI. 356 */ 357 if (errno == ENOMSG) { 358 if (!migration->event_precopy_empty_hit) { 359 trace_vfio_save_block_precopy_empty_hit(migration->vbasedev->name); 360 migration->event_precopy_empty_hit = true; 361 } 362 return 0; 363 } 364 365 return -errno; 366 } 367 if (data_size == 0) { 368 return 0; 369 } 370 371 /* Non-empty read: re-arm the trace event */ 372 migration->event_precopy_empty_hit = false; 373 374 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE); 375 qemu_put_be64(f, data_size); 376 qemu_put_buffer(f, migration->data_buffer, data_size); 377 vfio_mig_add_bytes_transferred(data_size); 378 379 trace_vfio_save_block(migration->vbasedev->name, data_size); 380 381 return qemu_file_get_error(f) ?: data_size; 382 } 383 384 static void vfio_update_estimated_pending_data(VFIOMigration *migration, 385 uint64_t data_size) 386 { 387 if (!data_size) { 388 /* 389 * Pre-copy emptied all the device state for now, update estimated sizes 390 * accordingly. 391 */ 392 migration->precopy_init_size = 0; 393 migration->precopy_dirty_size = 0; 394 395 return; 396 } 397 398 if (migration->precopy_init_size) { 399 uint64_t init_size = MIN(migration->precopy_init_size, data_size); 400 401 migration->precopy_init_size -= init_size; 402 data_size -= init_size; 403 } 404 405 migration->precopy_dirty_size -= MIN(migration->precopy_dirty_size, 406 data_size); 407 } 408 409 static bool vfio_precopy_supported(VFIODevice *vbasedev) 410 { 411 VFIOMigration *migration = vbasedev->migration; 412 413 return migration->mig_flags & VFIO_MIGRATION_PRE_COPY; 414 } 415 416 /* ---------------------------------------------------------------------- */ 417 418 static int vfio_save_prepare(void *opaque, Error **errp) 419 { 420 VFIODevice *vbasedev = opaque; 421 422 /* 423 * Snapshot doesn't use postcopy nor background snapshot, so allow snapshot 424 * even if they are on. 425 */ 426 if (runstate_check(RUN_STATE_SAVE_VM)) { 427 return 0; 428 } 429 430 if (migrate_postcopy_ram()) { 431 error_setg( 432 errp, "%s: VFIO migration is not supported with postcopy migration", 433 vbasedev->name); 434 return -EOPNOTSUPP; 435 } 436 437 if (migrate_background_snapshot()) { 438 error_setg( 439 errp, 440 "%s: VFIO migration is not supported with background snapshot", 441 vbasedev->name); 442 return -EOPNOTSUPP; 443 } 444 445 return 0; 446 } 447 448 static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp) 449 { 450 VFIODevice *vbasedev = opaque; 451 VFIOMigration *migration = vbasedev->migration; 452 uint64_t stop_copy_size = VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE; 453 int ret; 454 455 if (!vfio_multifd_setup(vbasedev, false, errp)) { 456 return -EINVAL; 457 } 458 459 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE); 460 461 vfio_query_stop_copy_size(vbasedev, &stop_copy_size); 462 migration->data_buffer_size = MIN(VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE, 463 stop_copy_size); 464 migration->data_buffer = g_try_malloc0(migration->data_buffer_size); 465 if (!migration->data_buffer) { 466 error_setg(errp, "%s: Failed to allocate migration data buffer", 467 vbasedev->name); 468 return -ENOMEM; 469 } 470 471 migration->event_save_iterate_started = false; 472 migration->event_precopy_empty_hit = false; 473 474 if (vfio_precopy_supported(vbasedev)) { 475 switch (migration->device_state) { 476 case VFIO_DEVICE_STATE_RUNNING: 477 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_PRE_COPY, 478 VFIO_DEVICE_STATE_RUNNING, errp); 479 if (ret) { 480 return ret; 481 } 482 483 vfio_query_precopy_size(migration); 484 485 break; 486 case VFIO_DEVICE_STATE_STOP: 487 /* vfio_save_complete_precopy() will go to STOP_COPY */ 488 break; 489 default: 490 error_setg(errp, "%s: Invalid device state %d", vbasedev->name, 491 migration->device_state); 492 return -EINVAL; 493 } 494 } 495 496 trace_vfio_save_setup(vbasedev->name, migration->data_buffer_size); 497 498 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); 499 500 ret = qemu_file_get_error(f); 501 if (ret < 0) { 502 error_setg_errno(errp, -ret, "%s: save setup failed", vbasedev->name); 503 } 504 505 return ret; 506 } 507 508 static void vfio_save_cleanup(void *opaque) 509 { 510 VFIODevice *vbasedev = opaque; 511 VFIOMigration *migration = vbasedev->migration; 512 Error *local_err = NULL; 513 int ret; 514 515 /* Currently a NOP, done for symmetry with load_cleanup() */ 516 vfio_multifd_cleanup(vbasedev); 517 518 /* 519 * Changing device state from STOP_COPY to STOP can take time. Do it here, 520 * after migration has completed, so it won't increase downtime. 521 */ 522 if (migration->device_state == VFIO_DEVICE_STATE_STOP_COPY) { 523 ret = vfio_migration_set_state_or_reset(vbasedev, 524 VFIO_DEVICE_STATE_STOP, 525 &local_err); 526 if (ret) { 527 error_report_err(local_err); 528 } 529 } 530 531 g_free(migration->data_buffer); 532 migration->data_buffer = NULL; 533 migration->precopy_init_size = 0; 534 migration->precopy_dirty_size = 0; 535 migration->initial_data_sent = false; 536 vfio_migration_cleanup(vbasedev); 537 trace_vfio_save_cleanup(vbasedev->name); 538 } 539 540 static void vfio_state_pending_estimate(void *opaque, uint64_t *must_precopy, 541 uint64_t *can_postcopy) 542 { 543 VFIODevice *vbasedev = opaque; 544 VFIOMigration *migration = vbasedev->migration; 545 546 if (!vfio_device_state_is_precopy(vbasedev)) { 547 return; 548 } 549 550 *must_precopy += 551 migration->precopy_init_size + migration->precopy_dirty_size; 552 553 trace_vfio_state_pending_estimate(vbasedev->name, *must_precopy, 554 *can_postcopy, 555 migration->precopy_init_size, 556 migration->precopy_dirty_size); 557 } 558 559 /* 560 * Migration size of VFIO devices can be as little as a few KBs or as big as 561 * many GBs. This value should be big enough to cover the worst case. 562 */ 563 #define VFIO_MIG_STOP_COPY_SIZE (100 * GiB) 564 565 static void vfio_state_pending_exact(void *opaque, uint64_t *must_precopy, 566 uint64_t *can_postcopy) 567 { 568 VFIODevice *vbasedev = opaque; 569 VFIOMigration *migration = vbasedev->migration; 570 uint64_t stop_copy_size = VFIO_MIG_STOP_COPY_SIZE; 571 572 /* 573 * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE is 574 * reported so downtime limit won't be violated. 575 */ 576 vfio_query_stop_copy_size(vbasedev, &stop_copy_size); 577 *must_precopy += stop_copy_size; 578 579 if (vfio_device_state_is_precopy(vbasedev)) { 580 vfio_query_precopy_size(migration); 581 } 582 583 trace_vfio_state_pending_exact(vbasedev->name, *must_precopy, *can_postcopy, 584 stop_copy_size, migration->precopy_init_size, 585 migration->precopy_dirty_size); 586 } 587 588 static bool vfio_is_active_iterate(void *opaque) 589 { 590 VFIODevice *vbasedev = opaque; 591 592 return vfio_device_state_is_precopy(vbasedev); 593 } 594 595 /* 596 * Note about migration rate limiting: VFIO migration buffer size is currently 597 * limited to 1MB, so there is no need to check if migration rate exceeded (as 598 * in the worst case it will exceed by 1MB). However, if the buffer size is 599 * later changed to a bigger value, migration rate should be enforced here. 600 */ 601 static int vfio_save_iterate(QEMUFile *f, void *opaque) 602 { 603 VFIODevice *vbasedev = opaque; 604 VFIOMigration *migration = vbasedev->migration; 605 ssize_t data_size; 606 607 if (!migration->event_save_iterate_started) { 608 trace_vfio_save_iterate_start(vbasedev->name); 609 migration->event_save_iterate_started = true; 610 } 611 612 data_size = vfio_save_block(f, migration); 613 if (data_size < 0) { 614 return data_size; 615 } 616 617 vfio_update_estimated_pending_data(migration, data_size); 618 619 if (migrate_switchover_ack() && !migration->precopy_init_size && 620 !migration->initial_data_sent) { 621 qemu_put_be64(f, VFIO_MIG_FLAG_DEV_INIT_DATA_SENT); 622 migration->initial_data_sent = true; 623 } else { 624 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); 625 } 626 627 trace_vfio_save_iterate(vbasedev->name, migration->precopy_init_size, 628 migration->precopy_dirty_size); 629 630 return !migration->precopy_init_size && !migration->precopy_dirty_size; 631 } 632 633 static int vfio_save_complete_precopy(QEMUFile *f, void *opaque) 634 { 635 VFIODevice *vbasedev = opaque; 636 ssize_t data_size; 637 int ret; 638 Error *local_err = NULL; 639 640 if (vfio_multifd_transfer_enabled(vbasedev)) { 641 vfio_multifd_emit_dummy_eos(vbasedev, f); 642 return 0; 643 } 644 645 trace_vfio_save_complete_precopy_start(vbasedev->name); 646 647 /* We reach here with device state STOP or STOP_COPY only */ 648 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_STOP_COPY, 649 VFIO_DEVICE_STATE_STOP, &local_err); 650 if (ret) { 651 error_report_err(local_err); 652 return ret; 653 } 654 655 do { 656 data_size = vfio_save_block(f, vbasedev->migration); 657 if (data_size < 0) { 658 return data_size; 659 } 660 } while (data_size); 661 662 qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); 663 ret = qemu_file_get_error(f); 664 665 trace_vfio_save_complete_precopy(vbasedev->name, ret); 666 667 return ret; 668 } 669 670 static void vfio_save_state(QEMUFile *f, void *opaque) 671 { 672 VFIODevice *vbasedev = opaque; 673 Error *local_err = NULL; 674 int ret; 675 676 if (vfio_multifd_transfer_enabled(vbasedev)) { 677 vfio_multifd_emit_dummy_eos(vbasedev, f); 678 return; 679 } 680 681 ret = vfio_save_device_config_state(f, opaque, &local_err); 682 if (ret) { 683 error_prepend(&local_err, 684 "vfio: Failed to save device config space of %s - ", 685 vbasedev->name); 686 qemu_file_set_error_obj(f, ret, local_err); 687 } 688 } 689 690 static int vfio_load_setup(QEMUFile *f, void *opaque, Error **errp) 691 { 692 VFIODevice *vbasedev = opaque; 693 VFIOMigration *migration = vbasedev->migration; 694 int ret; 695 696 if (!vfio_multifd_setup(vbasedev, true, errp)) { 697 return -EINVAL; 698 } 699 700 ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_RESUMING, 701 migration->device_state, errp); 702 if (ret) { 703 return ret; 704 } 705 706 return 0; 707 } 708 709 static int vfio_load_cleanup(void *opaque) 710 { 711 VFIODevice *vbasedev = opaque; 712 713 vfio_multifd_cleanup(vbasedev); 714 715 vfio_migration_cleanup(vbasedev); 716 trace_vfio_load_cleanup(vbasedev->name); 717 718 return 0; 719 } 720 721 static int vfio_load_state(QEMUFile *f, void *opaque, int version_id) 722 { 723 VFIODevice *vbasedev = opaque; 724 int ret = 0; 725 uint64_t data; 726 727 data = qemu_get_be64(f); 728 while (data != VFIO_MIG_FLAG_END_OF_STATE) { 729 730 trace_vfio_load_state(vbasedev->name, data); 731 732 switch (data) { 733 case VFIO_MIG_FLAG_DEV_CONFIG_STATE: 734 { 735 if (vfio_multifd_transfer_enabled(vbasedev)) { 736 error_report("%s: got DEV_CONFIG_STATE in main migration " 737 "channel but doing multifd transfer", 738 vbasedev->name); 739 return -EINVAL; 740 } 741 742 return vfio_load_device_config_state(f, opaque); 743 } 744 case VFIO_MIG_FLAG_DEV_SETUP_STATE: 745 { 746 data = qemu_get_be64(f); 747 if (data == VFIO_MIG_FLAG_END_OF_STATE) { 748 return ret; 749 } else { 750 error_report("%s: SETUP STATE: EOS not found 0x%"PRIx64, 751 vbasedev->name, data); 752 return -EINVAL; 753 } 754 break; 755 } 756 case VFIO_MIG_FLAG_DEV_DATA_STATE: 757 { 758 uint64_t data_size = qemu_get_be64(f); 759 760 if (data_size) { 761 ret = vfio_load_buffer(f, vbasedev, data_size); 762 if (ret < 0) { 763 return ret; 764 } 765 } 766 break; 767 } 768 case VFIO_MIG_FLAG_DEV_INIT_DATA_SENT: 769 { 770 if (!vfio_precopy_supported(vbasedev) || 771 !migrate_switchover_ack()) { 772 error_report("%s: Received INIT_DATA_SENT but switchover ack " 773 "is not used", vbasedev->name); 774 return -EINVAL; 775 } 776 777 ret = qemu_loadvm_approve_switchover(); 778 if (ret) { 779 error_report( 780 "%s: qemu_loadvm_approve_switchover failed, err=%d (%s)", 781 vbasedev->name, ret, strerror(-ret)); 782 } 783 784 return ret; 785 } 786 default: 787 error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data); 788 return -EINVAL; 789 } 790 791 data = qemu_get_be64(f); 792 ret = qemu_file_get_error(f); 793 if (ret) { 794 return ret; 795 } 796 } 797 return ret; 798 } 799 800 static bool vfio_switchover_ack_needed(void *opaque) 801 { 802 VFIODevice *vbasedev = opaque; 803 804 return vfio_precopy_supported(vbasedev); 805 } 806 807 static int vfio_switchover_start(void *opaque) 808 { 809 VFIODevice *vbasedev = opaque; 810 811 if (vfio_multifd_transfer_enabled(vbasedev)) { 812 return vfio_multifd_switchover_start(vbasedev); 813 } 814 815 return 0; 816 } 817 818 static const SaveVMHandlers savevm_vfio_handlers = { 819 .save_prepare = vfio_save_prepare, 820 .save_setup = vfio_save_setup, 821 .save_cleanup = vfio_save_cleanup, 822 .state_pending_estimate = vfio_state_pending_estimate, 823 .state_pending_exact = vfio_state_pending_exact, 824 .is_active_iterate = vfio_is_active_iterate, 825 .save_live_iterate = vfio_save_iterate, 826 .save_live_complete_precopy = vfio_save_complete_precopy, 827 .save_state = vfio_save_state, 828 .load_setup = vfio_load_setup, 829 .load_cleanup = vfio_load_cleanup, 830 .load_state = vfio_load_state, 831 .switchover_ack_needed = vfio_switchover_ack_needed, 832 /* 833 * Multifd support 834 */ 835 .load_state_buffer = vfio_multifd_load_state_buffer, 836 .switchover_start = vfio_switchover_start, 837 .save_live_complete_precopy_thread = vfio_multifd_save_complete_precopy_thread, 838 }; 839 840 /* ---------------------------------------------------------------------- */ 841 842 static void vfio_vmstate_change_prepare(void *opaque, bool running, 843 RunState state) 844 { 845 VFIODevice *vbasedev = opaque; 846 VFIOMigration *migration = vbasedev->migration; 847 enum vfio_device_mig_state new_state; 848 Error *local_err = NULL; 849 int ret; 850 851 new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ? 852 VFIO_DEVICE_STATE_PRE_COPY_P2P : 853 VFIO_DEVICE_STATE_RUNNING_P2P; 854 855 ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err); 856 if (ret) { 857 /* 858 * Migration should be aborted in this case, but vm_state_notify() 859 * currently does not support reporting failures. 860 */ 861 migration_file_set_error(ret, local_err); 862 } 863 864 trace_vfio_vmstate_change_prepare(vbasedev->name, running, 865 RunState_str(state), 866 mig_state_to_str(new_state)); 867 } 868 869 static void vfio_vmstate_change(void *opaque, bool running, RunState state) 870 { 871 VFIODevice *vbasedev = opaque; 872 enum vfio_device_mig_state new_state; 873 Error *local_err = NULL; 874 int ret; 875 876 if (running) { 877 new_state = VFIO_DEVICE_STATE_RUNNING; 878 } else { 879 new_state = 880 (vfio_device_state_is_precopy(vbasedev) && 881 (state == RUN_STATE_FINISH_MIGRATE || state == RUN_STATE_PAUSED)) ? 882 VFIO_DEVICE_STATE_STOP_COPY : 883 VFIO_DEVICE_STATE_STOP; 884 } 885 886 ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err); 887 if (ret) { 888 /* 889 * Migration should be aborted in this case, but vm_state_notify() 890 * currently does not support reporting failures. 891 */ 892 migration_file_set_error(ret, local_err); 893 } 894 895 trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state), 896 mig_state_to_str(new_state)); 897 } 898 899 static int vfio_migration_state_notifier(NotifierWithReturn *notifier, 900 MigrationEvent *e, Error **errp) 901 { 902 VFIOMigration *migration = container_of(notifier, VFIOMigration, 903 migration_state); 904 VFIODevice *vbasedev = migration->vbasedev; 905 Error *local_err = NULL; 906 int ret; 907 908 trace_vfio_migration_state_notifier(vbasedev->name, e->type); 909 910 if (e->type == MIG_EVENT_PRECOPY_FAILED) { 911 /* 912 * MigrationNotifyFunc may not return an error code and an Error 913 * object for MIG_EVENT_PRECOPY_FAILED. Hence, report the error 914 * locally and ignore the errp argument. 915 */ 916 ret = vfio_migration_set_state_or_reset(vbasedev, 917 VFIO_DEVICE_STATE_RUNNING, 918 &local_err); 919 if (ret) { 920 error_report_err(local_err); 921 } 922 } 923 return 0; 924 } 925 926 static void vfio_migration_free(VFIODevice *vbasedev) 927 { 928 g_free(vbasedev->migration); 929 vbasedev->migration = NULL; 930 } 931 932 static int vfio_migration_query_flags(VFIODevice *vbasedev, uint64_t *mig_flags) 933 { 934 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 935 sizeof(struct vfio_device_feature_migration), 936 sizeof(uint64_t))] = {}; 937 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 938 struct vfio_device_feature_migration *mig = 939 (struct vfio_device_feature_migration *)feature->data; 940 941 feature->argsz = sizeof(buf); 942 feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIGRATION; 943 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 944 return -errno; 945 } 946 947 *mig_flags = mig->flags; 948 949 return 0; 950 } 951 952 static bool vfio_dma_logging_supported(VFIODevice *vbasedev) 953 { 954 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 955 sizeof(uint64_t))] = {}; 956 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 957 958 feature->argsz = sizeof(buf); 959 feature->flags = VFIO_DEVICE_FEATURE_PROBE | 960 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 961 962 return !ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 963 } 964 965 static int vfio_migration_init(VFIODevice *vbasedev) 966 { 967 int ret; 968 Object *obj; 969 VFIOMigration *migration; 970 char id[256] = ""; 971 g_autofree char *path = NULL, *oid = NULL; 972 uint64_t mig_flags = 0; 973 VMChangeStateHandler *prepare_cb; 974 975 if (!vbasedev->ops->vfio_get_object) { 976 return -EINVAL; 977 } 978 979 obj = vbasedev->ops->vfio_get_object(vbasedev); 980 if (!obj) { 981 return -EINVAL; 982 } 983 984 ret = vfio_migration_query_flags(vbasedev, &mig_flags); 985 if (ret) { 986 return ret; 987 } 988 989 /* Basic migration functionality must be supported */ 990 if (!(mig_flags & VFIO_MIGRATION_STOP_COPY)) { 991 return -EOPNOTSUPP; 992 } 993 994 vbasedev->migration = g_new0(VFIOMigration, 1); 995 migration = vbasedev->migration; 996 migration->vbasedev = vbasedev; 997 migration->device_state = VFIO_DEVICE_STATE_RUNNING; 998 migration->data_fd = -1; 999 migration->mig_flags = mig_flags; 1000 1001 vbasedev->dirty_pages_supported = vfio_dma_logging_supported(vbasedev); 1002 1003 oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj))); 1004 if (oid) { 1005 path = g_strdup_printf("%s/vfio", oid); 1006 } else { 1007 path = g_strdup("vfio"); 1008 } 1009 strpadcpy(id, sizeof(id), path, '\0'); 1010 1011 register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, &savevm_vfio_handlers, 1012 vbasedev); 1013 1014 prepare_cb = migration->mig_flags & VFIO_MIGRATION_P2P ? 1015 vfio_vmstate_change_prepare : 1016 NULL; 1017 migration->vm_state = qdev_add_vm_change_state_handler_full( 1018 vbasedev->dev, vfio_vmstate_change, prepare_cb, vbasedev); 1019 migration_add_notifier(&migration->migration_state, 1020 vfio_migration_state_notifier); 1021 1022 return 0; 1023 } 1024 1025 static void vfio_migration_deinit(VFIODevice *vbasedev) 1026 { 1027 VFIOMigration *migration = vbasedev->migration; 1028 1029 migration_remove_notifier(&migration->migration_state); 1030 qemu_del_vm_change_state_handler(migration->vm_state); 1031 unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev); 1032 vfio_migration_free(vbasedev); 1033 vfio_unblock_multiple_devices_migration(); 1034 } 1035 1036 static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp) 1037 { 1038 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 1039 error_propagate(errp, err); 1040 return -EINVAL; 1041 } 1042 1043 vbasedev->migration_blocker = error_copy(err); 1044 error_free(err); 1045 1046 return migrate_add_blocker_normal(&vbasedev->migration_blocker, errp); 1047 } 1048 1049 /* ---------------------------------------------------------------------- */ 1050 1051 int64_t vfio_mig_bytes_transferred(void) 1052 { 1053 return MIN(qatomic_read(&bytes_transferred), INT64_MAX); 1054 } 1055 1056 void vfio_reset_bytes_transferred(void) 1057 { 1058 qatomic_set(&bytes_transferred, 0); 1059 } 1060 1061 void vfio_mig_add_bytes_transferred(unsigned long val) 1062 { 1063 qatomic_add(&bytes_transferred, val); 1064 } 1065 1066 /* 1067 * Return true when either migration initialized or blocker registered. 1068 * Currently only return false when adding blocker fails which will 1069 * de-register vfio device. 1070 */ 1071 bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp) 1072 { 1073 Error *err = NULL; 1074 int ret; 1075 1076 if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) { 1077 error_setg(&err, "%s: Migration is disabled for VFIO device", 1078 vbasedev->name); 1079 return !vfio_block_migration(vbasedev, err, errp); 1080 } 1081 1082 ret = vfio_migration_init(vbasedev); 1083 if (ret) { 1084 if (ret == -ENOTTY) { 1085 error_setg(&err, "%s: VFIO migration is not supported in kernel", 1086 vbasedev->name); 1087 } else { 1088 error_setg(&err, 1089 "%s: Migration couldn't be initialized for VFIO device, " 1090 "err: %d (%s)", 1091 vbasedev->name, ret, strerror(-ret)); 1092 } 1093 1094 return !vfio_block_migration(vbasedev, err, errp); 1095 } 1096 1097 if ((!vbasedev->dirty_pages_supported || 1098 vbasedev->device_dirty_page_tracking == ON_OFF_AUTO_OFF) && 1099 !vbasedev->iommu_dirty_tracking) { 1100 if (vbasedev->enable_migration == ON_OFF_AUTO_AUTO) { 1101 error_setg(&err, 1102 "%s: VFIO device doesn't support device and " 1103 "IOMMU dirty tracking", vbasedev->name); 1104 goto add_blocker; 1105 } 1106 1107 warn_report("%s: VFIO device doesn't support device and " 1108 "IOMMU dirty tracking", vbasedev->name); 1109 } 1110 1111 ret = vfio_block_multiple_devices_migration(vbasedev, errp); 1112 if (ret) { 1113 goto out_deinit; 1114 } 1115 1116 if (vfio_viommu_preset(vbasedev)) { 1117 error_setg(&err, "%s: Migration is currently not supported " 1118 "with vIOMMU enabled", vbasedev->name); 1119 goto add_blocker; 1120 } 1121 1122 trace_vfio_migration_realize(vbasedev->name); 1123 return true; 1124 1125 add_blocker: 1126 ret = vfio_block_migration(vbasedev, err, errp); 1127 out_deinit: 1128 if (ret) { 1129 vfio_migration_deinit(vbasedev); 1130 } 1131 return !ret; 1132 } 1133 1134 void vfio_migration_exit(VFIODevice *vbasedev) 1135 { 1136 if (vbasedev->migration) { 1137 vfio_migration_deinit(vbasedev); 1138 } 1139 1140 migrate_del_blocker(&vbasedev->migration_blocker); 1141 } 1142