1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/power/hibernate.c - Hibernation (a.k.a suspend-to-disk) support. 4 * 5 * Copyright (c) 2003 Patrick Mochel 6 * Copyright (c) 2003 Open Source Development Lab 7 * Copyright (c) 2004 Pavel Machek <pavel@ucw.cz> 8 * Copyright (c) 2009 Rafael J. Wysocki, Novell Inc. 9 * Copyright (C) 2012 Bojan Smojver <bojan@rexursive.com> 10 */ 11 12 #define pr_fmt(fmt) "PM: hibernation: " fmt 13 14 #include <crypto/acompress.h> 15 #include <linux/blkdev.h> 16 #include <linux/export.h> 17 #include <linux/suspend.h> 18 #include <linux/reboot.h> 19 #include <linux/string.h> 20 #include <linux/device.h> 21 #include <linux/async.h> 22 #include <linux/delay.h> 23 #include <linux/fs.h> 24 #include <linux/mount.h> 25 #include <linux/pm.h> 26 #include <linux/nmi.h> 27 #include <linux/console.h> 28 #include <linux/cpu.h> 29 #include <linux/freezer.h> 30 #include <linux/gfp.h> 31 #include <linux/syscore_ops.h> 32 #include <linux/ctype.h> 33 #include <linux/ktime.h> 34 #include <linux/security.h> 35 #include <linux/secretmem.h> 36 #include <trace/events/power.h> 37 38 #include "power.h" 39 40 41 static int nocompress; 42 static int noresume; 43 static int nohibernate; 44 static int resume_wait; 45 static unsigned int resume_delay; 46 static char resume_file[256] = CONFIG_PM_STD_PARTITION; 47 dev_t swsusp_resume_device; 48 sector_t swsusp_resume_block; 49 __visible int in_suspend __nosavedata; 50 51 static char hibernate_compressor[CRYPTO_MAX_ALG_NAME] = CONFIG_HIBERNATION_DEF_COMP; 52 53 /* 54 * Compression/decompression algorithm to be used while saving/loading 55 * image to/from disk. This would later be used in 'kernel/power/swap.c' 56 * to allocate comp streams. 57 */ 58 char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; 59 60 enum { 61 HIBERNATION_INVALID, 62 HIBERNATION_PLATFORM, 63 HIBERNATION_SHUTDOWN, 64 HIBERNATION_REBOOT, 65 #ifdef CONFIG_SUSPEND 66 HIBERNATION_SUSPEND, 67 #endif 68 HIBERNATION_TEST_RESUME, 69 /* keep last */ 70 __HIBERNATION_AFTER_LAST 71 }; 72 #define HIBERNATION_MAX (__HIBERNATION_AFTER_LAST-1) 73 #define HIBERNATION_FIRST (HIBERNATION_INVALID + 1) 74 75 static int hibernation_mode = HIBERNATION_SHUTDOWN; 76 77 bool freezer_test_done; 78 79 static const struct platform_hibernation_ops *hibernation_ops; 80 81 static atomic_t hibernate_atomic = ATOMIC_INIT(1); 82 83 bool hibernate_acquire(void) 84 { 85 return atomic_add_unless(&hibernate_atomic, -1, 0); 86 } 87 88 void hibernate_release(void) 89 { 90 atomic_inc(&hibernate_atomic); 91 } 92 93 bool hibernation_in_progress(void) 94 { 95 return !atomic_read(&hibernate_atomic); 96 } 97 98 bool hibernation_available(void) 99 { 100 return nohibernate == 0 && 101 !security_locked_down(LOCKDOWN_HIBERNATION) && 102 !secretmem_active() && !cxl_mem_active(); 103 } 104 105 /** 106 * hibernation_set_ops - Set the global hibernate operations. 107 * @ops: Hibernation operations to use in subsequent hibernation transitions. 108 */ 109 void hibernation_set_ops(const struct platform_hibernation_ops *ops) 110 { 111 unsigned int sleep_flags; 112 113 if (ops && !(ops->begin && ops->end && ops->pre_snapshot 114 && ops->prepare && ops->finish && ops->enter && ops->pre_restore 115 && ops->restore_cleanup && ops->leave)) { 116 WARN_ON(1); 117 return; 118 } 119 120 sleep_flags = lock_system_sleep(); 121 122 hibernation_ops = ops; 123 if (ops) 124 hibernation_mode = HIBERNATION_PLATFORM; 125 else if (hibernation_mode == HIBERNATION_PLATFORM) 126 hibernation_mode = HIBERNATION_SHUTDOWN; 127 128 unlock_system_sleep(sleep_flags); 129 } 130 EXPORT_SYMBOL_GPL(hibernation_set_ops); 131 132 static bool entering_platform_hibernation; 133 134 bool system_entering_hibernation(void) 135 { 136 return entering_platform_hibernation; 137 } 138 EXPORT_SYMBOL(system_entering_hibernation); 139 140 #ifdef CONFIG_PM_DEBUG 141 static unsigned int pm_test_delay = 5; 142 module_param(pm_test_delay, uint, 0644); 143 MODULE_PARM_DESC(pm_test_delay, 144 "Number of seconds to wait before resuming from hibernation test"); 145 static void hibernation_debug_sleep(void) 146 { 147 pr_info("hibernation debug: Waiting for %d second(s).\n", 148 pm_test_delay); 149 mdelay(pm_test_delay * 1000); 150 } 151 152 static int hibernation_test(int level) 153 { 154 if (pm_test_level == level) { 155 hibernation_debug_sleep(); 156 return 1; 157 } 158 return 0; 159 } 160 #else /* !CONFIG_PM_DEBUG */ 161 static int hibernation_test(int level) { return 0; } 162 #endif /* !CONFIG_PM_DEBUG */ 163 164 /** 165 * platform_begin - Call platform to start hibernation. 166 * @platform_mode: Whether or not to use the platform driver. 167 */ 168 static int platform_begin(int platform_mode) 169 { 170 return (platform_mode && hibernation_ops) ? 171 hibernation_ops->begin(PMSG_FREEZE) : 0; 172 } 173 174 /** 175 * platform_end - Call platform to finish transition to the working state. 176 * @platform_mode: Whether or not to use the platform driver. 177 */ 178 static void platform_end(int platform_mode) 179 { 180 if (platform_mode && hibernation_ops) 181 hibernation_ops->end(); 182 } 183 184 /** 185 * platform_pre_snapshot - Call platform to prepare the machine for hibernation. 186 * @platform_mode: Whether or not to use the platform driver. 187 * 188 * Use the platform driver to prepare the system for creating a hibernate image, 189 * if so configured, and return an error code if that fails. 190 */ 191 192 static int platform_pre_snapshot(int platform_mode) 193 { 194 return (platform_mode && hibernation_ops) ? 195 hibernation_ops->pre_snapshot() : 0; 196 } 197 198 /** 199 * platform_leave - Call platform to prepare a transition to the working state. 200 * @platform_mode: Whether or not to use the platform driver. 201 * 202 * Use the platform driver prepare to prepare the machine for switching to the 203 * normal mode of operation. 204 * 205 * This routine is called on one CPU with interrupts disabled. 206 */ 207 static void platform_leave(int platform_mode) 208 { 209 if (platform_mode && hibernation_ops) 210 hibernation_ops->leave(); 211 } 212 213 /** 214 * platform_finish - Call platform to switch the system to the working state. 215 * @platform_mode: Whether or not to use the platform driver. 216 * 217 * Use the platform driver to switch the machine to the normal mode of 218 * operation. 219 * 220 * This routine must be called after platform_prepare(). 221 */ 222 static void platform_finish(int platform_mode) 223 { 224 if (platform_mode && hibernation_ops) 225 hibernation_ops->finish(); 226 } 227 228 /** 229 * platform_pre_restore - Prepare for hibernate image restoration. 230 * @platform_mode: Whether or not to use the platform driver. 231 * 232 * Use the platform driver to prepare the system for resume from a hibernation 233 * image. 234 * 235 * If the restore fails after this function has been called, 236 * platform_restore_cleanup() must be called. 237 */ 238 static int platform_pre_restore(int platform_mode) 239 { 240 return (platform_mode && hibernation_ops) ? 241 hibernation_ops->pre_restore() : 0; 242 } 243 244 /** 245 * platform_restore_cleanup - Switch to the working state after failing restore. 246 * @platform_mode: Whether or not to use the platform driver. 247 * 248 * Use the platform driver to switch the system to the normal mode of operation 249 * after a failing restore. 250 * 251 * If platform_pre_restore() has been called before the failing restore, this 252 * function must be called too, regardless of the result of 253 * platform_pre_restore(). 254 */ 255 static void platform_restore_cleanup(int platform_mode) 256 { 257 if (platform_mode && hibernation_ops) 258 hibernation_ops->restore_cleanup(); 259 } 260 261 /** 262 * platform_recover - Recover from a failure to suspend devices. 263 * @platform_mode: Whether or not to use the platform driver. 264 */ 265 static void platform_recover(int platform_mode) 266 { 267 if (platform_mode && hibernation_ops && hibernation_ops->recover) 268 hibernation_ops->recover(); 269 } 270 271 /** 272 * swsusp_show_speed - Print time elapsed between two events during hibernation. 273 * @start: Starting event. 274 * @stop: Final event. 275 * @nr_pages: Number of memory pages processed between @start and @stop. 276 * @msg: Additional diagnostic message to print. 277 */ 278 void swsusp_show_speed(ktime_t start, ktime_t stop, 279 unsigned nr_pages, char *msg) 280 { 281 ktime_t diff; 282 u64 elapsed_centisecs64; 283 unsigned int centisecs; 284 unsigned int k; 285 unsigned int kps; 286 287 diff = ktime_sub(stop, start); 288 elapsed_centisecs64 = ktime_divns(diff, 10*NSEC_PER_MSEC); 289 centisecs = elapsed_centisecs64; 290 if (centisecs == 0) 291 centisecs = 1; /* avoid div-by-zero */ 292 k = nr_pages * (PAGE_SIZE / 1024); 293 kps = (k * 100) / centisecs; 294 pr_info("%s %u kbytes in %u.%02u seconds (%u.%02u MB/s)\n", 295 msg, k, centisecs / 100, centisecs % 100, kps / 1000, 296 (kps % 1000) / 10); 297 } 298 299 __weak int arch_resume_nosmt(void) 300 { 301 return 0; 302 } 303 304 /** 305 * create_image - Create a hibernation image. 306 * @platform_mode: Whether or not to use the platform driver. 307 * 308 * Execute device drivers' "late" and "noirq" freeze callbacks, create a 309 * hibernation image and run the drivers' "noirq" and "early" thaw callbacks. 310 * 311 * Control reappears in this routine after the subsequent restore. 312 */ 313 static int create_image(int platform_mode) 314 { 315 int error; 316 317 error = dpm_suspend_end(PMSG_FREEZE); 318 if (error) { 319 pr_err("Some devices failed to power down, aborting\n"); 320 return error; 321 } 322 323 error = platform_pre_snapshot(platform_mode); 324 if (error || hibernation_test(TEST_PLATFORM)) 325 goto Platform_finish; 326 327 error = pm_sleep_disable_secondary_cpus(); 328 if (error || hibernation_test(TEST_CPUS)) 329 goto Enable_cpus; 330 331 local_irq_disable(); 332 333 system_state = SYSTEM_SUSPEND; 334 335 error = syscore_suspend(); 336 if (error) { 337 pr_err("Some system devices failed to power down, aborting\n"); 338 goto Enable_irqs; 339 } 340 341 if (hibernation_test(TEST_CORE) || pm_wakeup_pending()) 342 goto Power_up; 343 344 in_suspend = 1; 345 save_processor_state(); 346 trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, true); 347 error = swsusp_arch_suspend(); 348 /* Restore control flow magically appears here */ 349 restore_processor_state(); 350 trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, false); 351 if (error) 352 pr_err("Error %d creating image\n", error); 353 354 if (!in_suspend) { 355 events_check_enabled = false; 356 clear_or_poison_free_pages(); 357 } 358 359 platform_leave(platform_mode); 360 361 Power_up: 362 syscore_resume(); 363 364 Enable_irqs: 365 system_state = SYSTEM_RUNNING; 366 local_irq_enable(); 367 368 Enable_cpus: 369 pm_sleep_enable_secondary_cpus(); 370 371 /* Allow architectures to do nosmt-specific post-resume dances */ 372 if (!in_suspend) 373 error = arch_resume_nosmt(); 374 375 Platform_finish: 376 platform_finish(platform_mode); 377 378 dpm_resume_start(in_suspend ? 379 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE); 380 381 return error; 382 } 383 384 /** 385 * hibernation_snapshot - Quiesce devices and create a hibernation image. 386 * @platform_mode: If set, use platform driver to prepare for the transition. 387 * 388 * This routine must be called with system_transition_mutex held. 389 */ 390 int hibernation_snapshot(int platform_mode) 391 { 392 pm_message_t msg; 393 int error; 394 395 pm_suspend_clear_flags(); 396 error = platform_begin(platform_mode); 397 if (error) 398 goto Close; 399 400 /* Preallocate image memory before shutting down devices. */ 401 error = hibernate_preallocate_memory(); 402 if (error) 403 goto Close; 404 405 error = freeze_kernel_threads(); 406 if (error) 407 goto Cleanup; 408 409 if (hibernation_test(TEST_FREEZER)) { 410 411 /* 412 * Indicate to the caller that we are returning due to a 413 * successful freezer test. 414 */ 415 freezer_test_done = true; 416 goto Thaw; 417 } 418 419 error = dpm_prepare(PMSG_FREEZE); 420 if (error) { 421 dpm_complete(PMSG_RECOVER); 422 goto Thaw; 423 } 424 425 console_suspend_all(); 426 pm_restrict_gfp_mask(); 427 428 error = dpm_suspend(PMSG_FREEZE); 429 430 if (error || hibernation_test(TEST_DEVICES)) 431 platform_recover(platform_mode); 432 else 433 error = create_image(platform_mode); 434 435 /* 436 * In the case that we call create_image() above, the control 437 * returns here (1) after the image has been created or the 438 * image creation has failed and (2) after a successful restore. 439 */ 440 441 /* We may need to release the preallocated image pages here. */ 442 if (error || !in_suspend) 443 swsusp_free(); 444 445 msg = in_suspend ? (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE; 446 dpm_resume(msg); 447 448 if (error || !in_suspend) 449 pm_restore_gfp_mask(); 450 451 console_resume_all(); 452 dpm_complete(msg); 453 454 Close: 455 platform_end(platform_mode); 456 return error; 457 458 Thaw: 459 thaw_kernel_threads(); 460 Cleanup: 461 swsusp_free(); 462 goto Close; 463 } 464 465 int __weak hibernate_resume_nonboot_cpu_disable(void) 466 { 467 return suspend_disable_secondary_cpus(); 468 } 469 470 /** 471 * resume_target_kernel - Restore system state from a hibernation image. 472 * @platform_mode: Whether or not to use the platform driver. 473 * 474 * Execute device drivers' "noirq" and "late" freeze callbacks, restore the 475 * contents of highmem that have not been restored yet from the image and run 476 * the low-level code that will restore the remaining contents of memory and 477 * switch to the just restored target kernel. 478 */ 479 static int resume_target_kernel(bool platform_mode) 480 { 481 int error; 482 483 error = dpm_suspend_end(PMSG_QUIESCE); 484 if (error) { 485 pr_err("Some devices failed to power down, aborting resume\n"); 486 return error; 487 } 488 489 error = platform_pre_restore(platform_mode); 490 if (error) 491 goto Cleanup; 492 493 cpuidle_pause(); 494 495 error = hibernate_resume_nonboot_cpu_disable(); 496 if (error) 497 goto Enable_cpus; 498 499 local_irq_disable(); 500 system_state = SYSTEM_SUSPEND; 501 502 error = syscore_suspend(); 503 if (error) 504 goto Enable_irqs; 505 506 save_processor_state(); 507 error = restore_highmem(); 508 if (!error) { 509 error = swsusp_arch_resume(); 510 /* 511 * The code below is only ever reached in case of a failure. 512 * Otherwise, execution continues at the place where 513 * swsusp_arch_suspend() was called. 514 */ 515 BUG_ON(!error); 516 /* 517 * This call to restore_highmem() reverts the changes made by 518 * the previous one. 519 */ 520 restore_highmem(); 521 } 522 /* 523 * The only reason why swsusp_arch_resume() can fail is memory being 524 * very tight, so we have to free it as soon as we can to avoid 525 * subsequent failures. 526 */ 527 swsusp_free(); 528 restore_processor_state(); 529 touch_softlockup_watchdog(); 530 531 syscore_resume(); 532 533 Enable_irqs: 534 system_state = SYSTEM_RUNNING; 535 local_irq_enable(); 536 537 Enable_cpus: 538 pm_sleep_enable_secondary_cpus(); 539 540 Cleanup: 541 platform_restore_cleanup(platform_mode); 542 543 dpm_resume_start(PMSG_RECOVER); 544 545 return error; 546 } 547 548 /** 549 * hibernation_restore - Quiesce devices and restore from a hibernation image. 550 * @platform_mode: If set, use platform driver to prepare for the transition. 551 * 552 * This routine must be called with system_transition_mutex held. If it is 553 * successful, control reappears in the restored target kernel in 554 * hibernation_snapshot(). 555 */ 556 int hibernation_restore(int platform_mode) 557 { 558 int error; 559 560 pm_prepare_console(); 561 console_suspend_all(); 562 pm_restrict_gfp_mask(); 563 error = dpm_suspend_start(PMSG_QUIESCE); 564 if (!error) { 565 error = resume_target_kernel(platform_mode); 566 /* 567 * The above should either succeed and jump to the new kernel, 568 * or return with an error. Otherwise things are just 569 * undefined, so let's be paranoid. 570 */ 571 BUG_ON(!error); 572 } 573 dpm_resume_end(PMSG_RECOVER); 574 pm_restore_gfp_mask(); 575 console_resume_all(); 576 pm_restore_console(); 577 return error; 578 } 579 580 /** 581 * hibernation_platform_enter - Power off the system using the platform driver. 582 */ 583 int hibernation_platform_enter(void) 584 { 585 int error; 586 587 if (!hibernation_ops) 588 return -ENOSYS; 589 590 /* 591 * We have cancelled the power transition by running 592 * hibernation_ops->finish() before saving the image, so we should let 593 * the firmware know that we're going to enter the sleep state after all 594 */ 595 error = hibernation_ops->begin(PMSG_HIBERNATE); 596 if (error) 597 goto Close; 598 599 entering_platform_hibernation = true; 600 console_suspend_all(); 601 error = dpm_suspend_start(PMSG_HIBERNATE); 602 if (error) { 603 if (hibernation_ops->recover) 604 hibernation_ops->recover(); 605 goto Resume_devices; 606 } 607 608 error = dpm_suspend_end(PMSG_HIBERNATE); 609 if (error) 610 goto Resume_devices; 611 612 error = hibernation_ops->prepare(); 613 if (error) 614 goto Platform_finish; 615 616 error = pm_sleep_disable_secondary_cpus(); 617 if (error) 618 goto Enable_cpus; 619 620 local_irq_disable(); 621 system_state = SYSTEM_SUSPEND; 622 623 error = syscore_suspend(); 624 if (error) 625 goto Enable_irqs; 626 627 if (pm_wakeup_pending()) { 628 error = -EAGAIN; 629 goto Power_up; 630 } 631 632 hibernation_ops->enter(); 633 /* We should never get here */ 634 while (1); 635 636 Power_up: 637 syscore_resume(); 638 Enable_irqs: 639 system_state = SYSTEM_RUNNING; 640 local_irq_enable(); 641 642 Enable_cpus: 643 pm_sleep_enable_secondary_cpus(); 644 645 Platform_finish: 646 hibernation_ops->finish(); 647 648 dpm_resume_start(PMSG_RESTORE); 649 650 Resume_devices: 651 entering_platform_hibernation = false; 652 dpm_resume_end(PMSG_RESTORE); 653 console_resume_all(); 654 655 Close: 656 hibernation_ops->end(); 657 658 return error; 659 } 660 661 /** 662 * power_down - Shut the machine down for hibernation. 663 * 664 * Use the platform driver, if configured, to put the system into the sleep 665 * state corresponding to hibernation, or try to power it off or reboot, 666 * depending on the value of hibernation_mode. 667 */ 668 static void power_down(void) 669 { 670 int error; 671 672 #ifdef CONFIG_SUSPEND 673 if (hibernation_mode == HIBERNATION_SUSPEND) { 674 error = suspend_devices_and_enter(mem_sleep_current); 675 if (error) { 676 hibernation_mode = hibernation_ops ? 677 HIBERNATION_PLATFORM : 678 HIBERNATION_SHUTDOWN; 679 } else { 680 /* Restore swap signature. */ 681 error = swsusp_unmark(); 682 if (error) 683 pr_err("Swap will be unusable! Try swapon -a.\n"); 684 685 return; 686 } 687 } 688 #endif 689 690 switch (hibernation_mode) { 691 case HIBERNATION_REBOOT: 692 kernel_restart(NULL); 693 break; 694 case HIBERNATION_PLATFORM: 695 error = hibernation_platform_enter(); 696 if (error == -EAGAIN || error == -EBUSY) { 697 swsusp_unmark(); 698 events_check_enabled = false; 699 pr_info("Wakeup event detected during hibernation, rolling back.\n"); 700 return; 701 } 702 fallthrough; 703 case HIBERNATION_SHUTDOWN: 704 if (kernel_can_power_off()) { 705 entering_platform_hibernation = true; 706 kernel_power_off(); 707 entering_platform_hibernation = false; 708 } 709 break; 710 } 711 kernel_halt(); 712 /* 713 * Valid image is on the disk, if we continue we risk serious data 714 * corruption after resume. 715 */ 716 pr_crit("Power down manually\n"); 717 while (1) 718 cpu_relax(); 719 } 720 721 static int load_image_and_restore(void) 722 { 723 int error; 724 unsigned int flags; 725 726 pm_pr_dbg("Loading hibernation image.\n"); 727 728 lock_device_hotplug(); 729 error = create_basic_memory_bitmaps(); 730 if (error) { 731 swsusp_close(); 732 goto Unlock; 733 } 734 735 error = swsusp_read(&flags); 736 swsusp_close(); 737 if (!error) 738 error = hibernation_restore(flags & SF_PLATFORM_MODE); 739 740 pr_err("Failed to load image, recovering.\n"); 741 swsusp_free(); 742 free_basic_memory_bitmaps(); 743 Unlock: 744 unlock_device_hotplug(); 745 746 return error; 747 } 748 749 #define COMPRESSION_ALGO_LZO "lzo" 750 #define COMPRESSION_ALGO_LZ4 "lz4" 751 752 /** 753 * hibernate - Carry out system hibernation, including saving the image. 754 */ 755 int hibernate(void) 756 { 757 bool snapshot_test = false; 758 unsigned int sleep_flags; 759 int error; 760 761 if (!hibernation_available()) { 762 pm_pr_dbg("Hibernation not available.\n"); 763 return -EPERM; 764 } 765 766 /* 767 * Query for the compression algorithm support if compression is enabled. 768 */ 769 if (!nocompress) { 770 strscpy(hib_comp_algo, hibernate_compressor); 771 if (!crypto_has_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC)) { 772 pr_err("%s compression is not available\n", hib_comp_algo); 773 return -EOPNOTSUPP; 774 } 775 } 776 777 sleep_flags = lock_system_sleep(); 778 /* The snapshot device should not be opened while we're running */ 779 if (!hibernate_acquire()) { 780 error = -EBUSY; 781 goto Unlock; 782 } 783 784 pr_info("hibernation entry\n"); 785 pm_prepare_console(); 786 error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION); 787 if (error) 788 goto Restore; 789 790 ksys_sync_helper(); 791 if (filesystem_freeze_enabled) 792 filesystems_freeze(); 793 794 error = freeze_processes(); 795 if (error) 796 goto Exit; 797 798 lock_device_hotplug(); 799 /* Allocate memory management structures */ 800 error = create_basic_memory_bitmaps(); 801 if (error) 802 goto Thaw; 803 804 error = hibernation_snapshot(hibernation_mode == HIBERNATION_PLATFORM); 805 if (error || freezer_test_done) 806 goto Free_bitmaps; 807 808 if (in_suspend) { 809 unsigned int flags = 0; 810 811 if (hibernation_mode == HIBERNATION_PLATFORM) 812 flags |= SF_PLATFORM_MODE; 813 if (nocompress) { 814 flags |= SF_NOCOMPRESS_MODE; 815 } else { 816 flags |= SF_CRC32_MODE; 817 818 /* 819 * By default, LZO compression is enabled. Use SF_COMPRESSION_ALG_LZ4 820 * to override this behaviour and use LZ4. 821 * 822 * Refer kernel/power/power.h for more details 823 */ 824 825 if (!strcmp(hib_comp_algo, COMPRESSION_ALGO_LZ4)) 826 flags |= SF_COMPRESSION_ALG_LZ4; 827 else 828 flags |= SF_COMPRESSION_ALG_LZO; 829 } 830 831 pm_pr_dbg("Writing hibernation image.\n"); 832 error = swsusp_write(flags); 833 swsusp_free(); 834 if (!error) { 835 if (hibernation_mode == HIBERNATION_TEST_RESUME) 836 snapshot_test = true; 837 else 838 power_down(); 839 } 840 in_suspend = 0; 841 pm_restore_gfp_mask(); 842 } else { 843 pm_pr_dbg("Hibernation image restored successfully.\n"); 844 } 845 846 Free_bitmaps: 847 free_basic_memory_bitmaps(); 848 Thaw: 849 unlock_device_hotplug(); 850 if (snapshot_test) { 851 pm_pr_dbg("Checking hibernation image\n"); 852 error = swsusp_check(false); 853 if (!error) 854 error = load_image_and_restore(); 855 } 856 thaw_processes(); 857 858 /* Don't bother checking whether freezer_test_done is true */ 859 freezer_test_done = false; 860 Exit: 861 filesystems_thaw(); 862 pm_notifier_call_chain(PM_POST_HIBERNATION); 863 Restore: 864 pm_restore_console(); 865 hibernate_release(); 866 Unlock: 867 unlock_system_sleep(sleep_flags); 868 pr_info("hibernation exit\n"); 869 870 return error; 871 } 872 873 /** 874 * hibernate_quiet_exec - Execute a function with all devices frozen. 875 * @func: Function to execute. 876 * @data: Data pointer to pass to @func. 877 * 878 * Return the @func return value or an error code if it cannot be executed. 879 */ 880 int hibernate_quiet_exec(int (*func)(void *data), void *data) 881 { 882 unsigned int sleep_flags; 883 int error; 884 885 sleep_flags = lock_system_sleep(); 886 887 if (!hibernate_acquire()) { 888 error = -EBUSY; 889 goto unlock; 890 } 891 892 pm_prepare_console(); 893 894 error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION); 895 if (error) 896 goto restore; 897 898 if (filesystem_freeze_enabled) 899 filesystems_freeze(); 900 901 error = freeze_processes(); 902 if (error) 903 goto exit; 904 905 lock_device_hotplug(); 906 907 pm_suspend_clear_flags(); 908 909 error = platform_begin(true); 910 if (error) 911 goto thaw; 912 913 error = freeze_kernel_threads(); 914 if (error) 915 goto thaw; 916 917 error = dpm_prepare(PMSG_FREEZE); 918 if (error) 919 goto dpm_complete; 920 921 console_suspend_all(); 922 923 error = dpm_suspend(PMSG_FREEZE); 924 if (error) 925 goto dpm_resume; 926 927 error = dpm_suspend_end(PMSG_FREEZE); 928 if (error) 929 goto dpm_resume; 930 931 error = platform_pre_snapshot(true); 932 if (error) 933 goto skip; 934 935 error = func(data); 936 937 skip: 938 platform_finish(true); 939 940 dpm_resume_start(PMSG_THAW); 941 942 dpm_resume: 943 dpm_resume(PMSG_THAW); 944 945 console_resume_all(); 946 947 dpm_complete: 948 dpm_complete(PMSG_THAW); 949 950 thaw_kernel_threads(); 951 952 thaw: 953 platform_end(true); 954 955 unlock_device_hotplug(); 956 957 thaw_processes(); 958 959 exit: 960 filesystems_thaw(); 961 pm_notifier_call_chain(PM_POST_HIBERNATION); 962 963 restore: 964 pm_restore_console(); 965 966 hibernate_release(); 967 968 unlock: 969 unlock_system_sleep(sleep_flags); 970 971 return error; 972 } 973 EXPORT_SYMBOL_GPL(hibernate_quiet_exec); 974 975 static int __init find_resume_device(void) 976 { 977 if (!strlen(resume_file)) 978 return -ENOENT; 979 980 pm_pr_dbg("Checking hibernation image partition %s\n", resume_file); 981 982 if (resume_delay) { 983 pr_info("Waiting %dsec before reading resume device ...\n", 984 resume_delay); 985 ssleep(resume_delay); 986 } 987 988 /* Check if the device is there */ 989 if (!early_lookup_bdev(resume_file, &swsusp_resume_device)) 990 return 0; 991 992 /* 993 * Some device discovery might still be in progress; we need to wait for 994 * this to finish. 995 */ 996 wait_for_device_probe(); 997 if (resume_wait) { 998 while (early_lookup_bdev(resume_file, &swsusp_resume_device)) 999 msleep(10); 1000 async_synchronize_full(); 1001 } 1002 1003 return early_lookup_bdev(resume_file, &swsusp_resume_device); 1004 } 1005 1006 static int software_resume(void) 1007 { 1008 int error; 1009 1010 pm_pr_dbg("Hibernation image partition %d:%d present\n", 1011 MAJOR(swsusp_resume_device), MINOR(swsusp_resume_device)); 1012 1013 pm_pr_dbg("Looking for hibernation image.\n"); 1014 1015 mutex_lock(&system_transition_mutex); 1016 error = swsusp_check(true); 1017 if (error) 1018 goto Unlock; 1019 1020 /* 1021 * Check if the hibernation image is compressed. If so, query for 1022 * the algorithm support. 1023 */ 1024 if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) { 1025 if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4) 1026 strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4); 1027 else 1028 strscpy(hib_comp_algo, COMPRESSION_ALGO_LZO); 1029 if (!crypto_has_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC)) { 1030 pr_err("%s compression is not available\n", hib_comp_algo); 1031 error = -EOPNOTSUPP; 1032 goto Unlock; 1033 } 1034 } 1035 1036 /* The snapshot device should not be opened while we're running */ 1037 if (!hibernate_acquire()) { 1038 error = -EBUSY; 1039 swsusp_close(); 1040 goto Unlock; 1041 } 1042 1043 pr_info("resume from hibernation\n"); 1044 pm_prepare_console(); 1045 error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE); 1046 if (error) 1047 goto Restore; 1048 1049 if (filesystem_freeze_enabled) 1050 filesystems_freeze(); 1051 1052 pm_pr_dbg("Preparing processes for hibernation restore.\n"); 1053 error = freeze_processes(); 1054 if (error) { 1055 filesystems_thaw(); 1056 goto Close_Finish; 1057 } 1058 1059 error = freeze_kernel_threads(); 1060 if (error) { 1061 thaw_processes(); 1062 filesystems_thaw(); 1063 goto Close_Finish; 1064 } 1065 1066 error = load_image_and_restore(); 1067 thaw_processes(); 1068 filesystems_thaw(); 1069 Finish: 1070 pm_notifier_call_chain(PM_POST_RESTORE); 1071 Restore: 1072 pm_restore_console(); 1073 pr_info("resume failed (%d)\n", error); 1074 hibernate_release(); 1075 /* For success case, the suspend path will release the lock */ 1076 Unlock: 1077 mutex_unlock(&system_transition_mutex); 1078 pm_pr_dbg("Hibernation image not present or could not be loaded.\n"); 1079 return error; 1080 Close_Finish: 1081 swsusp_close(); 1082 goto Finish; 1083 } 1084 1085 /** 1086 * software_resume_initcall - Resume from a saved hibernation image. 1087 * 1088 * This routine is called as a late initcall, when all devices have been 1089 * discovered and initialized already. 1090 * 1091 * The image reading code is called to see if there is a hibernation image 1092 * available for reading. If that is the case, devices are quiesced and the 1093 * contents of memory is restored from the saved image. 1094 * 1095 * If this is successful, control reappears in the restored target kernel in 1096 * hibernation_snapshot() which returns to hibernate(). Otherwise, the routine 1097 * attempts to recover gracefully and make the kernel return to the normal mode 1098 * of operation. 1099 */ 1100 static int __init software_resume_initcall(void) 1101 { 1102 /* 1103 * If the user said "noresume".. bail out early. 1104 */ 1105 if (noresume || !hibernation_available()) 1106 return 0; 1107 1108 if (!swsusp_resume_device) { 1109 int error = find_resume_device(); 1110 1111 if (error) 1112 return error; 1113 } 1114 1115 return software_resume(); 1116 } 1117 late_initcall_sync(software_resume_initcall); 1118 1119 1120 static const char * const hibernation_modes[] = { 1121 [HIBERNATION_PLATFORM] = "platform", 1122 [HIBERNATION_SHUTDOWN] = "shutdown", 1123 [HIBERNATION_REBOOT] = "reboot", 1124 #ifdef CONFIG_SUSPEND 1125 [HIBERNATION_SUSPEND] = "suspend", 1126 #endif 1127 [HIBERNATION_TEST_RESUME] = "test_resume", 1128 }; 1129 1130 /* 1131 * /sys/power/disk - Control hibernation mode. 1132 * 1133 * Hibernation can be handled in several ways. There are a few different ways 1134 * to put the system into the sleep state: using the platform driver (e.g. ACPI 1135 * or other hibernation_ops), powering it off or rebooting it (for testing 1136 * mostly). 1137 * 1138 * The sysfs file /sys/power/disk provides an interface for selecting the 1139 * hibernation mode to use. Reading from this file causes the available modes 1140 * to be printed. There are 3 modes that can be supported: 1141 * 1142 * 'platform' 1143 * 'shutdown' 1144 * 'reboot' 1145 * 1146 * If a platform hibernation driver is in use, 'platform' will be supported 1147 * and will be used by default. Otherwise, 'shutdown' will be used by default. 1148 * The selected option (i.e. the one corresponding to the current value of 1149 * hibernation_mode) is enclosed by a square bracket. 1150 * 1151 * To select a given hibernation mode it is necessary to write the mode's 1152 * string representation (as returned by reading from /sys/power/disk) back 1153 * into /sys/power/disk. 1154 */ 1155 1156 static ssize_t disk_show(struct kobject *kobj, struct kobj_attribute *attr, 1157 char *buf) 1158 { 1159 ssize_t count = 0; 1160 int i; 1161 1162 if (!hibernation_available()) 1163 return sysfs_emit(buf, "[disabled]\n"); 1164 1165 for (i = HIBERNATION_FIRST; i <= HIBERNATION_MAX; i++) { 1166 if (!hibernation_modes[i]) 1167 continue; 1168 switch (i) { 1169 case HIBERNATION_SHUTDOWN: 1170 case HIBERNATION_REBOOT: 1171 #ifdef CONFIG_SUSPEND 1172 case HIBERNATION_SUSPEND: 1173 #endif 1174 case HIBERNATION_TEST_RESUME: 1175 break; 1176 case HIBERNATION_PLATFORM: 1177 if (hibernation_ops) 1178 break; 1179 /* not a valid mode, continue with loop */ 1180 continue; 1181 } 1182 if (i == hibernation_mode) 1183 count += sysfs_emit_at(buf, count, "[%s] ", hibernation_modes[i]); 1184 else 1185 count += sysfs_emit_at(buf, count, "%s ", hibernation_modes[i]); 1186 } 1187 1188 /* Convert the last space to a newline if needed. */ 1189 if (count > 0) 1190 buf[count - 1] = '\n'; 1191 1192 return count; 1193 } 1194 1195 static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, 1196 const char *buf, size_t n) 1197 { 1198 int mode = HIBERNATION_INVALID; 1199 unsigned int sleep_flags; 1200 int error = 0; 1201 int len; 1202 char *p; 1203 int i; 1204 1205 if (!hibernation_available()) 1206 return -EPERM; 1207 1208 p = memchr(buf, '\n', n); 1209 len = p ? p - buf : n; 1210 1211 sleep_flags = lock_system_sleep(); 1212 for (i = HIBERNATION_FIRST; i <= HIBERNATION_MAX; i++) { 1213 if (len == strlen(hibernation_modes[i]) 1214 && !strncmp(buf, hibernation_modes[i], len)) { 1215 mode = i; 1216 break; 1217 } 1218 } 1219 if (mode != HIBERNATION_INVALID) { 1220 switch (mode) { 1221 case HIBERNATION_SHUTDOWN: 1222 case HIBERNATION_REBOOT: 1223 #ifdef CONFIG_SUSPEND 1224 case HIBERNATION_SUSPEND: 1225 #endif 1226 case HIBERNATION_TEST_RESUME: 1227 hibernation_mode = mode; 1228 break; 1229 case HIBERNATION_PLATFORM: 1230 if (hibernation_ops) 1231 hibernation_mode = mode; 1232 else 1233 error = -EINVAL; 1234 } 1235 } else 1236 error = -EINVAL; 1237 1238 if (!error) 1239 pm_pr_dbg("Hibernation mode set to '%s'\n", 1240 hibernation_modes[mode]); 1241 unlock_system_sleep(sleep_flags); 1242 return error ? error : n; 1243 } 1244 1245 power_attr(disk); 1246 1247 static ssize_t resume_show(struct kobject *kobj, struct kobj_attribute *attr, 1248 char *buf) 1249 { 1250 return sysfs_emit(buf, "%d:%d\n", MAJOR(swsusp_resume_device), 1251 MINOR(swsusp_resume_device)); 1252 } 1253 1254 static ssize_t resume_store(struct kobject *kobj, struct kobj_attribute *attr, 1255 const char *buf, size_t n) 1256 { 1257 unsigned int sleep_flags; 1258 int len = n; 1259 char *name; 1260 dev_t dev; 1261 int error; 1262 1263 if (!hibernation_available()) 1264 return n; 1265 1266 if (len && buf[len-1] == '\n') 1267 len--; 1268 name = kstrndup(buf, len, GFP_KERNEL); 1269 if (!name) 1270 return -ENOMEM; 1271 1272 error = lookup_bdev(name, &dev); 1273 if (error) { 1274 unsigned maj, min, offset; 1275 char *p, dummy; 1276 1277 error = 0; 1278 if (sscanf(name, "%u:%u%c", &maj, &min, &dummy) == 2 || 1279 sscanf(name, "%u:%u:%u:%c", &maj, &min, &offset, 1280 &dummy) == 3) { 1281 dev = MKDEV(maj, min); 1282 if (maj != MAJOR(dev) || min != MINOR(dev)) 1283 error = -EINVAL; 1284 } else { 1285 dev = new_decode_dev(simple_strtoul(name, &p, 16)); 1286 if (*p) 1287 error = -EINVAL; 1288 } 1289 } 1290 kfree(name); 1291 if (error) 1292 return error; 1293 1294 sleep_flags = lock_system_sleep(); 1295 swsusp_resume_device = dev; 1296 unlock_system_sleep(sleep_flags); 1297 1298 pm_pr_dbg("Configured hibernation resume from disk to %u\n", 1299 swsusp_resume_device); 1300 noresume = 0; 1301 software_resume(); 1302 return n; 1303 } 1304 1305 power_attr(resume); 1306 1307 static ssize_t resume_offset_show(struct kobject *kobj, 1308 struct kobj_attribute *attr, char *buf) 1309 { 1310 return sysfs_emit(buf, "%llu\n", (unsigned long long)swsusp_resume_block); 1311 } 1312 1313 static ssize_t resume_offset_store(struct kobject *kobj, 1314 struct kobj_attribute *attr, const char *buf, 1315 size_t n) 1316 { 1317 unsigned long long offset; 1318 int rc; 1319 1320 rc = kstrtoull(buf, 0, &offset); 1321 if (rc) 1322 return rc; 1323 swsusp_resume_block = offset; 1324 1325 return n; 1326 } 1327 1328 power_attr(resume_offset); 1329 1330 static ssize_t image_size_show(struct kobject *kobj, struct kobj_attribute *attr, 1331 char *buf) 1332 { 1333 return sysfs_emit(buf, "%lu\n", image_size); 1334 } 1335 1336 static ssize_t image_size_store(struct kobject *kobj, struct kobj_attribute *attr, 1337 const char *buf, size_t n) 1338 { 1339 unsigned long size; 1340 1341 if (sscanf(buf, "%lu", &size) == 1) { 1342 image_size = size; 1343 return n; 1344 } 1345 1346 return -EINVAL; 1347 } 1348 1349 power_attr(image_size); 1350 1351 static ssize_t reserved_size_show(struct kobject *kobj, 1352 struct kobj_attribute *attr, char *buf) 1353 { 1354 return sysfs_emit(buf, "%lu\n", reserved_size); 1355 } 1356 1357 static ssize_t reserved_size_store(struct kobject *kobj, 1358 struct kobj_attribute *attr, 1359 const char *buf, size_t n) 1360 { 1361 unsigned long size; 1362 1363 if (sscanf(buf, "%lu", &size) == 1) { 1364 reserved_size = size; 1365 return n; 1366 } 1367 1368 return -EINVAL; 1369 } 1370 1371 power_attr(reserved_size); 1372 1373 static struct attribute *g[] = { 1374 &disk_attr.attr, 1375 &resume_offset_attr.attr, 1376 &resume_attr.attr, 1377 &image_size_attr.attr, 1378 &reserved_size_attr.attr, 1379 NULL, 1380 }; 1381 1382 1383 static const struct attribute_group attr_group = { 1384 .attrs = g, 1385 }; 1386 1387 1388 static int __init pm_disk_init(void) 1389 { 1390 return sysfs_create_group(power_kobj, &attr_group); 1391 } 1392 1393 core_initcall(pm_disk_init); 1394 1395 1396 static int __init resume_setup(char *str) 1397 { 1398 if (noresume) 1399 return 1; 1400 1401 strscpy(resume_file, str); 1402 return 1; 1403 } 1404 1405 static int __init resume_offset_setup(char *str) 1406 { 1407 unsigned long long offset; 1408 1409 if (noresume) 1410 return 1; 1411 1412 if (sscanf(str, "%llu", &offset) == 1) 1413 swsusp_resume_block = offset; 1414 1415 return 1; 1416 } 1417 1418 static int __init hibernate_setup(char *str) 1419 { 1420 if (!strncmp(str, "noresume", 8)) { 1421 noresume = 1; 1422 } else if (!strncmp(str, "nocompress", 10)) { 1423 nocompress = 1; 1424 } else if (!strncmp(str, "no", 2)) { 1425 noresume = 1; 1426 nohibernate = 1; 1427 } else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) 1428 && !strncmp(str, "protect_image", 13)) { 1429 enable_restore_image_protection(); 1430 } 1431 return 1; 1432 } 1433 1434 static int __init noresume_setup(char *str) 1435 { 1436 noresume = 1; 1437 return 1; 1438 } 1439 1440 static int __init resumewait_setup(char *str) 1441 { 1442 resume_wait = 1; 1443 return 1; 1444 } 1445 1446 static int __init resumedelay_setup(char *str) 1447 { 1448 int rc = kstrtouint(str, 0, &resume_delay); 1449 1450 if (rc) 1451 pr_warn("resumedelay: bad option string '%s'\n", str); 1452 return 1; 1453 } 1454 1455 static int __init nohibernate_setup(char *str) 1456 { 1457 noresume = 1; 1458 nohibernate = 1; 1459 return 1; 1460 } 1461 1462 static const char * const comp_alg_enabled[] = { 1463 #if IS_ENABLED(CONFIG_CRYPTO_LZO) 1464 COMPRESSION_ALGO_LZO, 1465 #endif 1466 #if IS_ENABLED(CONFIG_CRYPTO_LZ4) 1467 COMPRESSION_ALGO_LZ4, 1468 #endif 1469 }; 1470 1471 static int hibernate_compressor_param_set(const char *compressor, 1472 const struct kernel_param *kp) 1473 { 1474 int index, ret; 1475 1476 if (!mutex_trylock(&system_transition_mutex)) 1477 return -EBUSY; 1478 1479 index = sysfs_match_string(comp_alg_enabled, compressor); 1480 if (index >= 0) { 1481 ret = param_set_copystring(comp_alg_enabled[index], kp); 1482 if (!ret) 1483 strscpy(hib_comp_algo, comp_alg_enabled[index]); 1484 } else { 1485 ret = index; 1486 } 1487 1488 mutex_unlock(&system_transition_mutex); 1489 1490 if (ret) 1491 pr_debug("Cannot set specified compressor %s\n", 1492 compressor); 1493 1494 return ret; 1495 } 1496 1497 static const struct kernel_param_ops hibernate_compressor_param_ops = { 1498 .set = hibernate_compressor_param_set, 1499 .get = param_get_string, 1500 }; 1501 1502 static struct kparam_string hibernate_compressor_param_string = { 1503 .maxlen = sizeof(hibernate_compressor), 1504 .string = hibernate_compressor, 1505 }; 1506 1507 module_param_cb(compressor, &hibernate_compressor_param_ops, 1508 &hibernate_compressor_param_string, 0644); 1509 MODULE_PARM_DESC(compressor, 1510 "Compression algorithm to be used with hibernation"); 1511 1512 __setup("noresume", noresume_setup); 1513 __setup("resume_offset=", resume_offset_setup); 1514 __setup("resume=", resume_setup); 1515 __setup("hibernate=", hibernate_setup); 1516 __setup("resumewait", resumewait_setup); 1517 __setup("resumedelay=", resumedelay_setup); 1518 __setup("nohibernate", nohibernate_setup); 1519