1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/madvise.c 4 * 5 * Copyright (C) 1999 Linus Torvalds 6 * Copyright (C) 2002 Christoph Hellwig 7 */ 8 9 #include <linux/mman.h> 10 #include <linux/pagemap.h> 11 #include <linux/syscalls.h> 12 #include <linux/mempolicy.h> 13 #include <linux/page-isolation.h> 14 #include <linux/page_idle.h> 15 #include <linux/userfaultfd_k.h> 16 #include <linux/hugetlb.h> 17 #include <linux/falloc.h> 18 #include <linux/fadvise.h> 19 #include <linux/sched.h> 20 #include <linux/sched/mm.h> 21 #include <linux/mm_inline.h> 22 #include <linux/string.h> 23 #include <linux/uio.h> 24 #include <linux/ksm.h> 25 #include <linux/fs.h> 26 #include <linux/file.h> 27 #include <linux/blkdev.h> 28 #include <linux/backing-dev.h> 29 #include <linux/pagewalk.h> 30 #include <linux/swap.h> 31 #include <linux/swapops.h> 32 #include <linux/shmem_fs.h> 33 #include <linux/mmu_notifier.h> 34 35 #include <asm/tlb.h> 36 37 #include "internal.h" 38 #include "swap.h" 39 40 /* 41 * Maximum number of attempts we make to install guard pages before we give up 42 * and return -ERESTARTNOINTR to have userspace try again. 43 */ 44 #define MAX_MADVISE_GUARD_RETRIES 3 45 46 struct madvise_walk_private { 47 struct mmu_gather *tlb; 48 bool pageout; 49 }; 50 51 /* 52 * Any behaviour which results in changes to the vma->vm_flags needs to 53 * take mmap_lock for writing. Others, which simply traverse vmas, need 54 * to only take it for reading. 55 */ 56 static int madvise_need_mmap_write(int behavior) 57 { 58 switch (behavior) { 59 case MADV_REMOVE: 60 case MADV_WILLNEED: 61 case MADV_DONTNEED: 62 case MADV_DONTNEED_LOCKED: 63 case MADV_COLD: 64 case MADV_PAGEOUT: 65 case MADV_FREE: 66 case MADV_POPULATE_READ: 67 case MADV_POPULATE_WRITE: 68 case MADV_COLLAPSE: 69 case MADV_GUARD_INSTALL: 70 case MADV_GUARD_REMOVE: 71 return 0; 72 default: 73 /* be safe, default to 1. list exceptions explicitly */ 74 return 1; 75 } 76 } 77 78 #ifdef CONFIG_ANON_VMA_NAME 79 struct anon_vma_name *anon_vma_name_alloc(const char *name) 80 { 81 struct anon_vma_name *anon_name; 82 size_t count; 83 84 /* Add 1 for NUL terminator at the end of the anon_name->name */ 85 count = strlen(name) + 1; 86 anon_name = kmalloc(struct_size(anon_name, name, count), GFP_KERNEL); 87 if (anon_name) { 88 kref_init(&anon_name->kref); 89 memcpy(anon_name->name, name, count); 90 } 91 92 return anon_name; 93 } 94 95 void anon_vma_name_free(struct kref *kref) 96 { 97 struct anon_vma_name *anon_name = 98 container_of(kref, struct anon_vma_name, kref); 99 kfree(anon_name); 100 } 101 102 struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma) 103 { 104 mmap_assert_locked(vma->vm_mm); 105 106 return vma->anon_name; 107 } 108 109 /* mmap_lock should be write-locked */ 110 static int replace_anon_vma_name(struct vm_area_struct *vma, 111 struct anon_vma_name *anon_name) 112 { 113 struct anon_vma_name *orig_name = anon_vma_name(vma); 114 115 if (!anon_name) { 116 vma->anon_name = NULL; 117 anon_vma_name_put(orig_name); 118 return 0; 119 } 120 121 if (anon_vma_name_eq(orig_name, anon_name)) 122 return 0; 123 124 vma->anon_name = anon_vma_name_reuse(anon_name); 125 anon_vma_name_put(orig_name); 126 127 return 0; 128 } 129 #else /* CONFIG_ANON_VMA_NAME */ 130 static int replace_anon_vma_name(struct vm_area_struct *vma, 131 struct anon_vma_name *anon_name) 132 { 133 if (anon_name) 134 return -EINVAL; 135 136 return 0; 137 } 138 #endif /* CONFIG_ANON_VMA_NAME */ 139 /* 140 * Update the vm_flags on region of a vma, splitting it or merging it as 141 * necessary. Must be called with mmap_lock held for writing; 142 * Caller should ensure anon_name stability by raising its refcount even when 143 * anon_name belongs to a valid vma because this function might free that vma. 144 */ 145 static int madvise_update_vma(struct vm_area_struct *vma, 146 struct vm_area_struct **prev, unsigned long start, 147 unsigned long end, unsigned long new_flags, 148 struct anon_vma_name *anon_name) 149 { 150 struct mm_struct *mm = vma->vm_mm; 151 int error; 152 VMA_ITERATOR(vmi, mm, start); 153 154 if (new_flags == vma->vm_flags && anon_vma_name_eq(anon_vma_name(vma), anon_name)) { 155 *prev = vma; 156 return 0; 157 } 158 159 vma = vma_modify_flags_name(&vmi, *prev, vma, start, end, new_flags, 160 anon_name); 161 if (IS_ERR(vma)) 162 return PTR_ERR(vma); 163 164 *prev = vma; 165 166 /* vm_flags is protected by the mmap_lock held in write mode. */ 167 vma_start_write(vma); 168 vm_flags_reset(vma, new_flags); 169 if (!vma->vm_file || vma_is_anon_shmem(vma)) { 170 error = replace_anon_vma_name(vma, anon_name); 171 if (error) 172 return error; 173 } 174 175 return 0; 176 } 177 178 #ifdef CONFIG_SWAP 179 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, 180 unsigned long end, struct mm_walk *walk) 181 { 182 struct vm_area_struct *vma = walk->private; 183 struct swap_iocb *splug = NULL; 184 pte_t *ptep = NULL; 185 spinlock_t *ptl; 186 unsigned long addr; 187 188 for (addr = start; addr < end; addr += PAGE_SIZE) { 189 pte_t pte; 190 swp_entry_t entry; 191 struct folio *folio; 192 193 if (!ptep++) { 194 ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 195 if (!ptep) 196 break; 197 } 198 199 pte = ptep_get(ptep); 200 if (!is_swap_pte(pte)) 201 continue; 202 entry = pte_to_swp_entry(pte); 203 if (unlikely(non_swap_entry(entry))) 204 continue; 205 206 pte_unmap_unlock(ptep, ptl); 207 ptep = NULL; 208 209 folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE, 210 vma, addr, &splug); 211 if (folio) 212 folio_put(folio); 213 } 214 215 if (ptep) 216 pte_unmap_unlock(ptep, ptl); 217 swap_read_unplug(splug); 218 cond_resched(); 219 220 return 0; 221 } 222 223 static const struct mm_walk_ops swapin_walk_ops = { 224 .pmd_entry = swapin_walk_pmd_entry, 225 .walk_lock = PGWALK_RDLOCK, 226 }; 227 228 static void shmem_swapin_range(struct vm_area_struct *vma, 229 unsigned long start, unsigned long end, 230 struct address_space *mapping) 231 { 232 XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); 233 pgoff_t end_index = linear_page_index(vma, end) - 1; 234 struct folio *folio; 235 struct swap_iocb *splug = NULL; 236 237 rcu_read_lock(); 238 xas_for_each(&xas, folio, end_index) { 239 unsigned long addr; 240 swp_entry_t entry; 241 242 if (!xa_is_value(folio)) 243 continue; 244 entry = radix_to_swp_entry(folio); 245 /* There might be swapin error entries in shmem mapping. */ 246 if (non_swap_entry(entry)) 247 continue; 248 249 addr = vma->vm_start + 250 ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); 251 xas_pause(&xas); 252 rcu_read_unlock(); 253 254 folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping), 255 vma, addr, &splug); 256 if (folio) 257 folio_put(folio); 258 259 rcu_read_lock(); 260 } 261 rcu_read_unlock(); 262 swap_read_unplug(splug); 263 } 264 #endif /* CONFIG_SWAP */ 265 266 /* 267 * Schedule all required I/O operations. Do not wait for completion. 268 */ 269 static long madvise_willneed(struct vm_area_struct *vma, 270 struct vm_area_struct **prev, 271 unsigned long start, unsigned long end) 272 { 273 struct mm_struct *mm = vma->vm_mm; 274 struct file *file = vma->vm_file; 275 loff_t offset; 276 277 *prev = vma; 278 #ifdef CONFIG_SWAP 279 if (!file) { 280 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma); 281 lru_add_drain(); /* Push any new pages onto the LRU now */ 282 return 0; 283 } 284 285 if (shmem_mapping(file->f_mapping)) { 286 shmem_swapin_range(vma, start, end, file->f_mapping); 287 lru_add_drain(); /* Push any new pages onto the LRU now */ 288 return 0; 289 } 290 #else 291 if (!file) 292 return -EBADF; 293 #endif 294 295 if (IS_DAX(file_inode(file))) { 296 /* no bad return value, but ignore advice */ 297 return 0; 298 } 299 300 /* 301 * Filesystem's fadvise may need to take various locks. We need to 302 * explicitly grab a reference because the vma (and hence the 303 * vma's reference to the file) can go away as soon as we drop 304 * mmap_lock. 305 */ 306 *prev = NULL; /* tell sys_madvise we drop mmap_lock */ 307 get_file(file); 308 offset = (loff_t)(start - vma->vm_start) 309 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 310 mmap_read_unlock(mm); 311 vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); 312 fput(file); 313 mmap_read_lock(mm); 314 return 0; 315 } 316 317 static inline bool can_do_file_pageout(struct vm_area_struct *vma) 318 { 319 if (!vma->vm_file) 320 return false; 321 /* 322 * paging out pagecache only for non-anonymous mappings that correspond 323 * to the files the calling process could (if tried) open for writing; 324 * otherwise we'd be including shared non-exclusive mappings, which 325 * opens a side channel. 326 */ 327 return inode_owner_or_capable(&nop_mnt_idmap, 328 file_inode(vma->vm_file)) || 329 file_permission(vma->vm_file, MAY_WRITE) == 0; 330 } 331 332 static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, 333 struct folio *folio, pte_t *ptep, 334 pte_t pte, bool *any_young, 335 bool *any_dirty) 336 { 337 const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; 338 int max_nr = (end - addr) / PAGE_SIZE; 339 340 return folio_pte_batch(folio, addr, ptep, pte, max_nr, fpb_flags, NULL, 341 any_young, any_dirty); 342 } 343 344 static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, 345 unsigned long addr, unsigned long end, 346 struct mm_walk *walk) 347 { 348 struct madvise_walk_private *private = walk->private; 349 struct mmu_gather *tlb = private->tlb; 350 bool pageout = private->pageout; 351 struct mm_struct *mm = tlb->mm; 352 struct vm_area_struct *vma = walk->vma; 353 pte_t *start_pte, *pte, ptent; 354 spinlock_t *ptl; 355 struct folio *folio = NULL; 356 LIST_HEAD(folio_list); 357 bool pageout_anon_only_filter; 358 unsigned int batch_count = 0; 359 int nr; 360 361 if (fatal_signal_pending(current)) 362 return -EINTR; 363 364 pageout_anon_only_filter = pageout && !vma_is_anonymous(vma) && 365 !can_do_file_pageout(vma); 366 367 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 368 if (pmd_trans_huge(*pmd)) { 369 pmd_t orig_pmd; 370 unsigned long next = pmd_addr_end(addr, end); 371 372 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 373 ptl = pmd_trans_huge_lock(pmd, vma); 374 if (!ptl) 375 return 0; 376 377 orig_pmd = *pmd; 378 if (is_huge_zero_pmd(orig_pmd)) 379 goto huge_unlock; 380 381 if (unlikely(!pmd_present(orig_pmd))) { 382 VM_BUG_ON(thp_migration_supported() && 383 !is_pmd_migration_entry(orig_pmd)); 384 goto huge_unlock; 385 } 386 387 folio = pmd_folio(orig_pmd); 388 389 /* Do not interfere with other mappings of this folio */ 390 if (folio_likely_mapped_shared(folio)) 391 goto huge_unlock; 392 393 if (pageout_anon_only_filter && !folio_test_anon(folio)) 394 goto huge_unlock; 395 396 if (next - addr != HPAGE_PMD_SIZE) { 397 int err; 398 399 folio_get(folio); 400 spin_unlock(ptl); 401 folio_lock(folio); 402 err = split_folio(folio); 403 folio_unlock(folio); 404 folio_put(folio); 405 if (!err) 406 goto regular_folio; 407 return 0; 408 } 409 410 if (!pageout && pmd_young(orig_pmd)) { 411 pmdp_invalidate(vma, addr, pmd); 412 orig_pmd = pmd_mkold(orig_pmd); 413 414 set_pmd_at(mm, addr, pmd, orig_pmd); 415 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 416 } 417 418 folio_clear_referenced(folio); 419 folio_test_clear_young(folio); 420 if (folio_test_active(folio)) 421 folio_set_workingset(folio); 422 if (pageout) { 423 if (folio_isolate_lru(folio)) { 424 if (folio_test_unevictable(folio)) 425 folio_putback_lru(folio); 426 else 427 list_add(&folio->lru, &folio_list); 428 } 429 } else 430 folio_deactivate(folio); 431 huge_unlock: 432 spin_unlock(ptl); 433 if (pageout) 434 reclaim_pages(&folio_list); 435 return 0; 436 } 437 438 regular_folio: 439 #endif 440 tlb_change_page_size(tlb, PAGE_SIZE); 441 restart: 442 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 443 if (!start_pte) 444 return 0; 445 flush_tlb_batched_pending(mm); 446 arch_enter_lazy_mmu_mode(); 447 for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { 448 nr = 1; 449 ptent = ptep_get(pte); 450 451 if (++batch_count == SWAP_CLUSTER_MAX) { 452 batch_count = 0; 453 if (need_resched()) { 454 arch_leave_lazy_mmu_mode(); 455 pte_unmap_unlock(start_pte, ptl); 456 cond_resched(); 457 goto restart; 458 } 459 } 460 461 if (pte_none(ptent)) 462 continue; 463 464 if (!pte_present(ptent)) 465 continue; 466 467 folio = vm_normal_folio(vma, addr, ptent); 468 if (!folio || folio_is_zone_device(folio)) 469 continue; 470 471 /* 472 * If we encounter a large folio, only split it if it is not 473 * fully mapped within the range we are operating on. Otherwise 474 * leave it as is so that it can be swapped out whole. If we 475 * fail to split a folio, leave it in place and advance to the 476 * next pte in the range. 477 */ 478 if (folio_test_large(folio)) { 479 bool any_young; 480 481 nr = madvise_folio_pte_batch(addr, end, folio, pte, 482 ptent, &any_young, NULL); 483 if (any_young) 484 ptent = pte_mkyoung(ptent); 485 486 if (nr < folio_nr_pages(folio)) { 487 int err; 488 489 if (folio_likely_mapped_shared(folio)) 490 continue; 491 if (pageout_anon_only_filter && !folio_test_anon(folio)) 492 continue; 493 if (!folio_trylock(folio)) 494 continue; 495 folio_get(folio); 496 arch_leave_lazy_mmu_mode(); 497 pte_unmap_unlock(start_pte, ptl); 498 start_pte = NULL; 499 err = split_folio(folio); 500 folio_unlock(folio); 501 folio_put(folio); 502 start_pte = pte = 503 pte_offset_map_lock(mm, pmd, addr, &ptl); 504 if (!start_pte) 505 break; 506 arch_enter_lazy_mmu_mode(); 507 if (!err) 508 nr = 0; 509 continue; 510 } 511 } 512 513 /* 514 * Do not interfere with other mappings of this folio and 515 * non-LRU folio. If we have a large folio at this point, we 516 * know it is fully mapped so if its mapcount is the same as its 517 * number of pages, it must be exclusive. 518 */ 519 if (!folio_test_lru(folio) || 520 folio_mapcount(folio) != folio_nr_pages(folio)) 521 continue; 522 523 if (pageout_anon_only_filter && !folio_test_anon(folio)) 524 continue; 525 526 if (!pageout && pte_young(ptent)) { 527 clear_young_dirty_ptes(vma, addr, pte, nr, 528 CYDP_CLEAR_YOUNG); 529 tlb_remove_tlb_entries(tlb, pte, nr, addr); 530 } 531 532 /* 533 * We are deactivating a folio for accelerating reclaiming. 534 * VM couldn't reclaim the folio unless we clear PG_young. 535 * As a side effect, it makes confuse idle-page tracking 536 * because they will miss recent referenced history. 537 */ 538 folio_clear_referenced(folio); 539 folio_test_clear_young(folio); 540 if (folio_test_active(folio)) 541 folio_set_workingset(folio); 542 if (pageout) { 543 if (folio_isolate_lru(folio)) { 544 if (folio_test_unevictable(folio)) 545 folio_putback_lru(folio); 546 else 547 list_add(&folio->lru, &folio_list); 548 } 549 } else 550 folio_deactivate(folio); 551 } 552 553 if (start_pte) { 554 arch_leave_lazy_mmu_mode(); 555 pte_unmap_unlock(start_pte, ptl); 556 } 557 if (pageout) 558 reclaim_pages(&folio_list); 559 cond_resched(); 560 561 return 0; 562 } 563 564 static const struct mm_walk_ops cold_walk_ops = { 565 .pmd_entry = madvise_cold_or_pageout_pte_range, 566 .walk_lock = PGWALK_RDLOCK, 567 }; 568 569 static void madvise_cold_page_range(struct mmu_gather *tlb, 570 struct vm_area_struct *vma, 571 unsigned long addr, unsigned long end) 572 { 573 struct madvise_walk_private walk_private = { 574 .pageout = false, 575 .tlb = tlb, 576 }; 577 578 tlb_start_vma(tlb, vma); 579 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); 580 tlb_end_vma(tlb, vma); 581 } 582 583 static inline bool can_madv_lru_vma(struct vm_area_struct *vma) 584 { 585 return !(vma->vm_flags & (VM_LOCKED|VM_PFNMAP|VM_HUGETLB)); 586 } 587 588 static long madvise_cold(struct vm_area_struct *vma, 589 struct vm_area_struct **prev, 590 unsigned long start_addr, unsigned long end_addr) 591 { 592 struct mm_struct *mm = vma->vm_mm; 593 struct mmu_gather tlb; 594 595 *prev = vma; 596 if (!can_madv_lru_vma(vma)) 597 return -EINVAL; 598 599 lru_add_drain(); 600 tlb_gather_mmu(&tlb, mm); 601 madvise_cold_page_range(&tlb, vma, start_addr, end_addr); 602 tlb_finish_mmu(&tlb); 603 604 return 0; 605 } 606 607 static void madvise_pageout_page_range(struct mmu_gather *tlb, 608 struct vm_area_struct *vma, 609 unsigned long addr, unsigned long end) 610 { 611 struct madvise_walk_private walk_private = { 612 .pageout = true, 613 .tlb = tlb, 614 }; 615 616 tlb_start_vma(tlb, vma); 617 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); 618 tlb_end_vma(tlb, vma); 619 } 620 621 static long madvise_pageout(struct vm_area_struct *vma, 622 struct vm_area_struct **prev, 623 unsigned long start_addr, unsigned long end_addr) 624 { 625 struct mm_struct *mm = vma->vm_mm; 626 struct mmu_gather tlb; 627 628 *prev = vma; 629 if (!can_madv_lru_vma(vma)) 630 return -EINVAL; 631 632 /* 633 * If the VMA belongs to a private file mapping, there can be private 634 * dirty pages which can be paged out if even this process is neither 635 * owner nor write capable of the file. We allow private file mappings 636 * further to pageout dirty anon pages. 637 */ 638 if (!vma_is_anonymous(vma) && (!can_do_file_pageout(vma) && 639 (vma->vm_flags & VM_MAYSHARE))) 640 return 0; 641 642 lru_add_drain(); 643 tlb_gather_mmu(&tlb, mm); 644 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); 645 tlb_finish_mmu(&tlb); 646 647 return 0; 648 } 649 650 static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, 651 unsigned long end, struct mm_walk *walk) 652 653 { 654 const cydp_t cydp_flags = CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY; 655 struct mmu_gather *tlb = walk->private; 656 struct mm_struct *mm = tlb->mm; 657 struct vm_area_struct *vma = walk->vma; 658 spinlock_t *ptl; 659 pte_t *start_pte, *pte, ptent; 660 struct folio *folio; 661 int nr_swap = 0; 662 unsigned long next; 663 int nr, max_nr; 664 665 next = pmd_addr_end(addr, end); 666 if (pmd_trans_huge(*pmd)) 667 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) 668 return 0; 669 670 tlb_change_page_size(tlb, PAGE_SIZE); 671 start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 672 if (!start_pte) 673 return 0; 674 flush_tlb_batched_pending(mm); 675 arch_enter_lazy_mmu_mode(); 676 for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) { 677 nr = 1; 678 ptent = ptep_get(pte); 679 680 if (pte_none(ptent)) 681 continue; 682 /* 683 * If the pte has swp_entry, just clear page table to 684 * prevent swap-in which is more expensive rather than 685 * (page allocation + zeroing). 686 */ 687 if (!pte_present(ptent)) { 688 swp_entry_t entry; 689 690 entry = pte_to_swp_entry(ptent); 691 if (!non_swap_entry(entry)) { 692 max_nr = (end - addr) / PAGE_SIZE; 693 nr = swap_pte_batch(pte, max_nr, ptent); 694 nr_swap -= nr; 695 free_swap_and_cache_nr(entry, nr); 696 clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); 697 } else if (is_hwpoison_entry(entry) || 698 is_poisoned_swp_entry(entry)) { 699 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 700 } 701 continue; 702 } 703 704 folio = vm_normal_folio(vma, addr, ptent); 705 if (!folio || folio_is_zone_device(folio)) 706 continue; 707 708 /* 709 * If we encounter a large folio, only split it if it is not 710 * fully mapped within the range we are operating on. Otherwise 711 * leave it as is so that it can be marked as lazyfree. If we 712 * fail to split a folio, leave it in place and advance to the 713 * next pte in the range. 714 */ 715 if (folio_test_large(folio)) { 716 bool any_young, any_dirty; 717 718 nr = madvise_folio_pte_batch(addr, end, folio, pte, 719 ptent, &any_young, &any_dirty); 720 721 if (nr < folio_nr_pages(folio)) { 722 int err; 723 724 if (folio_likely_mapped_shared(folio)) 725 continue; 726 if (!folio_trylock(folio)) 727 continue; 728 folio_get(folio); 729 arch_leave_lazy_mmu_mode(); 730 pte_unmap_unlock(start_pte, ptl); 731 start_pte = NULL; 732 err = split_folio(folio); 733 folio_unlock(folio); 734 folio_put(folio); 735 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 736 start_pte = pte; 737 if (!start_pte) 738 break; 739 arch_enter_lazy_mmu_mode(); 740 if (!err) 741 nr = 0; 742 continue; 743 } 744 745 if (any_young) 746 ptent = pte_mkyoung(ptent); 747 if (any_dirty) 748 ptent = pte_mkdirty(ptent); 749 } 750 751 if (folio_test_swapcache(folio) || folio_test_dirty(folio)) { 752 if (!folio_trylock(folio)) 753 continue; 754 /* 755 * If we have a large folio at this point, we know it is 756 * fully mapped so if its mapcount is the same as its 757 * number of pages, it must be exclusive. 758 */ 759 if (folio_mapcount(folio) != folio_nr_pages(folio)) { 760 folio_unlock(folio); 761 continue; 762 } 763 764 if (folio_test_swapcache(folio) && 765 !folio_free_swap(folio)) { 766 folio_unlock(folio); 767 continue; 768 } 769 770 folio_clear_dirty(folio); 771 folio_unlock(folio); 772 } 773 774 if (pte_young(ptent) || pte_dirty(ptent)) { 775 clear_young_dirty_ptes(vma, addr, pte, nr, cydp_flags); 776 tlb_remove_tlb_entries(tlb, pte, nr, addr); 777 } 778 folio_mark_lazyfree(folio); 779 } 780 781 if (nr_swap) 782 add_mm_counter(mm, MM_SWAPENTS, nr_swap); 783 if (start_pte) { 784 arch_leave_lazy_mmu_mode(); 785 pte_unmap_unlock(start_pte, ptl); 786 } 787 cond_resched(); 788 789 return 0; 790 } 791 792 static const struct mm_walk_ops madvise_free_walk_ops = { 793 .pmd_entry = madvise_free_pte_range, 794 .walk_lock = PGWALK_RDLOCK, 795 }; 796 797 static int madvise_free_single_vma(struct vm_area_struct *vma, 798 unsigned long start_addr, unsigned long end_addr) 799 { 800 struct mm_struct *mm = vma->vm_mm; 801 struct mmu_notifier_range range; 802 struct mmu_gather tlb; 803 804 /* MADV_FREE works for only anon vma at the moment */ 805 if (!vma_is_anonymous(vma)) 806 return -EINVAL; 807 808 range.start = max(vma->vm_start, start_addr); 809 if (range.start >= vma->vm_end) 810 return -EINVAL; 811 range.end = min(vma->vm_end, end_addr); 812 if (range.end <= vma->vm_start) 813 return -EINVAL; 814 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, 815 range.start, range.end); 816 817 lru_add_drain(); 818 tlb_gather_mmu(&tlb, mm); 819 update_hiwater_rss(mm); 820 821 mmu_notifier_invalidate_range_start(&range); 822 tlb_start_vma(&tlb, vma); 823 walk_page_range(vma->vm_mm, range.start, range.end, 824 &madvise_free_walk_ops, &tlb); 825 tlb_end_vma(&tlb, vma); 826 mmu_notifier_invalidate_range_end(&range); 827 tlb_finish_mmu(&tlb); 828 829 return 0; 830 } 831 832 /* 833 * Application no longer needs these pages. If the pages are dirty, 834 * it's OK to just throw them away. The app will be more careful about 835 * data it wants to keep. Be sure to free swap resources too. The 836 * zap_page_range_single call sets things up for shrink_active_list to actually 837 * free these pages later if no one else has touched them in the meantime, 838 * although we could add these pages to a global reuse list for 839 * shrink_active_list to pick up before reclaiming other pages. 840 * 841 * NB: This interface discards data rather than pushes it out to swap, 842 * as some implementations do. This has performance implications for 843 * applications like large transactional databases which want to discard 844 * pages in anonymous maps after committing to backing store the data 845 * that was kept in them. There is no reason to write this data out to 846 * the swap area if the application is discarding it. 847 * 848 * An interface that causes the system to free clean pages and flush 849 * dirty pages is already available as msync(MS_INVALIDATE). 850 */ 851 static long madvise_dontneed_single_vma(struct vm_area_struct *vma, 852 unsigned long start, unsigned long end) 853 { 854 struct zap_details details = { 855 .reclaim_pt = true, 856 .even_cows = true, 857 }; 858 859 zap_page_range_single(vma, start, end - start, &details); 860 return 0; 861 } 862 863 static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma, 864 unsigned long start, 865 unsigned long *end, 866 int behavior) 867 { 868 if (!is_vm_hugetlb_page(vma)) { 869 unsigned int forbidden = VM_PFNMAP; 870 871 if (behavior != MADV_DONTNEED_LOCKED) 872 forbidden |= VM_LOCKED; 873 874 return !(vma->vm_flags & forbidden); 875 } 876 877 if (behavior != MADV_DONTNEED && behavior != MADV_DONTNEED_LOCKED) 878 return false; 879 if (start & ~huge_page_mask(hstate_vma(vma))) 880 return false; 881 882 /* 883 * Madvise callers expect the length to be rounded up to PAGE_SIZE 884 * boundaries, and may be unaware that this VMA uses huge pages. 885 * Avoid unexpected data loss by rounding down the number of 886 * huge pages freed. 887 */ 888 *end = ALIGN_DOWN(*end, huge_page_size(hstate_vma(vma))); 889 890 return true; 891 } 892 893 static long madvise_dontneed_free(struct vm_area_struct *vma, 894 struct vm_area_struct **prev, 895 unsigned long start, unsigned long end, 896 int behavior) 897 { 898 struct mm_struct *mm = vma->vm_mm; 899 900 *prev = vma; 901 if (!madvise_dontneed_free_valid_vma(vma, start, &end, behavior)) 902 return -EINVAL; 903 904 if (start == end) 905 return 0; 906 907 if (!userfaultfd_remove(vma, start, end)) { 908 *prev = NULL; /* mmap_lock has been dropped, prev is stale */ 909 910 mmap_read_lock(mm); 911 vma = vma_lookup(mm, start); 912 if (!vma) 913 return -ENOMEM; 914 /* 915 * Potential end adjustment for hugetlb vma is OK as 916 * the check below keeps end within vma. 917 */ 918 if (!madvise_dontneed_free_valid_vma(vma, start, &end, 919 behavior)) 920 return -EINVAL; 921 if (end > vma->vm_end) { 922 /* 923 * Don't fail if end > vma->vm_end. If the old 924 * vma was split while the mmap_lock was 925 * released the effect of the concurrent 926 * operation may not cause madvise() to 927 * have an undefined result. There may be an 928 * adjacent next vma that we'll walk 929 * next. userfaultfd_remove() will generate an 930 * UFFD_EVENT_REMOVE repetition on the 931 * end-vma->vm_end range, but the manager can 932 * handle a repetition fine. 933 */ 934 end = vma->vm_end; 935 } 936 VM_WARN_ON(start >= end); 937 } 938 939 if (behavior == MADV_DONTNEED || behavior == MADV_DONTNEED_LOCKED) 940 return madvise_dontneed_single_vma(vma, start, end); 941 else if (behavior == MADV_FREE) 942 return madvise_free_single_vma(vma, start, end); 943 else 944 return -EINVAL; 945 } 946 947 static long madvise_populate(struct mm_struct *mm, unsigned long start, 948 unsigned long end, int behavior) 949 { 950 const bool write = behavior == MADV_POPULATE_WRITE; 951 int locked = 1; 952 long pages; 953 954 while (start < end) { 955 /* Populate (prefault) page tables readable/writable. */ 956 pages = faultin_page_range(mm, start, end, write, &locked); 957 if (!locked) { 958 mmap_read_lock(mm); 959 locked = 1; 960 } 961 if (pages < 0) { 962 switch (pages) { 963 case -EINTR: 964 return -EINTR; 965 case -EINVAL: /* Incompatible mappings / permissions. */ 966 return -EINVAL; 967 case -EHWPOISON: 968 return -EHWPOISON; 969 case -EFAULT: /* VM_FAULT_SIGBUS or VM_FAULT_SIGSEGV */ 970 return -EFAULT; 971 default: 972 pr_warn_once("%s: unhandled return value: %ld\n", 973 __func__, pages); 974 fallthrough; 975 case -ENOMEM: /* No VMA or out of memory. */ 976 return -ENOMEM; 977 } 978 } 979 start += pages * PAGE_SIZE; 980 } 981 return 0; 982 } 983 984 /* 985 * Application wants to free up the pages and associated backing store. 986 * This is effectively punching a hole into the middle of a file. 987 */ 988 static long madvise_remove(struct vm_area_struct *vma, 989 struct vm_area_struct **prev, 990 unsigned long start, unsigned long end) 991 { 992 loff_t offset; 993 int error; 994 struct file *f; 995 struct mm_struct *mm = vma->vm_mm; 996 997 *prev = NULL; /* tell sys_madvise we drop mmap_lock */ 998 999 if (vma->vm_flags & VM_LOCKED) 1000 return -EINVAL; 1001 1002 f = vma->vm_file; 1003 1004 if (!f || !f->f_mapping || !f->f_mapping->host) { 1005 return -EINVAL; 1006 } 1007 1008 if (!vma_is_shared_maywrite(vma)) 1009 return -EACCES; 1010 1011 offset = (loff_t)(start - vma->vm_start) 1012 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 1013 1014 /* 1015 * Filesystem's fallocate may need to take i_rwsem. We need to 1016 * explicitly grab a reference because the vma (and hence the 1017 * vma's reference to the file) can go away as soon as we drop 1018 * mmap_lock. 1019 */ 1020 get_file(f); 1021 if (userfaultfd_remove(vma, start, end)) { 1022 /* mmap_lock was not released by userfaultfd_remove() */ 1023 mmap_read_unlock(mm); 1024 } 1025 error = vfs_fallocate(f, 1026 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 1027 offset, end - start); 1028 fput(f); 1029 mmap_read_lock(mm); 1030 return error; 1031 } 1032 1033 static bool is_valid_guard_vma(struct vm_area_struct *vma, bool allow_locked) 1034 { 1035 vm_flags_t disallowed = VM_SPECIAL | VM_HUGETLB; 1036 1037 /* 1038 * A user could lock after setting a guard range but that's fine, as 1039 * they'd not be able to fault in. The issue arises when we try to zap 1040 * existing locked VMAs. We don't want to do that. 1041 */ 1042 if (!allow_locked) 1043 disallowed |= VM_LOCKED; 1044 1045 if (!vma_is_anonymous(vma)) 1046 return false; 1047 1048 if ((vma->vm_flags & (VM_MAYWRITE | disallowed)) != VM_MAYWRITE) 1049 return false; 1050 1051 return true; 1052 } 1053 1054 static bool is_guard_pte_marker(pte_t ptent) 1055 { 1056 return is_pte_marker(ptent) && 1057 is_guard_swp_entry(pte_to_swp_entry(ptent)); 1058 } 1059 1060 static int guard_install_pud_entry(pud_t *pud, unsigned long addr, 1061 unsigned long next, struct mm_walk *walk) 1062 { 1063 pud_t pudval = pudp_get(pud); 1064 1065 /* If huge return >0 so we abort the operation + zap. */ 1066 return pud_trans_huge(pudval) || pud_devmap(pudval); 1067 } 1068 1069 static int guard_install_pmd_entry(pmd_t *pmd, unsigned long addr, 1070 unsigned long next, struct mm_walk *walk) 1071 { 1072 pmd_t pmdval = pmdp_get(pmd); 1073 1074 /* If huge return >0 so we abort the operation + zap. */ 1075 return pmd_trans_huge(pmdval) || pmd_devmap(pmdval); 1076 } 1077 1078 static int guard_install_pte_entry(pte_t *pte, unsigned long addr, 1079 unsigned long next, struct mm_walk *walk) 1080 { 1081 pte_t pteval = ptep_get(pte); 1082 unsigned long *nr_pages = (unsigned long *)walk->private; 1083 1084 /* If there is already a guard page marker, we have nothing to do. */ 1085 if (is_guard_pte_marker(pteval)) { 1086 (*nr_pages)++; 1087 1088 return 0; 1089 } 1090 1091 /* If populated return >0 so we abort the operation + zap. */ 1092 return 1; 1093 } 1094 1095 static int guard_install_set_pte(unsigned long addr, unsigned long next, 1096 pte_t *ptep, struct mm_walk *walk) 1097 { 1098 unsigned long *nr_pages = (unsigned long *)walk->private; 1099 1100 /* Simply install a PTE marker, this causes segfault on access. */ 1101 *ptep = make_pte_marker(PTE_MARKER_GUARD); 1102 (*nr_pages)++; 1103 1104 return 0; 1105 } 1106 1107 static const struct mm_walk_ops guard_install_walk_ops = { 1108 .pud_entry = guard_install_pud_entry, 1109 .pmd_entry = guard_install_pmd_entry, 1110 .pte_entry = guard_install_pte_entry, 1111 .install_pte = guard_install_set_pte, 1112 .walk_lock = PGWALK_RDLOCK, 1113 }; 1114 1115 static long madvise_guard_install(struct vm_area_struct *vma, 1116 struct vm_area_struct **prev, 1117 unsigned long start, unsigned long end) 1118 { 1119 long err; 1120 int i; 1121 1122 *prev = vma; 1123 if (!is_valid_guard_vma(vma, /* allow_locked = */false)) 1124 return -EINVAL; 1125 1126 /* 1127 * If we install guard markers, then the range is no longer 1128 * empty from a page table perspective and therefore it's 1129 * appropriate to have an anon_vma. 1130 * 1131 * This ensures that on fork, we copy page tables correctly. 1132 */ 1133 err = anon_vma_prepare(vma); 1134 if (err) 1135 return err; 1136 1137 /* 1138 * Optimistically try to install the guard marker pages first. If any 1139 * non-guard pages are encountered, give up and zap the range before 1140 * trying again. 1141 * 1142 * We try a few times before giving up and releasing back to userland to 1143 * loop around, releasing locks in the process to avoid contention. This 1144 * would only happen if there was a great many racing page faults. 1145 * 1146 * In most cases we should simply install the guard markers immediately 1147 * with no zap or looping. 1148 */ 1149 for (i = 0; i < MAX_MADVISE_GUARD_RETRIES; i++) { 1150 unsigned long nr_pages = 0; 1151 1152 /* Returns < 0 on error, == 0 if success, > 0 if zap needed. */ 1153 err = walk_page_range_mm(vma->vm_mm, start, end, 1154 &guard_install_walk_ops, &nr_pages); 1155 if (err < 0) 1156 return err; 1157 1158 if (err == 0) { 1159 unsigned long nr_expected_pages = PHYS_PFN(end - start); 1160 1161 VM_WARN_ON(nr_pages != nr_expected_pages); 1162 return 0; 1163 } 1164 1165 /* 1166 * OK some of the range have non-guard pages mapped, zap 1167 * them. This leaves existing guard pages in place. 1168 */ 1169 zap_page_range_single(vma, start, end - start, NULL); 1170 } 1171 1172 /* 1173 * We were unable to install the guard pages due to being raced by page 1174 * faults. This should not happen ordinarily. We return to userspace and 1175 * immediately retry, relieving lock contention. 1176 */ 1177 return restart_syscall(); 1178 } 1179 1180 static int guard_remove_pud_entry(pud_t *pud, unsigned long addr, 1181 unsigned long next, struct mm_walk *walk) 1182 { 1183 pud_t pudval = pudp_get(pud); 1184 1185 /* If huge, cannot have guard pages present, so no-op - skip. */ 1186 if (pud_trans_huge(pudval) || pud_devmap(pudval)) 1187 walk->action = ACTION_CONTINUE; 1188 1189 return 0; 1190 } 1191 1192 static int guard_remove_pmd_entry(pmd_t *pmd, unsigned long addr, 1193 unsigned long next, struct mm_walk *walk) 1194 { 1195 pmd_t pmdval = pmdp_get(pmd); 1196 1197 /* If huge, cannot have guard pages present, so no-op - skip. */ 1198 if (pmd_trans_huge(pmdval) || pmd_devmap(pmdval)) 1199 walk->action = ACTION_CONTINUE; 1200 1201 return 0; 1202 } 1203 1204 static int guard_remove_pte_entry(pte_t *pte, unsigned long addr, 1205 unsigned long next, struct mm_walk *walk) 1206 { 1207 pte_t ptent = ptep_get(pte); 1208 1209 if (is_guard_pte_marker(ptent)) { 1210 /* Simply clear the PTE marker. */ 1211 pte_clear_not_present_full(walk->mm, addr, pte, false); 1212 update_mmu_cache(walk->vma, addr, pte); 1213 } 1214 1215 return 0; 1216 } 1217 1218 static const struct mm_walk_ops guard_remove_walk_ops = { 1219 .pud_entry = guard_remove_pud_entry, 1220 .pmd_entry = guard_remove_pmd_entry, 1221 .pte_entry = guard_remove_pte_entry, 1222 .walk_lock = PGWALK_RDLOCK, 1223 }; 1224 1225 static long madvise_guard_remove(struct vm_area_struct *vma, 1226 struct vm_area_struct **prev, 1227 unsigned long start, unsigned long end) 1228 { 1229 *prev = vma; 1230 /* 1231 * We're ok with removing guards in mlock()'d ranges, as this is a 1232 * non-destructive action. 1233 */ 1234 if (!is_valid_guard_vma(vma, /* allow_locked = */true)) 1235 return -EINVAL; 1236 1237 return walk_page_range(vma->vm_mm, start, end, 1238 &guard_remove_walk_ops, NULL); 1239 } 1240 1241 /* 1242 * Apply an madvise behavior to a region of a vma. madvise_update_vma 1243 * will handle splitting a vm area into separate areas, each area with its own 1244 * behavior. 1245 */ 1246 static int madvise_vma_behavior(struct vm_area_struct *vma, 1247 struct vm_area_struct **prev, 1248 unsigned long start, unsigned long end, 1249 unsigned long behavior) 1250 { 1251 int error; 1252 struct anon_vma_name *anon_name; 1253 unsigned long new_flags = vma->vm_flags; 1254 1255 if (unlikely(!can_modify_vma_madv(vma, behavior))) 1256 return -EPERM; 1257 1258 switch (behavior) { 1259 case MADV_REMOVE: 1260 return madvise_remove(vma, prev, start, end); 1261 case MADV_WILLNEED: 1262 return madvise_willneed(vma, prev, start, end); 1263 case MADV_COLD: 1264 return madvise_cold(vma, prev, start, end); 1265 case MADV_PAGEOUT: 1266 return madvise_pageout(vma, prev, start, end); 1267 case MADV_FREE: 1268 case MADV_DONTNEED: 1269 case MADV_DONTNEED_LOCKED: 1270 return madvise_dontneed_free(vma, prev, start, end, behavior); 1271 case MADV_NORMAL: 1272 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ; 1273 break; 1274 case MADV_SEQUENTIAL: 1275 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ; 1276 break; 1277 case MADV_RANDOM: 1278 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ; 1279 break; 1280 case MADV_DONTFORK: 1281 new_flags |= VM_DONTCOPY; 1282 break; 1283 case MADV_DOFORK: 1284 if (vma->vm_flags & VM_IO) 1285 return -EINVAL; 1286 new_flags &= ~VM_DONTCOPY; 1287 break; 1288 case MADV_WIPEONFORK: 1289 /* MADV_WIPEONFORK is only supported on anonymous memory. */ 1290 if (vma->vm_file || vma->vm_flags & VM_SHARED) 1291 return -EINVAL; 1292 new_flags |= VM_WIPEONFORK; 1293 break; 1294 case MADV_KEEPONFORK: 1295 if (vma->vm_flags & VM_DROPPABLE) 1296 return -EINVAL; 1297 new_flags &= ~VM_WIPEONFORK; 1298 break; 1299 case MADV_DONTDUMP: 1300 new_flags |= VM_DONTDUMP; 1301 break; 1302 case MADV_DODUMP: 1303 if ((!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) || 1304 (vma->vm_flags & VM_DROPPABLE)) 1305 return -EINVAL; 1306 new_flags &= ~VM_DONTDUMP; 1307 break; 1308 case MADV_MERGEABLE: 1309 case MADV_UNMERGEABLE: 1310 error = ksm_madvise(vma, start, end, behavior, &new_flags); 1311 if (error) 1312 goto out; 1313 break; 1314 case MADV_HUGEPAGE: 1315 case MADV_NOHUGEPAGE: 1316 error = hugepage_madvise(vma, &new_flags, behavior); 1317 if (error) 1318 goto out; 1319 break; 1320 case MADV_COLLAPSE: 1321 return madvise_collapse(vma, prev, start, end); 1322 case MADV_GUARD_INSTALL: 1323 return madvise_guard_install(vma, prev, start, end); 1324 case MADV_GUARD_REMOVE: 1325 return madvise_guard_remove(vma, prev, start, end); 1326 } 1327 1328 anon_name = anon_vma_name(vma); 1329 anon_vma_name_get(anon_name); 1330 error = madvise_update_vma(vma, prev, start, end, new_flags, 1331 anon_name); 1332 anon_vma_name_put(anon_name); 1333 1334 out: 1335 /* 1336 * madvise() returns EAGAIN if kernel resources, such as 1337 * slab, are temporarily unavailable. 1338 */ 1339 if (error == -ENOMEM) 1340 error = -EAGAIN; 1341 return error; 1342 } 1343 1344 #ifdef CONFIG_MEMORY_FAILURE 1345 /* 1346 * Error injection support for memory error handling. 1347 */ 1348 static int madvise_inject_error(int behavior, 1349 unsigned long start, unsigned long end) 1350 { 1351 unsigned long size; 1352 1353 if (!capable(CAP_SYS_ADMIN)) 1354 return -EPERM; 1355 1356 1357 for (; start < end; start += size) { 1358 unsigned long pfn; 1359 struct page *page; 1360 int ret; 1361 1362 ret = get_user_pages_fast(start, 1, 0, &page); 1363 if (ret != 1) 1364 return ret; 1365 pfn = page_to_pfn(page); 1366 1367 /* 1368 * When soft offlining hugepages, after migrating the page 1369 * we dissolve it, therefore in the second loop "page" will 1370 * no longer be a compound page. 1371 */ 1372 size = page_size(compound_head(page)); 1373 1374 if (behavior == MADV_SOFT_OFFLINE) { 1375 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n", 1376 pfn, start); 1377 ret = soft_offline_page(pfn, MF_COUNT_INCREASED); 1378 } else { 1379 pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n", 1380 pfn, start); 1381 ret = memory_failure(pfn, MF_ACTION_REQUIRED | MF_COUNT_INCREASED | MF_SW_SIMULATED); 1382 if (ret == -EOPNOTSUPP) 1383 ret = 0; 1384 } 1385 1386 if (ret) 1387 return ret; 1388 } 1389 1390 return 0; 1391 } 1392 #endif 1393 1394 static bool 1395 madvise_behavior_valid(int behavior) 1396 { 1397 switch (behavior) { 1398 case MADV_DOFORK: 1399 case MADV_DONTFORK: 1400 case MADV_NORMAL: 1401 case MADV_SEQUENTIAL: 1402 case MADV_RANDOM: 1403 case MADV_REMOVE: 1404 case MADV_WILLNEED: 1405 case MADV_DONTNEED: 1406 case MADV_DONTNEED_LOCKED: 1407 case MADV_FREE: 1408 case MADV_COLD: 1409 case MADV_PAGEOUT: 1410 case MADV_POPULATE_READ: 1411 case MADV_POPULATE_WRITE: 1412 #ifdef CONFIG_KSM 1413 case MADV_MERGEABLE: 1414 case MADV_UNMERGEABLE: 1415 #endif 1416 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1417 case MADV_HUGEPAGE: 1418 case MADV_NOHUGEPAGE: 1419 case MADV_COLLAPSE: 1420 #endif 1421 case MADV_DONTDUMP: 1422 case MADV_DODUMP: 1423 case MADV_WIPEONFORK: 1424 case MADV_KEEPONFORK: 1425 case MADV_GUARD_INSTALL: 1426 case MADV_GUARD_REMOVE: 1427 #ifdef CONFIG_MEMORY_FAILURE 1428 case MADV_SOFT_OFFLINE: 1429 case MADV_HWPOISON: 1430 #endif 1431 return true; 1432 1433 default: 1434 return false; 1435 } 1436 } 1437 1438 /* Can we invoke process_madvise() on a remote mm for the specified behavior? */ 1439 static bool process_madvise_remote_valid(int behavior) 1440 { 1441 switch (behavior) { 1442 case MADV_COLD: 1443 case MADV_PAGEOUT: 1444 case MADV_WILLNEED: 1445 case MADV_COLLAPSE: 1446 return true; 1447 default: 1448 return false; 1449 } 1450 } 1451 1452 /* 1453 * Walk the vmas in range [start,end), and call the visit function on each one. 1454 * The visit function will get start and end parameters that cover the overlap 1455 * between the current vma and the original range. Any unmapped regions in the 1456 * original range will result in this function returning -ENOMEM while still 1457 * calling the visit function on all of the existing vmas in the range. 1458 * Must be called with the mmap_lock held for reading or writing. 1459 */ 1460 static 1461 int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, 1462 unsigned long end, unsigned long arg, 1463 int (*visit)(struct vm_area_struct *vma, 1464 struct vm_area_struct **prev, unsigned long start, 1465 unsigned long end, unsigned long arg)) 1466 { 1467 struct vm_area_struct *vma; 1468 struct vm_area_struct *prev; 1469 unsigned long tmp; 1470 int unmapped_error = 0; 1471 1472 /* 1473 * If the interval [start,end) covers some unmapped address 1474 * ranges, just ignore them, but return -ENOMEM at the end. 1475 * - different from the way of handling in mlock etc. 1476 */ 1477 vma = find_vma_prev(mm, start, &prev); 1478 if (vma && start > vma->vm_start) 1479 prev = vma; 1480 1481 for (;;) { 1482 int error; 1483 1484 /* Still start < end. */ 1485 if (!vma) 1486 return -ENOMEM; 1487 1488 /* Here start < (end|vma->vm_end). */ 1489 if (start < vma->vm_start) { 1490 unmapped_error = -ENOMEM; 1491 start = vma->vm_start; 1492 if (start >= end) 1493 break; 1494 } 1495 1496 /* Here vma->vm_start <= start < (end|vma->vm_end) */ 1497 tmp = vma->vm_end; 1498 if (end < tmp) 1499 tmp = end; 1500 1501 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ 1502 error = visit(vma, &prev, start, tmp, arg); 1503 if (error) 1504 return error; 1505 start = tmp; 1506 if (prev && start < prev->vm_end) 1507 start = prev->vm_end; 1508 if (start >= end) 1509 break; 1510 if (prev) 1511 vma = find_vma(mm, prev->vm_end); 1512 else /* madvise_remove dropped mmap_lock */ 1513 vma = find_vma(mm, start); 1514 } 1515 1516 return unmapped_error; 1517 } 1518 1519 #ifdef CONFIG_ANON_VMA_NAME 1520 static int madvise_vma_anon_name(struct vm_area_struct *vma, 1521 struct vm_area_struct **prev, 1522 unsigned long start, unsigned long end, 1523 unsigned long anon_name) 1524 { 1525 int error; 1526 1527 /* Only anonymous mappings can be named */ 1528 if (vma->vm_file && !vma_is_anon_shmem(vma)) 1529 return -EBADF; 1530 1531 error = madvise_update_vma(vma, prev, start, end, vma->vm_flags, 1532 (struct anon_vma_name *)anon_name); 1533 1534 /* 1535 * madvise() returns EAGAIN if kernel resources, such as 1536 * slab, are temporarily unavailable. 1537 */ 1538 if (error == -ENOMEM) 1539 error = -EAGAIN; 1540 return error; 1541 } 1542 1543 int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, 1544 unsigned long len_in, struct anon_vma_name *anon_name) 1545 { 1546 unsigned long end; 1547 unsigned long len; 1548 1549 if (start & ~PAGE_MASK) 1550 return -EINVAL; 1551 len = (len_in + ~PAGE_MASK) & PAGE_MASK; 1552 1553 /* Check to see whether len was rounded up from small -ve to zero */ 1554 if (len_in && !len) 1555 return -EINVAL; 1556 1557 end = start + len; 1558 if (end < start) 1559 return -EINVAL; 1560 1561 if (end == start) 1562 return 0; 1563 1564 return madvise_walk_vmas(mm, start, end, (unsigned long)anon_name, 1565 madvise_vma_anon_name); 1566 } 1567 #endif /* CONFIG_ANON_VMA_NAME */ 1568 /* 1569 * The madvise(2) system call. 1570 * 1571 * Applications can use madvise() to advise the kernel how it should 1572 * handle paging I/O in this VM area. The idea is to help the kernel 1573 * use appropriate read-ahead and caching techniques. The information 1574 * provided is advisory only, and can be safely disregarded by the 1575 * kernel without affecting the correct operation of the application. 1576 * 1577 * behavior values: 1578 * MADV_NORMAL - the default behavior is to read clusters. This 1579 * results in some read-ahead and read-behind. 1580 * MADV_RANDOM - the system should read the minimum amount of data 1581 * on any access, since it is unlikely that the appli- 1582 * cation will need more than what it asks for. 1583 * MADV_SEQUENTIAL - pages in the given range will probably be accessed 1584 * once, so they can be aggressively read ahead, and 1585 * can be freed soon after they are accessed. 1586 * MADV_WILLNEED - the application is notifying the system to read 1587 * some pages ahead. 1588 * MADV_DONTNEED - the application is finished with the given range, 1589 * so the kernel can free resources associated with it. 1590 * MADV_FREE - the application marks pages in the given range as lazy free, 1591 * where actual purges are postponed until memory pressure happens. 1592 * MADV_REMOVE - the application wants to free up the given range of 1593 * pages and associated backing store. 1594 * MADV_DONTFORK - omit this area from child's address space when forking: 1595 * typically, to avoid COWing pages pinned by get_user_pages(). 1596 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking. 1597 * MADV_WIPEONFORK - present the child process with zero-filled memory in this 1598 * range after a fork. 1599 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK 1600 * MADV_HWPOISON - trigger memory error handler as if the given memory range 1601 * were corrupted by unrecoverable hardware memory failure. 1602 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory. 1603 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in 1604 * this area with pages of identical content from other such areas. 1605 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others. 1606 * MADV_HUGEPAGE - the application wants to back the given range by transparent 1607 * huge pages in the future. Existing pages might be coalesced and 1608 * new pages might be allocated as THP. 1609 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by 1610 * transparent huge pages so the existing pages will not be 1611 * coalesced into THP and new pages will not be allocated as THP. 1612 * MADV_COLLAPSE - synchronously coalesce pages into new THP. 1613 * MADV_DONTDUMP - the application wants to prevent pages in the given range 1614 * from being included in its core dump. 1615 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. 1616 * MADV_COLD - the application is not expected to use this memory soon, 1617 * deactivate pages in this range so that they can be reclaimed 1618 * easily if memory pressure happens. 1619 * MADV_PAGEOUT - the application is not expected to use this memory soon, 1620 * page out the pages in this range immediately. 1621 * MADV_POPULATE_READ - populate (prefault) page tables readable by 1622 * triggering read faults if required 1623 * MADV_POPULATE_WRITE - populate (prefault) page tables writable by 1624 * triggering write faults if required 1625 * 1626 * return values: 1627 * zero - success 1628 * -EINVAL - start + len < 0, start is not page-aligned, 1629 * "behavior" is not a valid value, or application 1630 * is attempting to release locked or shared pages, 1631 * or the specified address range includes file, Huge TLB, 1632 * MAP_SHARED or VMPFNMAP range. 1633 * -ENOMEM - addresses in the specified range are not currently 1634 * mapped, or are outside the AS of the process. 1635 * -EIO - an I/O error occurred while paging in data. 1636 * -EBADF - map exists, but area maps something that isn't a file. 1637 * -EAGAIN - a kernel resource was temporarily unavailable. 1638 * -EPERM - memory is sealed. 1639 */ 1640 int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior) 1641 { 1642 unsigned long end; 1643 int error; 1644 int write; 1645 size_t len; 1646 struct blk_plug plug; 1647 1648 if (!madvise_behavior_valid(behavior)) 1649 return -EINVAL; 1650 1651 if (!PAGE_ALIGNED(start)) 1652 return -EINVAL; 1653 len = PAGE_ALIGN(len_in); 1654 1655 /* Check to see whether len was rounded up from small -ve to zero */ 1656 if (len_in && !len) 1657 return -EINVAL; 1658 1659 end = start + len; 1660 if (end < start) 1661 return -EINVAL; 1662 1663 if (end == start) 1664 return 0; 1665 1666 #ifdef CONFIG_MEMORY_FAILURE 1667 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE) 1668 return madvise_inject_error(behavior, start, start + len_in); 1669 #endif 1670 1671 write = madvise_need_mmap_write(behavior); 1672 if (write) { 1673 if (mmap_write_lock_killable(mm)) 1674 return -EINTR; 1675 } else { 1676 mmap_read_lock(mm); 1677 } 1678 1679 start = untagged_addr_remote(mm, start); 1680 end = start + len; 1681 1682 blk_start_plug(&plug); 1683 switch (behavior) { 1684 case MADV_POPULATE_READ: 1685 case MADV_POPULATE_WRITE: 1686 error = madvise_populate(mm, start, end, behavior); 1687 break; 1688 default: 1689 error = madvise_walk_vmas(mm, start, end, behavior, 1690 madvise_vma_behavior); 1691 break; 1692 } 1693 blk_finish_plug(&plug); 1694 1695 if (write) 1696 mmap_write_unlock(mm); 1697 else 1698 mmap_read_unlock(mm); 1699 1700 return error; 1701 } 1702 1703 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) 1704 { 1705 return do_madvise(current->mm, start, len_in, behavior); 1706 } 1707 1708 /* Perform an madvise operation over a vector of addresses and lengths. */ 1709 static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, 1710 int behavior) 1711 { 1712 ssize_t ret = 0; 1713 size_t total_len; 1714 1715 total_len = iov_iter_count(iter); 1716 1717 while (iov_iter_count(iter)) { 1718 ret = do_madvise(mm, (unsigned long)iter_iov_addr(iter), 1719 iter_iov_len(iter), behavior); 1720 /* 1721 * An madvise operation is attempting to restart the syscall, 1722 * but we cannot proceed as it would not be correct to repeat 1723 * the operation in aggregate, and would be surprising to the 1724 * user. 1725 * 1726 * As we have already dropped locks, it is safe to just loop and 1727 * try again. We check for fatal signals in case we need exit 1728 * early anyway. 1729 */ 1730 if (ret == -ERESTARTNOINTR) { 1731 if (fatal_signal_pending(current)) { 1732 ret = -EINTR; 1733 break; 1734 } 1735 continue; 1736 } 1737 if (ret < 0) 1738 break; 1739 iov_iter_advance(iter, iter_iov_len(iter)); 1740 } 1741 1742 ret = (total_len - iov_iter_count(iter)) ? : ret; 1743 1744 return ret; 1745 } 1746 1747 SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, 1748 size_t, vlen, int, behavior, unsigned int, flags) 1749 { 1750 ssize_t ret; 1751 struct iovec iovstack[UIO_FASTIOV]; 1752 struct iovec *iov = iovstack; 1753 struct iov_iter iter; 1754 struct task_struct *task; 1755 struct mm_struct *mm; 1756 unsigned int f_flags; 1757 1758 if (flags != 0) { 1759 ret = -EINVAL; 1760 goto out; 1761 } 1762 1763 ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter); 1764 if (ret < 0) 1765 goto out; 1766 1767 task = pidfd_get_task(pidfd, &f_flags); 1768 if (IS_ERR(task)) { 1769 ret = PTR_ERR(task); 1770 goto free_iov; 1771 } 1772 1773 /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */ 1774 mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); 1775 if (IS_ERR(mm)) { 1776 ret = PTR_ERR(mm); 1777 goto release_task; 1778 } 1779 1780 /* 1781 * We need only perform this check if we are attempting to manipulate a 1782 * remote process's address space. 1783 */ 1784 if (mm != current->mm && !process_madvise_remote_valid(behavior)) { 1785 ret = -EINVAL; 1786 goto release_mm; 1787 } 1788 1789 /* 1790 * Require CAP_SYS_NICE for influencing process performance. Note that 1791 * only non-destructive hints are currently supported for remote 1792 * processes. 1793 */ 1794 if (mm != current->mm && !capable(CAP_SYS_NICE)) { 1795 ret = -EPERM; 1796 goto release_mm; 1797 } 1798 1799 ret = vector_madvise(mm, &iter, behavior); 1800 1801 release_mm: 1802 mmput(mm); 1803 release_task: 1804 put_task_struct(task); 1805 free_iov: 1806 kfree(iov); 1807 out: 1808 return ret; 1809 } 1810