Lines Matching +full:tlb +full:- +full:split
1 // SPDX-License-Identifier: GPL-2.0
13 #include <linux/page-isolation.h>
28 #include <linux/backing-dev.h>
35 #include <asm/tlb.h>
41 struct mmu_gather *tlb; member
46 * Any behaviour which results in changes to the vma->vm_flags needs to
76 /* Add 1 for NUL terminator at the end of the anon_name->name */ in anon_vma_name_alloc()
80 kref_init(&anon_name->kref); in anon_vma_name_alloc()
81 memcpy(anon_name->name, name, count); in anon_vma_name_alloc()
96 mmap_assert_locked(vma->vm_mm); in anon_vma_name()
98 return vma->anon_name; in anon_vma_name()
101 /* mmap_lock should be write-locked */
108 vma->anon_name = NULL; in replace_anon_vma_name()
116 vma->anon_name = anon_vma_name_reuse(anon_name); in replace_anon_vma_name()
126 return -EINVAL; in replace_anon_vma_name()
142 struct mm_struct *mm = vma->vm_mm; in madvise_update_vma()
146 if (new_flags == vma->vm_flags && anon_vma_name_eq(anon_vma_name(vma), anon_name)) { in madvise_update_vma()
161 if (!vma->vm_file || vma_is_anon_shmem(vma)) { in madvise_update_vma()
174 struct vm_area_struct *vma = walk->private; in swapin_walk_pmd_entry()
186 ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in swapin_walk_pmd_entry()
224 XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); in shmem_swapin_range()
225 pgoff_t end_index = linear_page_index(vma, end) - 1; in shmem_swapin_range()
241 addr = vma->vm_start + in shmem_swapin_range()
242 ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); in shmem_swapin_range()
265 struct mm_struct *mm = vma->vm_mm; in madvise_willneed()
266 struct file *file = vma->vm_file; in madvise_willneed()
272 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma); in madvise_willneed()
277 if (shmem_mapping(file->f_mapping)) { in madvise_willneed()
278 shmem_swapin_range(vma, start, end, file->f_mapping); in madvise_willneed()
284 return -EBADF; in madvise_willneed()
300 offset = (loff_t)(start - vma->vm_start) in madvise_willneed()
301 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); in madvise_willneed()
303 vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); in madvise_willneed()
311 if (!vma->vm_file) in can_do_file_pageout()
314 * paging out pagecache only for non-anonymous mappings that correspond in can_do_file_pageout()
316 * otherwise we'd be including shared non-exclusive mappings, which in can_do_file_pageout()
320 file_inode(vma->vm_file)) || in can_do_file_pageout()
321 file_permission(vma->vm_file, MAY_WRITE) == 0; in can_do_file_pageout()
328 struct madvise_walk_private *private = walk->private; in madvise_cold_or_pageout_pte_range()
329 struct mmu_gather *tlb = private->tlb; in madvise_cold_or_pageout_pte_range() local
330 bool pageout = private->pageout; in madvise_cold_or_pageout_pte_range()
331 struct mm_struct *mm = tlb->mm; in madvise_cold_or_pageout_pte_range()
332 struct vm_area_struct *vma = walk->vma; in madvise_cold_or_pageout_pte_range()
341 return -EINTR; in madvise_cold_or_pageout_pte_range()
351 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_cold_or_pageout_pte_range()
375 if (next - addr != HPAGE_PMD_SIZE) { in madvise_cold_or_pageout_pte_range()
394 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_cold_or_pageout_pte_range()
406 list_add(&folio->lru, &folio_list); in madvise_cold_or_pageout_pte_range()
419 tlb_change_page_size(tlb, PAGE_SIZE); in madvise_cold_or_pageout_pte_range()
421 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in madvise_cold_or_pageout_pte_range()
450 * Creating a THP page is expensive so split it only if we in madvise_cold_or_pageout_pte_range()
451 * are sure it's worth. Split it if we are only owner. in madvise_cold_or_pageout_pte_range()
476 pte--; in madvise_cold_or_pageout_pte_range()
477 addr -= PAGE_SIZE; in madvise_cold_or_pageout_pte_range()
483 * non-LRU folio. in madvise_cold_or_pageout_pte_range()
495 tlb->fullmm); in madvise_cold_or_pageout_pte_range()
498 tlb_remove_tlb_entry(tlb, pte, addr); in madvise_cold_or_pageout_pte_range()
504 * As a side effect, it makes confuse idle-page tracking in madvise_cold_or_pageout_pte_range()
516 list_add(&folio->lru, &folio_list); in madvise_cold_or_pageout_pte_range()
538 static void madvise_cold_page_range(struct mmu_gather *tlb, in madvise_cold_page_range() argument
544 .tlb = tlb, in madvise_cold_page_range()
547 tlb_start_vma(tlb, vma); in madvise_cold_page_range()
548 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); in madvise_cold_page_range()
549 tlb_end_vma(tlb, vma); in madvise_cold_page_range()
554 return !(vma->vm_flags & (VM_LOCKED|VM_PFNMAP|VM_HUGETLB)); in can_madv_lru_vma()
561 struct mm_struct *mm = vma->vm_mm; in madvise_cold()
562 struct mmu_gather tlb; in madvise_cold() local
566 return -EINVAL; in madvise_cold()
569 tlb_gather_mmu(&tlb, mm); in madvise_cold()
570 madvise_cold_page_range(&tlb, vma, start_addr, end_addr); in madvise_cold()
571 tlb_finish_mmu(&tlb); in madvise_cold()
576 static void madvise_pageout_page_range(struct mmu_gather *tlb, in madvise_pageout_page_range() argument
582 .tlb = tlb, in madvise_pageout_page_range()
585 tlb_start_vma(tlb, vma); in madvise_pageout_page_range()
586 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); in madvise_pageout_page_range()
587 tlb_end_vma(tlb, vma); in madvise_pageout_page_range()
594 struct mm_struct *mm = vma->vm_mm; in madvise_pageout()
595 struct mmu_gather tlb; in madvise_pageout() local
599 return -EINVAL; in madvise_pageout()
608 (vma->vm_flags & VM_MAYSHARE))) in madvise_pageout()
612 tlb_gather_mmu(&tlb, mm); in madvise_pageout()
613 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); in madvise_pageout()
614 tlb_finish_mmu(&tlb); in madvise_pageout()
623 struct mmu_gather *tlb = walk->private; in madvise_free_pte_range() local
624 struct mm_struct *mm = tlb->mm; in madvise_free_pte_range()
625 struct vm_area_struct *vma = walk->vma; in madvise_free_pte_range()
634 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) in madvise_free_pte_range()
637 tlb_change_page_size(tlb, PAGE_SIZE); in madvise_free_pte_range()
650 * prevent swap-in which is more expensive rather than in madvise_free_pte_range()
658 nr_swap--; in madvise_free_pte_range()
660 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); in madvise_free_pte_range()
663 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); in madvise_free_pte_range()
674 * is owned by only this process, split it and in madvise_free_pte_range()
698 pte--; in madvise_free_pte_range()
699 addr -= PAGE_SIZE; in madvise_free_pte_range()
727 * Some of architecture(ex, PPC) don't update TLB in madvise_free_pte_range()
733 tlb->fullmm); in madvise_free_pte_range()
738 tlb_remove_tlb_entry(tlb, pte, addr); in madvise_free_pte_range()
762 struct mm_struct *mm = vma->vm_mm; in madvise_free_single_vma()
764 struct mmu_gather tlb; in madvise_free_single_vma() local
768 return -EINVAL; in madvise_free_single_vma()
770 range.start = max(vma->vm_start, start_addr); in madvise_free_single_vma()
771 if (range.start >= vma->vm_end) in madvise_free_single_vma()
772 return -EINVAL; in madvise_free_single_vma()
773 range.end = min(vma->vm_end, end_addr); in madvise_free_single_vma()
774 if (range.end <= vma->vm_start) in madvise_free_single_vma()
775 return -EINVAL; in madvise_free_single_vma()
780 tlb_gather_mmu(&tlb, mm); in madvise_free_single_vma()
784 tlb_start_vma(&tlb, vma); in madvise_free_single_vma()
785 walk_page_range(vma->vm_mm, range.start, range.end, in madvise_free_single_vma()
786 &madvise_free_walk_ops, &tlb); in madvise_free_single_vma()
787 tlb_end_vma(&tlb, vma); in madvise_free_single_vma()
789 tlb_finish_mmu(&tlb); in madvise_free_single_vma()
816 zap_page_range_single(vma, start, end - start, NULL); in madvise_dontneed_single_vma()
831 return !(vma->vm_flags & forbidden); in madvise_dontneed_free_valid_vma()
855 struct mm_struct *mm = vma->vm_mm; in madvise_dontneed_free()
859 return -EINVAL; in madvise_dontneed_free()
870 return -ENOMEM; in madvise_dontneed_free()
877 return -EINVAL; in madvise_dontneed_free()
878 if (end > vma->vm_end) { in madvise_dontneed_free()
880 * Don't fail if end > vma->vm_end. If the old in madvise_dontneed_free()
881 * vma was split while the mmap_lock was in madvise_dontneed_free()
888 * end-vma->vm_end range, but the manager can in madvise_dontneed_free()
891 end = vma->vm_end; in madvise_dontneed_free()
901 return -EINVAL; in madvise_dontneed_free()
910 struct mm_struct *mm = vma->vm_mm; in madvise_populate()
920 * our VMA might have been split. in madvise_populate()
922 if (!vma || start >= vma->vm_end) { in madvise_populate()
925 return -ENOMEM; in madvise_populate()
928 tmp_end = min_t(unsigned long, end, vma->vm_end); in madvise_populate()
940 case -EINTR: in madvise_populate()
941 return -EINTR; in madvise_populate()
942 case -EINVAL: /* Incompatible mappings / permissions. */ in madvise_populate()
943 return -EINVAL; in madvise_populate()
944 case -EHWPOISON: in madvise_populate()
945 return -EHWPOISON; in madvise_populate()
946 case -EFAULT: /* VM_FAULT_SIGBUS or VM_FAULT_SIGSEGV */ in madvise_populate()
947 return -EFAULT; in madvise_populate()
952 case -ENOMEM: in madvise_populate()
953 return -ENOMEM; in madvise_populate()
972 struct mm_struct *mm = vma->vm_mm; in madvise_remove()
976 if (vma->vm_flags & VM_LOCKED) in madvise_remove()
977 return -EINVAL; in madvise_remove()
979 f = vma->vm_file; in madvise_remove()
981 if (!f || !f->f_mapping || !f->f_mapping->host) { in madvise_remove()
982 return -EINVAL; in madvise_remove()
986 return -EACCES; in madvise_remove()
988 offset = (loff_t)(start - vma->vm_start) in madvise_remove()
989 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); in madvise_remove()
1004 offset, end - start); in madvise_remove()
1022 unsigned long new_flags = vma->vm_flags; in madvise_vma_behavior()
1053 if (vma->vm_flags & VM_IO) in madvise_vma_behavior()
1054 return -EINVAL; in madvise_vma_behavior()
1059 if (vma->vm_file || vma->vm_flags & VM_SHARED) in madvise_vma_behavior()
1060 return -EINVAL; in madvise_vma_behavior()
1071 return -EINVAL; in madvise_vma_behavior()
1101 if (error == -ENOMEM) in madvise_vma_behavior()
1102 error = -EAGAIN; in madvise_vma_behavior()
1116 return -EPERM; in madvise_inject_error()
1144 if (ret == -EOPNOTSUPP) in madvise_inject_error()
1215 * original range will result in this function returning -ENOMEM while still
1233 * ranges, just ignore them, but return -ENOMEM at the end. in madvise_walk_vmas()
1234 * - different from the way of handling in mlock etc. in madvise_walk_vmas()
1237 if (vma && start > vma->vm_start) in madvise_walk_vmas()
1245 return -ENOMEM; in madvise_walk_vmas()
1247 /* Here start < (end|vma->vm_end). */ in madvise_walk_vmas()
1248 if (start < vma->vm_start) { in madvise_walk_vmas()
1249 unmapped_error = -ENOMEM; in madvise_walk_vmas()
1250 start = vma->vm_start; in madvise_walk_vmas()
1255 /* Here vma->vm_start <= start < (end|vma->vm_end) */ in madvise_walk_vmas()
1256 tmp = vma->vm_end; in madvise_walk_vmas()
1260 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ in madvise_walk_vmas()
1265 if (prev && start < prev->vm_end) in madvise_walk_vmas()
1266 start = prev->vm_end; in madvise_walk_vmas()
1270 vma = find_vma(mm, prev->vm_end); in madvise_walk_vmas()
1287 if (vma->vm_file && !vma_is_anon_shmem(vma)) in madvise_vma_anon_name()
1288 return -EBADF; in madvise_vma_anon_name()
1290 error = madvise_update_vma(vma, prev, start, end, vma->vm_flags, in madvise_vma_anon_name()
1297 if (error == -ENOMEM) in madvise_vma_anon_name()
1298 error = -EAGAIN; in madvise_vma_anon_name()
1309 return -EINVAL; in madvise_set_anon_name()
1312 /* Check to see whether len was rounded up from small -ve to zero */ in madvise_set_anon_name()
1314 return -EINVAL; in madvise_set_anon_name()
1318 return -EINVAL; in madvise_set_anon_name()
1332 * use appropriate read-ahead and caching techniques. The information
1337 * MADV_NORMAL - the default behavior is to read clusters. This
1338 * results in some read-ahead and read-behind.
1339 * MADV_RANDOM - the system should read the minimum amount of data
1340 * on any access, since it is unlikely that the appli-
1342 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
1345 * MADV_WILLNEED - the application is notifying the system to read
1347 * MADV_DONTNEED - the application is finished with the given range,
1349 * MADV_FREE - the application marks pages in the given range as lazy free,
1351 * MADV_REMOVE - the application wants to free up the given range of
1353 * MADV_DONTFORK - omit this area from child's address space when forking:
1355 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
1356 * MADV_WIPEONFORK - present the child process with zero-filled memory in this
1358 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
1359 * MADV_HWPOISON - trigger memory error handler as if the given memory range
1361 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
1362 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
1364 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1365 * MADV_HUGEPAGE - the application wants to back the given range by transparent
1368 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
1371 * MADV_COLLAPSE - synchronously coalesce pages into new THP.
1372 * MADV_DONTDUMP - the application wants to prevent pages in the given range
1374 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
1375 * MADV_COLD - the application is not expected to use this memory soon,
1378 * MADV_PAGEOUT - the application is not expected to use this memory soon,
1380 * MADV_POPULATE_READ - populate (prefault) page tables readable by
1382 * MADV_POPULATE_WRITE - populate (prefault) page tables writable by
1386 * zero - success
1387 * -EINVAL - start + len < 0, start is not page-aligned,
1390 * or the specified address range includes file, Huge TLB,
1392 * -ENOMEM - addresses in the specified range are not currently
1394 * -EIO - an I/O error occurred while paging in data.
1395 * -EBADF - map exists, but area maps something that isn't a file.
1396 * -EAGAIN - a kernel resource was temporarily unavailable.
1407 return -EINVAL; in do_madvise()
1410 return -EINVAL; in do_madvise()
1413 /* Check to see whether len was rounded up from small -ve to zero */ in do_madvise()
1415 return -EINVAL; in do_madvise()
1419 return -EINVAL; in do_madvise()
1432 return -EINTR; in do_madvise()
1454 return do_madvise(current->mm, start, len_in, behavior); in SYSCALL_DEFINE3()
1470 ret = -EINVAL; in SYSCALL_DEFINE5()
1485 ret = -EINVAL; in SYSCALL_DEFINE5()
1492 ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH; in SYSCALL_DEFINE5()
1498 * only non-destructive hints are currently supported. in SYSCALL_DEFINE5()
1501 ret = -EPERM; in SYSCALL_DEFINE5()
1515 ret = (total_len - iov_iter_count(&iter)) ? : ret; in SYSCALL_DEFINE5()