Lines Matching refs:vma
149 static void anon_vma_chain_link(struct vm_area_struct *vma, in anon_vma_chain_link() argument
153 avc->vma = vma; in anon_vma_chain_link()
155 list_add(&avc->same_vma, &vma->anon_vma_chain); in anon_vma_chain_link()
185 int __anon_vma_prepare(struct vm_area_struct *vma) in __anon_vma_prepare() argument
187 struct mm_struct *mm = vma->vm_mm; in __anon_vma_prepare()
198 anon_vma = find_mergeable_anon_vma(vma); in __anon_vma_prepare()
211 if (likely(!vma->anon_vma)) { in __anon_vma_prepare()
212 vma->anon_vma = anon_vma; in __anon_vma_prepare()
213 anon_vma_chain_link(vma, avc, anon_vma); in __anon_vma_prepare()
333 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) in anon_vma_fork() argument
344 vma->anon_vma = NULL; in anon_vma_fork()
350 error = anon_vma_clone(vma, pvma); in anon_vma_fork()
355 if (vma->anon_vma) in anon_vma_fork()
380 vma->anon_vma = anon_vma; in anon_vma_fork()
382 anon_vma_chain_link(vma, avc, anon_vma); in anon_vma_fork()
391 unlink_anon_vmas(vma); in anon_vma_fork()
395 void unlink_anon_vmas(struct vm_area_struct *vma) in unlink_anon_vmas() argument
404 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { in unlink_anon_vmas()
422 if (vma->anon_vma) { in unlink_anon_vmas()
423 vma->anon_vma->num_active_vmas--; in unlink_anon_vmas()
429 vma->anon_vma = NULL; in unlink_anon_vmas()
438 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { in unlink_anon_vmas()
788 const struct page *page, const struct vm_area_struct *vma) in page_address_in_vma() argument
796 if (!vma->anon_vma || !anon_vma || in page_address_in_vma()
797 vma->anon_vma->root != anon_vma->root) in page_address_in_vma()
799 } else if (!vma->vm_file) { in page_address_in_vma()
801 } else if (vma->vm_file->f_mapping != folio->mapping) { in page_address_in_vma()
806 return vma_address(vma, page_pgoff(folio, page), 1); in page_address_in_vma()
849 struct vm_area_struct *vma, unsigned long address, void *arg) in folio_referenced_one() argument
852 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
858 if (vma->vm_flags & VM_LOCKED) { in folio_referenced_one()
877 mlock_vma_folio(folio, vma); in folio_referenced_one()
888 if ((!atomic_read(&vma->vm_mm->mm_users) || in folio_referenced_one()
889 check_stable_address_space(vma->vm_mm)) && in folio_referenced_one()
901 if (ptep_clear_flush_young_notify(vma, address, in folio_referenced_one()
905 if (pmdp_clear_flush_young_notify(vma, address, in folio_referenced_one()
923 pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; in folio_referenced_one()
932 static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) in invalid_folio_referenced_vma() argument
943 if (!vma_has_recency(vma)) in invalid_folio_referenced_vma()
950 if (memcg && !mm_match_cgroup(vma->vm_mm, memcg)) in invalid_folio_referenced_vma()
1009 struct vm_area_struct *vma = pvmw->vma; in page_vma_mkclean_one() local
1018 vma->vm_mm, address, vma_address_end(pvmw)); in page_vma_mkclean_one()
1040 flush_cache_page(vma, address, pte_pfn(entry)); in page_vma_mkclean_one()
1041 entry = ptep_clear_flush(vma, address, pte); in page_vma_mkclean_one()
1044 set_pte_at(vma->vm_mm, address, pte, entry); in page_vma_mkclean_one()
1054 flush_cache_range(vma, address, in page_vma_mkclean_one()
1056 entry = pmdp_invalidate(vma, address, pmd); in page_vma_mkclean_one()
1059 set_pmd_at(vma->vm_mm, address, pmd, entry); in page_vma_mkclean_one()
1076 static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, in page_mkclean_one() argument
1079 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); in page_mkclean_one()
1087 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) in invalid_mkclean_vma() argument
1089 if (vma->vm_flags & VM_SHARED) in invalid_mkclean_vma()
1128 struct vm_area_struct *vma, unsigned long address, void *arg) in mapping_wrprotect_range_one() argument
1135 .vma = vma, in mapping_wrprotect_range_one()
1207 struct vm_area_struct *vma) in pfn_mkclean_range() argument
1213 .vma = vma, in pfn_mkclean_range()
1217 if (invalid_mkclean_vma(vma, NULL)) in pfn_mkclean_range()
1220 pvmw.address = vma_address(vma, pgoff, nr_pages); in pfn_mkclean_range()
1221 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); in pfn_mkclean_range()
1249 struct page *page, int nr_pages, struct vm_area_struct *vma, in __folio_add_rmap() argument
1266 nr = folio_add_return_large_mapcount(folio, orig_nr_pages, vma); in __folio_add_rmap()
1283 folio_add_large_mapcount(folio, orig_nr_pages, vma); in __folio_add_rmap()
1291 nr = folio_inc_return_large_mapcount(folio, vma); in __folio_add_rmap()
1319 folio_inc_large_mapcount(folio, vma); in __folio_add_rmap()
1336 void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma) in folio_move_anon_rmap() argument
1338 void *anon_vma = vma->anon_vma; in folio_move_anon_rmap()
1341 VM_BUG_ON_VMA(!anon_vma, vma); in folio_move_anon_rmap()
1359 static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, in __folio_set_anon() argument
1362 struct anon_vma *anon_vma = vma->anon_vma; in __folio_set_anon()
1381 folio->index = linear_page_index(vma, address); in __folio_set_anon()
1392 const struct page *page, struct vm_area_struct *vma, in __page_check_anon_rmap() argument
1406 VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, in __page_check_anon_rmap()
1408 VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address), in __page_check_anon_rmap()
1413 struct page *page, int nr_pages, struct vm_area_struct *vma, in __folio_add_anon_rmap() argument
1420 __folio_add_rmap(folio, page, nr_pages, vma, level); in __folio_add_anon_rmap()
1423 __page_check_anon_rmap(folio, page, vma, address); in __folio_add_anon_rmap()
1472 mlock_vma_folio(folio, vma); in __folio_add_anon_rmap()
1492 int nr_pages, struct vm_area_struct *vma, unsigned long address, in folio_add_anon_rmap_ptes() argument
1495 __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, in folio_add_anon_rmap_ptes()
1513 struct vm_area_struct *vma, unsigned long address, rmap_t flags) in folio_add_anon_rmap_pmd() argument
1516 __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, in folio_add_anon_rmap_pmd()
1538 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, in folio_add_new_anon_rmap() argument
1551 if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE)) in folio_add_new_anon_rmap()
1553 __folio_set_anon(folio, vma, address, exclusive); in folio_add_new_anon_rmap()
1574 folio_set_large_mapcount(folio, nr, vma); in folio_add_new_anon_rmap()
1581 folio_set_large_mapcount(folio, 1, vma); in folio_add_new_anon_rmap()
1589 VM_WARN_ON_ONCE(address < vma->vm_start || in folio_add_new_anon_rmap()
1590 address + (nr << PAGE_SHIFT) > vma->vm_end); in folio_add_new_anon_rmap()
1597 struct page *page, int nr_pages, struct vm_area_struct *vma, in __folio_add_file_rmap() argument
1602 __folio_add_rmap(folio, page, nr_pages, vma, level); in __folio_add_file_rmap()
1611 mlock_vma_folio(folio, vma); in __folio_add_file_rmap()
1626 int nr_pages, struct vm_area_struct *vma) in folio_add_file_rmap_ptes() argument
1628 __folio_add_file_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE); in folio_add_file_rmap_ptes()
1642 struct vm_area_struct *vma) in folio_add_file_rmap_pmd() argument
1645 __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD); in folio_add_file_rmap_pmd()
1662 struct vm_area_struct *vma) in folio_add_file_rmap_pud() argument
1666 __folio_add_file_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD); in folio_add_file_rmap_pud()
1673 struct page *page, int nr_pages, struct vm_area_struct *vma, in __folio_remove_rmap() argument
1690 nr = folio_sub_return_large_mapcount(folio, nr_pages, vma); in __folio_remove_rmap()
1702 folio_sub_large_mapcount(folio, nr_pages, vma); in __folio_remove_rmap()
1719 nr = folio_dec_return_large_mapcount(folio, vma); in __folio_remove_rmap()
1731 folio_dec_large_mapcount(folio, vma); in __folio_remove_rmap()
1775 munlock_vma_folio(folio, vma); in __folio_remove_rmap()
1790 int nr_pages, struct vm_area_struct *vma) in folio_remove_rmap_ptes() argument
1792 __folio_remove_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE); in folio_remove_rmap_ptes()
1806 struct vm_area_struct *vma) in folio_remove_rmap_pmd() argument
1809 __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD); in folio_remove_rmap_pmd()
1826 struct vm_area_struct *vma) in folio_remove_rmap_pud() argument
1830 __folio_remove_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD); in folio_remove_rmap_pud()
1841 struct vm_area_struct *vma = pvmw->vma; in folio_unmap_pte_batch() local
1850 end_addr = pmd_addr_end(addr, vma->vm_end); in folio_unmap_pte_batch()
1865 static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, in try_to_unmap_one() argument
1868 struct mm_struct *mm = vma->vm_mm; in try_to_unmap_one()
1869 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_unmap_one()
1898 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in try_to_unmap_one()
1905 adjust_range_if_pmd_sharing_possible(vma, &range.start, in try_to_unmap_one()
1909 hsz = huge_page_size(hstate_vma(vma)); in try_to_unmap_one()
1918 (vma->vm_flags & VM_LOCKED)) { in try_to_unmap_one()
1945 mlock_vma_folio(folio, vma); in try_to_unmap_one()
1951 if (unmap_huge_pmd_locked(vma, pvmw.address, pvmw.pmd, folio)) in try_to_unmap_one()
1966 split_huge_pmd_locked(vma, pvmw.address, in try_to_unmap_one()
2009 flush_cache_range(vma, range.start, range.end); in try_to_unmap_one()
2023 if (!hugetlb_vma_trylock_write(vma)) in try_to_unmap_one()
2025 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_unmap_one()
2026 hugetlb_vma_unlock_write(vma); in try_to_unmap_one()
2027 flush_tlb_range(vma, in try_to_unmap_one()
2041 hugetlb_vma_unlock_write(vma); in try_to_unmap_one()
2043 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
2049 flush_cache_range(vma, address, end_addr); in try_to_unmap_one()
2064 flush_tlb_range(vma, address, end_addr); in try_to_unmap_one()
2076 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); in try_to_unmap_one()
2092 !userfaultfd_armed(vma)) { in try_to_unmap_one()
2137 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { in try_to_unmap_one()
2172 if (arch_unmap_one(mm, vma, address, pteval) < 0) { in try_to_unmap_one()
2226 folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); in try_to_unmap_one()
2228 if (vma->vm_flags & VM_LOCKED) in try_to_unmap_one()
2251 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) in invalid_migration_vma() argument
2253 return vma_is_temporary_stack(vma); in invalid_migration_vma()
2293 static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, in try_to_migrate_one() argument
2296 struct mm_struct *mm = vma->vm_mm; in try_to_migrate_one()
2297 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_migrate_one()
2324 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in try_to_migrate_one()
2331 adjust_range_if_pmd_sharing_possible(vma, &range.start, in try_to_migrate_one()
2335 hsz = huge_page_size(hstate_vma(vma)); in try_to_migrate_one()
2343 split_huge_pmd_locked(vma, pvmw.address, in try_to_migrate_one()
2394 flush_cache_range(vma, range.start, range.end); in try_to_migrate_one()
2408 if (!hugetlb_vma_trylock_write(vma)) { in try_to_migrate_one()
2413 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_migrate_one()
2414 hugetlb_vma_unlock_write(vma); in try_to_migrate_one()
2415 flush_tlb_range(vma, in try_to_migrate_one()
2431 hugetlb_vma_unlock_write(vma); in try_to_migrate_one()
2434 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2439 flush_cache_page(vma, address, pfn); in try_to_migrate_one()
2454 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2483 !userfaultfd_armed(vma)) { in try_to_migrate_one()
2504 if (arch_unmap_one(mm, vma, address, pteval) < 0) { in try_to_migrate_one()
2580 folio_remove_rmap_pte(folio, subpage, vma); in try_to_migrate_one()
2581 if (vma->vm_flags & VM_LOCKED) in try_to_migrate_one()
2681 struct vm_area_struct *vma; in make_device_exclusive() local
2703 &vma); in make_device_exclusive()
2734 fw_folio = folio_walk_start(&fw, vma, addr, 0); in make_device_exclusive()
2738 folio_walk_end(&fw, vma); in make_device_exclusive()
2746 flush_cache_page(vma, addr, page_to_pfn(page)); in make_device_exclusive()
2747 fw.pte = ptep_clear_flush(vma, addr, fw.ptep); in make_device_exclusive()
2765 folio_walk_end(&fw, vma); in make_device_exclusive()
2845 struct vm_area_struct *vma = avc->vma; in rmap_walk_anon() local
2846 unsigned long address = vma_address(vma, pgoff_start, in rmap_walk_anon()
2849 VM_BUG_ON_VMA(address == -EFAULT, vma); in rmap_walk_anon()
2852 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) in rmap_walk_anon()
2855 if (!rwc->rmap_one(folio, vma, address, rwc->arg)) in rmap_walk_anon()
2890 struct vm_area_struct *vma; in __rmap_walk_file() local
2908 vma_interval_tree_foreach(vma, &mapping->i_mmap, in __rmap_walk_file()
2910 unsigned long address = vma_address(vma, pgoff_start, nr_pages); in __rmap_walk_file()
2912 VM_BUG_ON_VMA(address == -EFAULT, vma); in __rmap_walk_file()
2915 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) in __rmap_walk_file()
2918 if (!rwc->rmap_one(folio, vma, address, rwc->arg)) in __rmap_walk_file()
2982 void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma, in hugetlb_add_anon_rmap() argument
2997 struct vm_area_struct *vma, unsigned long address) in hugetlb_add_new_anon_rmap() argument
3001 BUG_ON(address < vma->vm_start || address >= vma->vm_end); in hugetlb_add_new_anon_rmap()
3006 __folio_set_anon(folio, vma, address, true); in hugetlb_add_new_anon_rmap()