Lines Matching refs:folio
130 struct folio *folio, void *shadow) in page_cache_delete() argument
132 XA_STATE(xas, &mapping->i_pages, folio->index); in page_cache_delete()
137 xas_set_order(&xas, folio->index, folio_order(folio)); in page_cache_delete()
138 nr = folio_nr_pages(folio); in page_cache_delete()
140 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in page_cache_delete()
145 folio->mapping = NULL; in page_cache_delete()
151 struct folio *folio) in filemap_unaccount_folio() argument
155 VM_BUG_ON_FOLIO(folio_mapped(folio), folio); in filemap_unaccount_folio()
156 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { in filemap_unaccount_folio()
158 current->comm, folio_pfn(folio)); in filemap_unaccount_folio()
159 dump_page(&folio->page, "still mapped when deleted"); in filemap_unaccount_folio()
163 if (mapping_exiting(mapping) && !folio_test_large(folio)) { in filemap_unaccount_folio()
164 int mapcount = folio_mapcount(folio); in filemap_unaccount_folio()
166 if (folio_ref_count(folio) >= mapcount + 2) { in filemap_unaccount_folio()
173 atomic_set(&folio->_mapcount, -1); in filemap_unaccount_folio()
174 folio_ref_sub(folio, mapcount); in filemap_unaccount_folio()
180 if (folio_test_hugetlb(folio)) in filemap_unaccount_folio()
183 nr = folio_nr_pages(folio); in filemap_unaccount_folio()
185 lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); in filemap_unaccount_folio()
186 if (folio_test_swapbacked(folio)) { in filemap_unaccount_folio()
187 lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); in filemap_unaccount_folio()
188 if (folio_test_pmd_mappable(folio)) in filemap_unaccount_folio()
189 lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); in filemap_unaccount_folio()
190 } else if (folio_test_pmd_mappable(folio)) { in filemap_unaccount_folio()
191 lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); in filemap_unaccount_folio()
194 if (test_bit(AS_KERNEL_FILE, &folio->mapping->flags)) in filemap_unaccount_folio()
195 mod_node_page_state(folio_pgdat(folio), in filemap_unaccount_folio()
212 if (WARN_ON_ONCE(folio_test_dirty(folio) && in filemap_unaccount_folio()
214 folio_account_cleaned(folio, inode_to_wb(mapping->host)); in filemap_unaccount_folio()
222 void __filemap_remove_folio(struct folio *folio, void *shadow) in __filemap_remove_folio() argument
224 struct address_space *mapping = folio->mapping; in __filemap_remove_folio()
226 trace_mm_filemap_delete_from_page_cache(folio); in __filemap_remove_folio()
227 filemap_unaccount_folio(mapping, folio); in __filemap_remove_folio()
228 page_cache_delete(mapping, folio, shadow); in __filemap_remove_folio()
231 void filemap_free_folio(struct address_space *mapping, struct folio *folio) in filemap_free_folio() argument
233 void (*free_folio)(struct folio *); in filemap_free_folio()
237 free_folio(folio); in filemap_free_folio()
239 folio_put_refs(folio, folio_nr_pages(folio)); in filemap_free_folio()
250 void filemap_remove_folio(struct folio *folio) in filemap_remove_folio() argument
252 struct address_space *mapping = folio->mapping; in filemap_remove_folio()
254 BUG_ON(!folio_test_locked(folio)); in filemap_remove_folio()
257 __filemap_remove_folio(folio, NULL); in filemap_remove_folio()
263 filemap_free_folio(mapping, folio); in filemap_remove_folio()
285 struct folio *folio; in page_cache_delete_batch() local
288 xas_for_each(&xas, folio, ULONG_MAX) { in page_cache_delete_batch()
293 if (xa_is_value(folio)) in page_cache_delete_batch()
302 if (folio != fbatch->folios[i]) { in page_cache_delete_batch()
303 VM_BUG_ON_FOLIO(folio->index > in page_cache_delete_batch()
304 fbatch->folios[i]->index, folio); in page_cache_delete_batch()
308 WARN_ON_ONCE(!folio_test_locked(folio)); in page_cache_delete_batch()
310 folio->mapping = NULL; in page_cache_delete_batch()
315 total_pages += folio_nr_pages(folio); in page_cache_delete_batch()
331 struct folio *folio = fbatch->folios[i]; in delete_from_page_cache_batch() local
333 trace_mm_filemap_delete_from_page_cache(folio); in delete_from_page_cache_batch()
334 filemap_unaccount_folio(mapping, folio); in delete_from_page_cache_batch()
482 struct folio *folio; in filemap_range_has_page() local
491 folio = xas_find(&xas, max); in filemap_range_has_page()
492 if (xas_retry(&xas, folio)) in filemap_range_has_page()
495 if (xa_is_value(folio)) in filemap_range_has_page()
506 return folio != NULL; in filemap_range_has_page()
530 struct folio *folio = fbatch.folios[i]; in __filemap_fdatawait_range() local
532 folio_wait_writeback(folio); in __filemap_fdatawait_range()
642 struct folio *folio; in filemap_range_has_writeback() local
648 xas_for_each(&xas, folio, max) { in filemap_range_has_writeback()
649 if (xas_retry(&xas, folio)) in filemap_range_has_writeback()
651 if (xa_is_value(folio)) in filemap_range_has_writeback()
653 if (folio_test_dirty(folio) || folio_test_locked(folio) || in filemap_range_has_writeback()
654 folio_test_writeback(folio)) in filemap_range_has_writeback()
658 return folio != NULL; in filemap_range_has_writeback()
811 void replace_page_cache_folio(struct folio *old, struct folio *new) in replace_page_cache_folio()
814 void (*free_folio)(struct folio *) = mapping->a_ops->free_folio; in replace_page_cache_folio()
849 struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) in __filemap_add_folio() argument
851 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); in __filemap_add_folio()
854 unsigned int forder = folio_order(folio); in __filemap_add_folio()
856 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in __filemap_add_folio()
857 VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); in __filemap_add_folio()
858 VM_BUG_ON_FOLIO(folio_order(folio) < mapping_min_folio_order(mapping), in __filemap_add_folio()
859 folio); in __filemap_add_folio()
862 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); in __filemap_add_folio()
863 huge = folio_test_hugetlb(folio); in __filemap_add_folio()
864 nr = folio_nr_pages(folio); in __filemap_add_folio()
867 folio_ref_add(folio, nr); in __filemap_add_folio()
868 folio->mapping = mapping; in __filemap_add_folio()
869 folio->index = xas.xa_index; in __filemap_add_folio()
915 xas_store(&xas, folio); in __filemap_add_folio()
923 lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); in __filemap_add_folio()
924 if (folio_test_pmd_mappable(folio)) in __filemap_add_folio()
925 lruvec_stat_mod_folio(folio, in __filemap_add_folio()
939 trace_mm_filemap_add_to_page_cache(folio); in __filemap_add_folio()
942 folio->mapping = NULL; in __filemap_add_folio()
944 folio_put_refs(folio, nr); in __filemap_add_folio()
949 int filemap_add_folio(struct address_space *mapping, struct folio *folio, in filemap_add_folio() argument
959 ret = mem_cgroup_charge(folio, NULL, gfp); in filemap_add_folio()
965 __folio_set_locked(folio); in filemap_add_folio()
966 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); in filemap_add_folio()
968 mem_cgroup_uncharge(folio); in filemap_add_folio()
969 __folio_clear_locked(folio); in filemap_add_folio()
979 WARN_ON_ONCE(folio_test_active(folio)); in filemap_add_folio()
981 workingset_refault(folio, shadow); in filemap_add_folio()
982 folio_add_lru(folio); in filemap_add_folio()
984 mod_node_page_state(folio_pgdat(folio), in filemap_add_folio()
986 folio_nr_pages(folio)); in filemap_add_folio()
993 struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order, in filemap_alloc_folio_noprof()
997 struct folio *folio; in filemap_alloc_folio_noprof() local
1008 folio = __folio_alloc_node_noprof(gfp, order, n); in filemap_alloc_folio_noprof()
1009 } while (!folio && read_mems_allowed_retry(cpuset_mems_cookie)); in filemap_alloc_folio_noprof()
1011 return folio; in filemap_alloc_folio_noprof()
1070 static wait_queue_head_t *folio_waitqueue(struct folio *folio) in folio_waitqueue() argument
1072 return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)]; in folio_waitqueue()
1149 if (test_bit(key->bit_nr, &key->folio->flags.f)) in wake_page_function()
1152 if (test_and_set_bit(key->bit_nr, &key->folio->flags.f)) in wake_page_function()
1184 static void folio_wake_bit(struct folio *folio, int bit_nr) in folio_wake_bit() argument
1186 wait_queue_head_t *q = folio_waitqueue(folio); in folio_wake_bit()
1190 key.folio = folio; in folio_wake_bit()
1207 folio_clear_waiters(folio); in folio_wake_bit()
1231 static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, in folio_trylock_flag() argument
1235 if (test_and_set_bit(bit_nr, &folio->flags.f)) in folio_trylock_flag()
1237 } else if (test_bit(bit_nr, &folio->flags.f)) in folio_trylock_flag()
1244 static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, in folio_wait_bit_common() argument
1247 wait_queue_head_t *q = folio_waitqueue(folio); in folio_wait_bit_common()
1256 !folio_test_uptodate(folio) && folio_test_workingset(folio)) { in folio_wait_bit_common()
1264 wait_page.folio = folio; in folio_wait_bit_common()
1290 folio_set_waiters(folio); in folio_wait_bit_common()
1291 if (!folio_trylock_flag(folio, bit_nr, wait)) in folio_wait_bit_common()
1304 folio_put(folio); in folio_wait_bit_common()
1341 if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0)))) in folio_wait_bit_common()
1408 struct folio *folio = softleaf_to_folio(entry); in softleaf_entry_wait_on_locked() local
1410 q = folio_waitqueue(folio); in softleaf_entry_wait_on_locked()
1411 if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { in softleaf_entry_wait_on_locked()
1419 wait_page.folio = folio; in softleaf_entry_wait_on_locked()
1424 folio_set_waiters(folio); in softleaf_entry_wait_on_locked()
1425 if (!folio_trylock_flag(folio, PG_locked, wait)) in softleaf_entry_wait_on_locked()
1465 void folio_wait_bit(struct folio *folio, int bit_nr) in folio_wait_bit() argument
1467 folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); in folio_wait_bit()
1471 int folio_wait_bit_killable(struct folio *folio, int bit_nr) in folio_wait_bit_killable() argument
1473 return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED); in folio_wait_bit_killable()
1490 static int folio_put_wait_locked(struct folio *folio, int state) in folio_put_wait_locked() argument
1492 return folio_wait_bit_common(folio, PG_locked, state, DROP); in folio_put_wait_locked()
1504 void folio_unlock(struct folio *folio) in folio_unlock() argument
1509 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in folio_unlock()
1510 if (folio_xor_flags_has_waiters(folio, 1 << PG_locked)) in folio_unlock()
1511 folio_wake_bit(folio, PG_locked); in folio_unlock()
1529 void folio_end_read(struct folio *folio, bool success) in folio_end_read() argument
1535 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in folio_end_read()
1536 VM_BUG_ON_FOLIO(success && folio_test_uptodate(folio), folio); in folio_end_read()
1540 if (folio_xor_flags_has_waiters(folio, mask)) in folio_end_read()
1541 folio_wake_bit(folio, PG_locked); in folio_end_read()
1556 void folio_end_private_2(struct folio *folio) in folio_end_private_2() argument
1558 VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio); in folio_end_private_2()
1559 clear_bit_unlock(PG_private_2, folio_flags(folio, 0)); in folio_end_private_2()
1560 folio_wake_bit(folio, PG_private_2); in folio_end_private_2()
1561 folio_put(folio); in folio_end_private_2()
1571 void folio_wait_private_2(struct folio *folio) in folio_wait_private_2() argument
1573 while (folio_test_private_2(folio)) in folio_wait_private_2()
1574 folio_wait_bit(folio, PG_private_2); in folio_wait_private_2()
1589 int folio_wait_private_2_killable(struct folio *folio) in folio_wait_private_2_killable() argument
1593 while (folio_test_private_2(folio)) { in folio_wait_private_2_killable()
1594 ret = folio_wait_bit_killable(folio, PG_private_2); in folio_wait_private_2_killable()
1603 static void filemap_end_dropbehind(struct folio *folio) in filemap_end_dropbehind() argument
1605 struct address_space *mapping = folio->mapping; in filemap_end_dropbehind()
1607 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in filemap_end_dropbehind()
1609 if (folio_test_writeback(folio) || folio_test_dirty(folio)) in filemap_end_dropbehind()
1611 if (!folio_test_clear_dropbehind(folio)) in filemap_end_dropbehind()
1614 folio_unmap_invalidate(mapping, folio, 0); in filemap_end_dropbehind()
1622 void folio_end_dropbehind(struct folio *folio) in folio_end_dropbehind() argument
1624 if (!folio_test_dropbehind(folio)) in folio_end_dropbehind()
1634 if (in_task() && folio_trylock(folio)) { in folio_end_dropbehind()
1635 filemap_end_dropbehind(folio); in folio_end_dropbehind()
1636 folio_unlock(folio); in folio_end_dropbehind()
1650 void folio_end_writeback_no_dropbehind(struct folio *folio) in folio_end_writeback_no_dropbehind() argument
1652 VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); in folio_end_writeback_no_dropbehind()
1661 if (folio_test_reclaim(folio)) { in folio_end_writeback_no_dropbehind()
1662 folio_clear_reclaim(folio); in folio_end_writeback_no_dropbehind()
1663 folio_rotate_reclaimable(folio); in folio_end_writeback_no_dropbehind()
1666 if (__folio_end_writeback(folio)) in folio_end_writeback_no_dropbehind()
1667 folio_wake_bit(folio, PG_writeback); in folio_end_writeback_no_dropbehind()
1669 acct_reclaim_writeback(folio); in folio_end_writeback_no_dropbehind()
1681 void folio_end_writeback(struct folio *folio) in folio_end_writeback() argument
1683 VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); in folio_end_writeback()
1691 folio_get(folio); in folio_end_writeback()
1692 folio_end_writeback_no_dropbehind(folio); in folio_end_writeback()
1693 folio_end_dropbehind(folio); in folio_end_writeback()
1694 folio_put(folio); in folio_end_writeback()
1702 void __folio_lock(struct folio *folio) in __folio_lock() argument
1704 folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE, in __folio_lock()
1709 int __folio_lock_killable(struct folio *folio) in __folio_lock_killable() argument
1711 return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE, in __folio_lock_killable()
1716 static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait) in __folio_lock_async() argument
1718 struct wait_queue_head *q = folio_waitqueue(folio); in __folio_lock_async()
1721 wait->folio = folio; in __folio_lock_async()
1726 folio_set_waiters(folio); in __folio_lock_async()
1727 ret = !folio_trylock(folio); in __folio_lock_async()
1753 vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf) in __folio_lock_or_retry() argument
1767 folio_wait_locked_killable(folio); in __folio_lock_or_retry()
1769 folio_wait_locked(folio); in __folio_lock_or_retry()
1775 ret = __folio_lock_killable(folio); in __folio_lock_or_retry()
1781 __folio_lock(folio); in __folio_lock_or_retry()
1895 struct folio *folio; in filemap_get_entry() local
1900 folio = xas_load(&xas); in filemap_get_entry()
1901 if (xas_retry(&xas, folio)) in filemap_get_entry()
1907 if (!folio || xa_is_value(folio)) in filemap_get_entry()
1910 if (!folio_try_get(folio)) in filemap_get_entry()
1913 if (unlikely(folio != xas_reload(&xas))) { in filemap_get_entry()
1914 folio_put(folio); in filemap_get_entry()
1920 return folio; in filemap_get_entry()
1940 struct folio *__filemap_get_folio_mpol(struct address_space *mapping, in __filemap_get_folio_mpol()
1943 struct folio *folio; in __filemap_get_folio_mpol() local
1946 folio = filemap_get_entry(mapping, index); in __filemap_get_folio_mpol()
1947 if (xa_is_value(folio)) in __filemap_get_folio_mpol()
1948 folio = NULL; in __filemap_get_folio_mpol()
1949 if (!folio) in __filemap_get_folio_mpol()
1954 if (!folio_trylock(folio)) { in __filemap_get_folio_mpol()
1955 folio_put(folio); in __filemap_get_folio_mpol()
1959 folio_lock(folio); in __filemap_get_folio_mpol()
1963 if (unlikely(folio->mapping != mapping)) { in __filemap_get_folio_mpol()
1964 folio_unlock(folio); in __filemap_get_folio_mpol()
1965 folio_put(folio); in __filemap_get_folio_mpol()
1968 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); in __filemap_get_folio_mpol()
1972 folio_mark_accessed(folio); in __filemap_get_folio_mpol()
1975 if (folio_test_idle(folio)) in __filemap_get_folio_mpol()
1976 folio_clear_idle(folio); in __filemap_get_folio_mpol()
1980 folio_wait_stable(folio); in __filemap_get_folio_mpol()
1982 if (!folio && (fgp_flags & FGP_CREAT)) { in __filemap_get_folio_mpol()
2011 folio = filemap_alloc_folio(alloc_gfp, order, policy); in __filemap_get_folio_mpol()
2012 if (!folio) in __filemap_get_folio_mpol()
2017 __folio_set_referenced(folio); in __filemap_get_folio_mpol()
2019 __folio_set_dropbehind(folio); in __filemap_get_folio_mpol()
2021 err = filemap_add_folio(mapping, folio, index, gfp); in __filemap_get_folio_mpol()
2024 folio_put(folio); in __filemap_get_folio_mpol()
2025 folio = NULL; in __filemap_get_folio_mpol()
2047 if (folio && (fgp_flags & FGP_FOR_MMAP)) in __filemap_get_folio_mpol()
2048 folio_unlock(folio); in __filemap_get_folio_mpol()
2051 if (!folio) in __filemap_get_folio_mpol()
2054 if (folio_test_dropbehind(folio) && !(fgp_flags & FGP_DONTCACHE)) in __filemap_get_folio_mpol()
2055 folio_clear_dropbehind(folio); in __filemap_get_folio_mpol()
2056 return folio; in __filemap_get_folio_mpol()
2060 static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, in find_get_entry()
2063 struct folio *folio; in find_get_entry() local
2067 folio = xas_find(xas, max); in find_get_entry()
2069 folio = xas_find_marked(xas, max, mark); in find_get_entry()
2071 if (xas_retry(xas, folio)) in find_get_entry()
2078 if (!folio || xa_is_value(folio)) in find_get_entry()
2079 return folio; in find_get_entry()
2081 if (!folio_try_get(folio)) in find_get_entry()
2084 if (unlikely(folio != xas_reload(xas))) { in find_get_entry()
2085 folio_put(folio); in find_get_entry()
2089 return folio; in find_get_entry()
2119 struct folio *folio; in find_get_entries() local
2122 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { in find_get_entries()
2124 if (!folio_batch_add(fbatch, folio)) in find_get_entries()
2132 folio = fbatch->folios[idx]; in find_get_entries()
2133 if (!xa_is_value(folio)) in find_get_entries()
2134 nr = folio_nr_pages(folio); in find_get_entries()
2168 struct folio *folio; in find_lock_entries() local
2171 while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { in find_lock_entries()
2175 if (!xa_is_value(folio)) { in find_lock_entries()
2176 nr = folio_nr_pages(folio); in find_lock_entries()
2177 base = folio->index; in find_lock_entries()
2184 if (!folio_trylock(folio)) in find_lock_entries()
2186 if (folio->mapping != mapping || in find_lock_entries()
2187 folio_test_writeback(folio)) in find_lock_entries()
2189 VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), in find_lock_entries()
2190 folio); in find_lock_entries()
2205 if (!folio_batch_add(fbatch, folio)) in find_lock_entries()
2209 folio_unlock(folio); in find_lock_entries()
2211 folio_put(folio); in find_lock_entries()
2259 struct folio *folio; in filemap_get_folios_contig() local
2263 for (folio = xas_load(&xas); folio && xas.xa_index <= end; in filemap_get_folios_contig()
2264 folio = xas_next(&xas)) { in filemap_get_folios_contig()
2265 if (xas_retry(&xas, folio)) in filemap_get_folios_contig()
2271 if (xa_is_value(folio)) in filemap_get_folios_contig()
2275 if (xa_is_sibling(folio)) in filemap_get_folios_contig()
2278 if (!folio_try_get(folio)) in filemap_get_folios_contig()
2281 if (unlikely(folio != xas_reload(&xas))) in filemap_get_folios_contig()
2284 if (!folio_batch_add(fbatch, folio)) { in filemap_get_folios_contig()
2285 nr = folio_nr_pages(folio); in filemap_get_folios_contig()
2286 *start = folio->index + nr; in filemap_get_folios_contig()
2289 xas_advance(&xas, folio_next_index(folio) - 1); in filemap_get_folios_contig()
2292 folio_put(folio); in filemap_get_folios_contig()
2302 folio = fbatch->folios[nr - 1]; in filemap_get_folios_contig()
2303 *start = folio_next_index(folio); in filemap_get_folios_contig()
2334 struct folio *folio; in filemap_get_folios_tag() local
2337 while ((folio = find_get_entry(&xas, end, tag)) != NULL) { in filemap_get_folios_tag()
2343 if (xa_is_value(folio)) in filemap_get_folios_tag()
2345 if (!folio_batch_add(fbatch, folio)) { in filemap_get_folios_tag()
2346 unsigned long nr = folio_nr_pages(folio); in filemap_get_folios_tag()
2347 *start = folio->index + nr; in filemap_get_folios_tag()
2389 struct folio *folio; in filemap_get_folios_dirty() local
2392 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { in filemap_get_folios_dirty()
2393 if (xa_is_value(folio)) in filemap_get_folios_dirty()
2395 if (folio_trylock(folio)) { in filemap_get_folios_dirty()
2396 bool clean = !folio_test_dirty(folio) && in filemap_get_folios_dirty()
2397 !folio_test_writeback(folio); in filemap_get_folios_dirty()
2398 folio_unlock(folio); in filemap_get_folios_dirty()
2400 folio_put(folio); in filemap_get_folios_dirty()
2404 if (!folio_batch_add(fbatch, folio)) { in filemap_get_folios_dirty()
2405 unsigned long nr = folio_nr_pages(folio); in filemap_get_folios_dirty()
2406 *start = folio->index + nr; in filemap_get_folios_dirty()
2459 struct folio *folio; in filemap_get_read_batch() local
2462 for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { in filemap_get_read_batch()
2463 if (xas_retry(&xas, folio)) in filemap_get_read_batch()
2465 if (xas.xa_index > max || xa_is_value(folio)) in filemap_get_read_batch()
2467 if (xa_is_sibling(folio)) in filemap_get_read_batch()
2469 if (!folio_try_get(folio)) in filemap_get_read_batch()
2472 if (unlikely(folio != xas_reload(&xas))) in filemap_get_read_batch()
2475 if (!folio_batch_add(fbatch, folio)) in filemap_get_read_batch()
2477 if (!folio_test_uptodate(folio)) in filemap_get_read_batch()
2479 if (folio_test_readahead(folio)) in filemap_get_read_batch()
2481 xas_advance(&xas, folio_next_index(folio) - 1); in filemap_get_read_batch()
2484 folio_put(folio); in filemap_get_read_batch()
2492 struct folio *folio) in filemap_read_folio() argument
2494 bool workingset = folio_test_workingset(folio); in filemap_read_folio()
2501 error = filler(file, folio); in filemap_read_folio()
2507 error = folio_wait_locked_killable(folio); in filemap_read_folio()
2510 if (folio_test_uptodate(folio)) in filemap_read_folio()
2518 loff_t pos, size_t count, struct folio *folio, in filemap_range_uptodate() argument
2521 if (folio_test_uptodate(folio)) in filemap_range_uptodate()
2528 if (mapping->host->i_blkbits >= folio_shift(folio)) in filemap_range_uptodate()
2531 if (folio_pos(folio) > pos) { in filemap_range_uptodate()
2532 count -= folio_pos(folio) - pos; in filemap_range_uptodate()
2535 pos -= folio_pos(folio); in filemap_range_uptodate()
2538 if (pos == 0 && count >= folio_size(folio)) in filemap_range_uptodate()
2541 return mapping->a_ops->is_partially_uptodate(folio, pos, count); in filemap_range_uptodate()
2546 struct folio *folio, bool need_uptodate) in filemap_update_page() argument
2557 if (!folio_trylock(folio)) { in filemap_update_page()
2567 folio_put_wait_locked(folio, TASK_KILLABLE); in filemap_update_page()
2570 error = __folio_lock_async(folio, iocb->ki_waitq); in filemap_update_page()
2576 if (!folio->mapping) in filemap_update_page()
2580 if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio, in filemap_update_page()
2589 folio); in filemap_update_page()
2592 folio_unlock(folio); in filemap_update_page()
2596 folio_put(folio); in filemap_update_page()
2603 struct folio *folio; in filemap_create_folio() local
2611 folio = filemap_alloc_folio(mapping_gfp_mask(mapping), min_order, NULL); in filemap_create_folio()
2612 if (!folio) in filemap_create_folio()
2615 __folio_set_dropbehind(folio); in filemap_create_folio()
2632 error = filemap_add_folio(mapping, folio, index, in filemap_create_folio()
2640 folio); in filemap_create_folio()
2645 folio_batch_add(fbatch, folio); in filemap_create_folio()
2649 folio_put(folio); in filemap_create_folio()
2654 struct address_space *mapping, struct folio *folio, in filemap_readahead() argument
2657 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index); in filemap_readahead()
2663 page_cache_async_ra(&ractl, folio, last_index - folio->index); in filemap_readahead()
2674 struct folio *folio; in filemap_get_pages() local
2707 folio = fbatch->folios[folio_batch_count(fbatch) - 1]; in filemap_get_pages()
2708 if (folio_test_readahead(folio)) { in filemap_get_pages()
2709 err = filemap_readahead(iocb, filp, mapping, folio, last_index); in filemap_get_pages()
2713 if (!folio_test_uptodate(folio)) { in filemap_get_pages()
2718 err = filemap_update_page(iocb, mapping, count, folio, in filemap_get_pages()
2728 folio_put(folio); in filemap_get_pages()
2736 static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio) in pos_same_folio() argument
2738 unsigned int shift = folio_shift(folio); in pos_same_folio()
2743 static void filemap_end_dropbehind_read(struct folio *folio) in filemap_end_dropbehind_read() argument
2745 if (!folio_test_dropbehind(folio)) in filemap_end_dropbehind_read()
2747 if (folio_test_writeback(folio) || folio_test_dirty(folio)) in filemap_end_dropbehind_read()
2749 if (folio_trylock(folio)) { in filemap_end_dropbehind_read()
2750 filemap_end_dropbehind(folio); in filemap_end_dropbehind_read()
2751 folio_unlock(folio); in filemap_end_dropbehind_read()
2837 struct folio *folio = fbatch.folios[i]; in filemap_read() local
2838 size_t fsize = folio_size(folio); in filemap_read()
2844 if (end_offset < folio_pos(folio)) in filemap_read()
2847 folio_mark_accessed(folio); in filemap_read()
2854 flush_dcache_folio(folio); in filemap_read()
2856 copied = copy_folio_to_iter(folio, offset, bytes, iter); in filemap_read()
2869 struct folio *folio = fbatch.folios[i]; in filemap_read() local
2871 filemap_end_dropbehind_read(folio); in filemap_read()
2872 folio_put(folio); in filemap_read()
3005 struct folio *folio, loff_t fpos, size_t size) in splice_folio_into_pipe() argument
3008 size_t spliced = 0, offset = offset_in_folio(folio, fpos); in splice_folio_into_pipe()
3010 page = folio_page(folio, offset / PAGE_SIZE); in splice_folio_into_pipe()
3011 size = min(size, folio_size(folio) - offset); in splice_folio_into_pipe()
3024 folio_get(folio); in splice_folio_into_pipe()
3108 struct folio *folio = fbatch.folios[i]; in filemap_splice_read() local
3111 if (folio_pos(folio) >= end_offset) in filemap_splice_read()
3113 folio_mark_accessed(folio); in filemap_splice_read()
3121 flush_dcache_folio(folio); in filemap_splice_read()
3124 n = splice_folio_into_pipe(pipe, folio, *ppos, n); in filemap_splice_read()
3147 struct address_space *mapping, struct folio *folio, in folio_seek_hole_data() argument
3153 if (xa_is_value(folio) || folio_test_uptodate(folio)) in folio_seek_hole_data()
3160 folio_lock(folio); in folio_seek_hole_data()
3161 if (unlikely(folio->mapping != mapping)) in folio_seek_hole_data()
3164 offset = offset_in_folio(folio, start) & ~(bsz - 1); in folio_seek_hole_data()
3167 if (ops->is_partially_uptodate(folio, offset, bsz) == in folio_seek_hole_data()
3172 } while (offset < folio_size(folio)); in folio_seek_hole_data()
3174 folio_unlock(folio); in folio_seek_hole_data()
3179 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio) in seek_folio_size() argument
3181 if (xa_is_value(folio)) in seek_folio_size()
3183 return folio_size(folio); in seek_folio_size()
3210 struct folio *folio; in mapping_seek_hole_data() local
3216 while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { in mapping_seek_hole_data()
3226 seek_size = seek_folio_size(&xas, folio); in mapping_seek_hole_data()
3228 start = folio_seek_hole_data(&xas, mapping, folio, start, pos, in mapping_seek_hole_data()
3236 if (!xa_is_value(folio)) in mapping_seek_hole_data()
3237 folio_put(folio); in mapping_seek_hole_data()
3243 if (folio && !xa_is_value(folio)) in mapping_seek_hole_data()
3244 folio_put(folio); in mapping_seek_hole_data()
3264 static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, in lock_folio_maybe_drop_mmap() argument
3267 if (folio_trylock(folio)) in lock_folio_maybe_drop_mmap()
3280 if (__folio_lock_killable(folio)) { in lock_folio_maybe_drop_mmap()
3293 __folio_lock(folio); in lock_folio_maybe_drop_mmap()
3415 struct folio *folio) in do_async_mmap_readahead() argument
3433 if (likely(!folio_test_locked(folio))) { in do_async_mmap_readahead()
3439 if (folio_test_readahead(folio)) { in do_async_mmap_readahead()
3441 page_cache_async_ra(&ractl, folio, ra->ra_pages); in do_async_mmap_readahead()
3520 struct folio *folio; in filemap_fault() local
3533 folio = filemap_get_folio(mapping, index); in filemap_fault()
3534 if (likely(!IS_ERR(folio))) { in filemap_fault()
3540 fpin = do_async_mmap_readahead(vmf, folio); in filemap_fault()
3541 if (unlikely(!folio_test_uptodate(folio))) { in filemap_fault()
3564 folio = __filemap_get_folio(mapping, index, in filemap_fault()
3567 if (IS_ERR(folio)) { in filemap_fault()
3575 if (!lock_folio_maybe_drop_mmap(vmf, folio, &fpin)) in filemap_fault()
3579 if (unlikely(folio->mapping != mapping)) { in filemap_fault()
3580 folio_unlock(folio); in filemap_fault()
3581 folio_put(folio); in filemap_fault()
3584 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); in filemap_fault()
3591 if (unlikely(!folio_test_uptodate(folio))) { in filemap_fault()
3599 folio_unlock(folio); in filemap_fault()
3600 folio_put(folio); in filemap_fault()
3618 folio_unlock(folio); in filemap_fault()
3630 folio_unlock(folio); in filemap_fault()
3631 folio_put(folio); in filemap_fault()
3635 vmf->page = folio_file_page(folio, index); in filemap_fault()
3646 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); in filemap_fault()
3649 folio_put(folio); in filemap_fault()
3663 if (!IS_ERR(folio)) in filemap_fault()
3664 folio_put(folio); in filemap_fault()
3673 static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, in filemap_map_pmd() argument
3680 folio_unlock(folio); in filemap_map_pmd()
3681 folio_put(folio); in filemap_map_pmd()
3685 if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) { in filemap_map_pmd()
3686 struct page *page = folio_file_page(folio, start); in filemap_map_pmd()
3687 vm_fault_t ret = do_set_pmd(vmf, folio, page); in filemap_map_pmd()
3690 folio_unlock(folio); in filemap_map_pmd()
3701 static struct folio *next_uptodate_folio(struct xa_state *xas, in next_uptodate_folio()
3704 struct folio *folio = xas_next_entry(xas, end_pgoff); in next_uptodate_folio() local
3708 if (!folio) in next_uptodate_folio()
3710 if (xas_retry(xas, folio)) in next_uptodate_folio()
3712 if (xa_is_value(folio)) in next_uptodate_folio()
3714 if (!folio_try_get(folio)) in next_uptodate_folio()
3716 if (folio_test_locked(folio)) in next_uptodate_folio()
3719 if (unlikely(folio != xas_reload(xas))) in next_uptodate_folio()
3721 if (!folio_test_uptodate(folio) || folio_test_readahead(folio)) in next_uptodate_folio()
3723 if (!folio_trylock(folio)) in next_uptodate_folio()
3725 if (folio->mapping != mapping) in next_uptodate_folio()
3727 if (!folio_test_uptodate(folio)) in next_uptodate_folio()
3732 return folio; in next_uptodate_folio()
3734 folio_unlock(folio); in next_uptodate_folio()
3736 folio_put(folio); in next_uptodate_folio()
3737 } while ((folio = xas_next_entry(xas, end_pgoff)) != NULL); in next_uptodate_folio()
3747 struct folio *folio, unsigned long start, in filemap_map_folio_range() argument
3752 struct address_space *mapping = folio->mapping; in filemap_map_folio_range()
3755 struct page *page = folio_page(folio, start); in filemap_map_folio_range()
3769 if ((file_end >= folio_next_index(folio) || shmem_mapping(mapping)) && in filemap_map_folio_range()
3770 folio_within_vma(folio, vmf->vma) && in filemap_map_folio_range()
3771 (addr0 & PMD_MASK) == ((addr0 + folio_size(folio) - 1) & PMD_MASK)) { in filemap_map_folio_range()
3775 nr_pages = folio_nr_pages(folio); in filemap_map_folio_range()
3789 if (!folio_test_workingset(folio)) in filemap_map_folio_range()
3804 set_pte_range(vmf, folio, page, count, addr); in filemap_map_folio_range()
3806 folio_ref_add(folio, count - ref_from_caller); in filemap_map_folio_range()
3820 set_pte_range(vmf, folio, page, count, addr); in filemap_map_folio_range()
3822 folio_ref_add(folio, count - ref_from_caller); in filemap_map_folio_range()
3831 folio_ref_dec(folio); in filemap_map_folio_range()
3837 struct folio *folio, unsigned long addr, in filemap_map_order0_folio() argument
3841 struct page *page = &folio->page; in filemap_map_order0_folio()
3847 if (!folio_test_workingset(folio)) in filemap_map_order0_folio()
3861 set_pte_range(vmf, folio, page, 1, addr); in filemap_map_order0_folio()
3867 folio_ref_dec(folio); in filemap_map_order0_folio()
3880 struct folio *folio; in filemap_map_pages() local
3887 folio = next_uptodate_folio(&xas, mapping, end_pgoff); in filemap_map_pages()
3888 if (!folio) in filemap_map_pages()
3901 if ((file_end >= folio_next_index(folio) || shmem_mapping(mapping)) && in filemap_map_pages()
3902 filemap_map_pmd(vmf, folio, start_pgoff)) { in filemap_map_pages()
3910 folio_unlock(folio); in filemap_map_pages()
3911 folio_put(folio); in filemap_map_pages()
3915 folio_type = mm_counter_file(folio); in filemap_map_pages()
3922 end = folio_next_index(folio) - 1; in filemap_map_pages()
3925 if (!folio_test_large(folio)) in filemap_map_pages()
3927 folio, addr, &rss, &mmap_miss); in filemap_map_pages()
3929 ret |= filemap_map_folio_range(vmf, folio, in filemap_map_pages()
3930 xas.xa_index - folio->index, addr, in filemap_map_pages()
3933 folio_unlock(folio); in filemap_map_pages()
3934 } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL); in filemap_map_pages()
3954 struct folio *folio = page_folio(vmf->page); in filemap_page_mkwrite() local
3959 folio_lock(folio); in filemap_page_mkwrite()
3960 if (folio->mapping != mapping) { in filemap_page_mkwrite()
3961 folio_unlock(folio); in filemap_page_mkwrite()
3970 folio_mark_dirty(folio); in filemap_page_mkwrite()
3971 folio_wait_stable(folio); in filemap_page_mkwrite()
4053 static struct folio *do_read_cache_folio(struct address_space *mapping, in do_read_cache_folio()
4056 struct folio *folio; in do_read_cache_folio() local
4062 folio = filemap_get_folio(mapping, index); in do_read_cache_folio()
4063 if (IS_ERR(folio)) { in do_read_cache_folio()
4064 folio = filemap_alloc_folio(gfp, mapping_min_folio_order(mapping), NULL); in do_read_cache_folio()
4065 if (!folio) in do_read_cache_folio()
4068 err = filemap_add_folio(mapping, folio, index, gfp); in do_read_cache_folio()
4070 folio_put(folio); in do_read_cache_folio()
4079 if (folio_test_uptodate(folio)) in do_read_cache_folio()
4082 if (!folio_trylock(folio)) { in do_read_cache_folio()
4083 folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); in do_read_cache_folio()
4088 if (!folio->mapping) { in do_read_cache_folio()
4089 folio_unlock(folio); in do_read_cache_folio()
4090 folio_put(folio); in do_read_cache_folio()
4095 if (folio_test_uptodate(folio)) { in do_read_cache_folio()
4096 folio_unlock(folio); in do_read_cache_folio()
4101 err = filemap_read_folio(file, filler, folio); in do_read_cache_folio()
4103 folio_put(folio); in do_read_cache_folio()
4110 folio_mark_accessed(folio); in do_read_cache_folio()
4111 return folio; in do_read_cache_folio()
4130 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, in read_cache_folio()
4155 struct folio *mapping_read_folio_gfp(struct address_space *mapping, in mapping_read_folio_gfp()
4165 struct folio *folio; in do_read_cache_page() local
4167 folio = do_read_cache_folio(mapping, index, filler, file, gfp); in do_read_cache_page()
4168 if (IS_ERR(folio)) in do_read_cache_page()
4169 return &folio->page; in do_read_cache_page()
4170 return folio_file_page(folio, index); in do_read_cache_page()
4302 struct folio *folio; in generic_perform_write() local
4320 &folio, &fsdata); in generic_perform_write()
4324 offset = offset_in_folio(folio, pos); in generic_perform_write()
4325 if (bytes > folio_size(folio) - offset) in generic_perform_write()
4326 bytes = folio_size(folio) - offset; in generic_perform_write()
4329 flush_dcache_folio(folio); in generic_perform_write()
4337 copied = copy_folio_from_iter_atomic(folio, offset, bytes, i); in generic_perform_write()
4338 flush_dcache_folio(folio); in generic_perform_write()
4341 folio, fsdata); in generic_perform_write()
4488 bool filemap_release_folio(struct folio *folio, gfp_t gfp) in filemap_release_folio() argument
4490 struct address_space * const mapping = folio->mapping; in filemap_release_folio()
4492 BUG_ON(!folio_test_locked(folio)); in filemap_release_folio()
4493 if (!folio_needs_release(folio)) in filemap_release_folio()
4495 if (folio_test_writeback(folio)) in filemap_release_folio()
4499 return mapping->a_ops->release_folio(folio, gfp); in filemap_release_folio()
4500 return try_to_free_buffers(folio); in filemap_release_folio()
4567 struct folio *folio; in filemap_cachestat() local
4573 xas_for_each(&xas, folio, last_index) { in filemap_cachestat()
4589 if (xas_retry(&xas, folio)) in filemap_cachestat()
4604 if (xa_is_value(folio)) { in filemap_cachestat()
4606 void *shadow = (void *)folio; in filemap_cachestat()
4614 swp_entry_t swp = radix_to_swp_entry(folio); in filemap_cachestat()