Lines Matching +full:data +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
28 #include <linux/error-injection.h>
31 #include <linux/backing-dev.h>
63 * finished 'unifying' the page and buffer cache and SMP-threaded the
64 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
66 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
72 * ->i_mmap_rwsem (truncate_pagecache)
73 * ->private_lock (__free_pte->__set_page_dirty_buffers)
74 * ->swap_lock (exclusive_swap_page, others)
75 * ->i_pages lock
77 * ->i_mutex
78 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
80 * ->mmap_lock
81 * ->i_mmap_rwsem
82 * ->page_table_lock or pte_lock (various, mainly in memory.c)
83 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
85 * ->mmap_lock
86 * ->lock_page (access_process_vm)
88 * ->i_mutex (generic_perform_write)
89 * ->mmap_lock (fault_in_pages_readable->do_page_fault)
91 * bdi->wb.list_lock
92 * sb_lock (fs/fs-writeback.c)
93 * ->i_pages lock (__sync_single_inode)
95 * ->i_mmap_rwsem
96 * ->anon_vma.lock (vma_adjust)
98 * ->anon_vma.lock
99 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
101 * ->page_table_lock or pte_lock
102 * ->swap_lock (try_to_unmap_one)
103 * ->private_lock (try_to_unmap_one)
104 * ->i_pages lock (try_to_unmap_one)
105 * ->pgdat->lru_lock (follow_page->mark_page_accessed)
106 * ->pgdat->lru_lock (check_pte_range->isolate_lru_page)
107 * ->private_lock (page_remove_rmap->set_page_dirty)
108 * ->i_pages lock (page_remove_rmap->set_page_dirty)
109 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
110 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
111 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
112 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
113 * ->inode->i_lock (zap_pte_range->set_page_dirty)
114 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
116 * ->i_mmap_rwsem
117 * ->tasklist_lock (memory_failure, collect_procs_ao)
120 static void page_cache_delete(struct address_space *mapping, in page_cache_delete() argument
123 XA_STATE(xas, &mapping->i_pages, page->index); in page_cache_delete()
126 mapping_set_update(&xas, mapping); in page_cache_delete()
130 xas_set_order(&xas, page->index, compound_order(page)); in page_cache_delete()
141 page->mapping = NULL; in page_cache_delete()
142 /* Leave page->index set: truncation lookup relies upon it */ in page_cache_delete()
145 mapping->nrexceptional += nr; in page_cache_delete()
154 mapping->nrpages -= nr; in page_cache_delete()
157 static void unaccount_page_cache_page(struct address_space *mapping, in unaccount_page_cache_page() argument
165 * stale data around in the cleancache once our page is gone in unaccount_page_cache_page()
170 cleancache_invalidate_page(mapping, page); in unaccount_page_cache_page()
178 current->comm, page_to_pfn(page)); in unaccount_page_cache_page()
184 if (mapping_exiting(mapping) && in unaccount_page_cache_page()
203 __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr); in unaccount_page_cache_page()
205 __mod_lruvec_page_state(page, NR_SHMEM, -nr); in unaccount_page_cache_page()
210 filemap_nr_thps_dec(mapping); in unaccount_page_cache_page()
216 * unwritten data. in unaccount_page_cache_page()
224 account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); in unaccount_page_cache_page()
229 * sure the page is locked and that nobody else uses it - or that usage
234 struct address_space *mapping = page->mapping; in __delete_from_page_cache() local
238 unaccount_page_cache_page(mapping, page); in __delete_from_page_cache()
239 page_cache_delete(mapping, page, shadow); in __delete_from_page_cache()
242 static void page_cache_free_page(struct address_space *mapping, in page_cache_free_page() argument
247 freepage = mapping->a_ops->freepage; in page_cache_free_page()
260 * delete_from_page_cache - delete page from page cache
269 struct address_space *mapping = page_mapping(page); in delete_from_page_cache() local
273 xa_lock_irqsave(&mapping->i_pages, flags); in delete_from_page_cache()
275 xa_unlock_irqrestore(&mapping->i_pages, flags); in delete_from_page_cache()
277 page_cache_free_page(mapping, page); in delete_from_page_cache()
282 * page_cache_delete_batch - delete several pages from page cache
283 * @mapping: the mapping to which pages belong
286 * The function walks over mapping->i_pages and removes pages passed in @pvec
287 * from the mapping. The function expects @pvec to be sorted by page index
289 * It tolerates holes in @pvec (mapping entries at those indices are not
295 static void page_cache_delete_batch(struct address_space *mapping, in page_cache_delete_batch() argument
298 XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); in page_cache_delete_batch()
303 mapping_set_update(&xas, mapping); in page_cache_delete_batch()
318 if (page != pvec->pages[i]) { in page_cache_delete_batch()
319 VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index, in page_cache_delete_batch()
326 if (page->index == xas.xa_index) in page_cache_delete_batch()
327 page->mapping = NULL; in page_cache_delete_batch()
328 /* Leave page->index set: truncation lookup relies on it */ in page_cache_delete_batch()
332 * page or the index is of the last sub-page of this compound in page_cache_delete_batch()
335 if (page->index + compound_nr(page) - 1 == xas.xa_index) in page_cache_delete_batch()
340 mapping->nrpages -= total_pages; in page_cache_delete_batch()
343 void delete_from_page_cache_batch(struct address_space *mapping, in delete_from_page_cache_batch() argument
352 xa_lock_irqsave(&mapping->i_pages, flags); in delete_from_page_cache_batch()
354 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); in delete_from_page_cache_batch()
356 unaccount_page_cache_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
358 page_cache_delete_batch(mapping, pvec); in delete_from_page_cache_batch()
359 xa_unlock_irqrestore(&mapping->i_pages, flags); in delete_from_page_cache_batch()
362 page_cache_free_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
365 int filemap_check_errors(struct address_space *mapping) in filemap_check_errors() argument
369 if (test_bit(AS_ENOSPC, &mapping->flags) && in filemap_check_errors()
370 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_errors()
371 ret = -ENOSPC; in filemap_check_errors()
372 if (test_bit(AS_EIO, &mapping->flags) && in filemap_check_errors()
373 test_and_clear_bit(AS_EIO, &mapping->flags)) in filemap_check_errors()
374 ret = -EIO; in filemap_check_errors()
379 static int filemap_check_and_keep_errors(struct address_space *mapping) in filemap_check_and_keep_errors() argument
382 if (test_bit(AS_EIO, &mapping->flags)) in filemap_check_and_keep_errors()
383 return -EIO; in filemap_check_and_keep_errors()
384 if (test_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_and_keep_errors()
385 return -ENOSPC; in filemap_check_and_keep_errors()
390 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
391 * @mapping: address space structure to write
396 * Start writeback against all of a mapping's dirty pages that lie
399 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
406 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in __filemap_fdatawrite_range() argument
417 if (!mapping_can_writeback(mapping) || in __filemap_fdatawrite_range()
418 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) in __filemap_fdatawrite_range()
421 wbc_attach_fdatawrite_inode(&wbc, mapping->host); in __filemap_fdatawrite_range()
422 ret = do_writepages(mapping, &wbc); in __filemap_fdatawrite_range()
427 static inline int __filemap_fdatawrite(struct address_space *mapping, in __filemap_fdatawrite() argument
430 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); in __filemap_fdatawrite()
433 int filemap_fdatawrite(struct address_space *mapping) in filemap_fdatawrite() argument
435 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); in filemap_fdatawrite()
439 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in filemap_fdatawrite_range() argument
442 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); in filemap_fdatawrite_range()
447 * filemap_flush - mostly a non-blocking flush
448 * @mapping: target address_space
450 * This is a mostly non-blocking flush. Not suitable for data-integrity
451 * purposes - I/O may not be started against all dirty pages.
455 int filemap_flush(struct address_space *mapping) in filemap_flush() argument
457 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); in filemap_flush()
462 * filemap_range_has_page - check if a page exists in range.
463 * @mapping: address space within which to check
473 bool filemap_range_has_page(struct address_space *mapping, in filemap_range_has_page() argument
477 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_has_page()
504 static void __filemap_fdatawait_range(struct address_space *mapping, in __filemap_fdatawait_range() argument
519 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, in __filemap_fdatawait_range()
536 * filemap_fdatawait_range - wait for writeback to complete
537 * @mapping: address space structure to wait for
541 * Walk the list of under-writeback pages of the given address space
551 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, in filemap_fdatawait_range() argument
554 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range()
555 return filemap_check_errors(mapping); in filemap_fdatawait_range()
560 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
561 * @mapping: address space structure to wait for
565 * Walk the list of under-writeback pages of the given address space in the
570 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
573 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, in filemap_fdatawait_range_keep_errors() argument
576 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range_keep_errors()
577 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_range_keep_errors()
582 * file_fdatawait_range - wait for writeback to complete
587 * Walk the list of under-writeback pages of the address space that file
589 * status of the address space vs. the file->f_wb_err cursor and return it.
595 * Return: error status of the address space vs. the file->f_wb_err cursor.
599 struct address_space *mapping = file->f_mapping; in file_fdatawait_range() local
601 __filemap_fdatawait_range(mapping, start_byte, end_byte); in file_fdatawait_range()
607 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
608 * @mapping: address space structure to wait for
610 * Walk the list of under-writeback pages of the given address space
615 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
620 int filemap_fdatawait_keep_errors(struct address_space *mapping) in filemap_fdatawait_keep_errors() argument
622 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); in filemap_fdatawait_keep_errors()
623 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_keep_errors()
628 static bool mapping_needs_writeback(struct address_space *mapping) in mapping_needs_writeback() argument
630 if (dax_mapping(mapping)) in mapping_needs_writeback()
631 return mapping->nrexceptional; in mapping_needs_writeback()
633 return mapping->nrpages; in mapping_needs_writeback()
637 * filemap_write_and_wait_range - write out & wait on a file range
638 * @mapping: the address_space for the pages
642 * Write out and wait upon file offsets lstart->lend, inclusive.
645 * that this function can be used to write to the very end-of-file (end = -1).
649 int filemap_write_and_wait_range(struct address_space *mapping, in filemap_write_and_wait_range() argument
654 if (mapping_needs_writeback(mapping)) { in filemap_write_and_wait_range()
655 err = __filemap_fdatawrite_range(mapping, lstart, lend, in filemap_write_and_wait_range()
659 * written partially (e.g. -ENOSPC), so we wait for it. in filemap_write_and_wait_range()
660 * But the -EIO is special case, it may indicate the worst in filemap_write_and_wait_range()
663 if (err != -EIO) { in filemap_write_and_wait_range()
664 int err2 = filemap_fdatawait_range(mapping, in filemap_write_and_wait_range()
670 filemap_check_errors(mapping); in filemap_write_and_wait_range()
673 err = filemap_check_errors(mapping); in filemap_write_and_wait_range()
679 void __filemap_set_wb_err(struct address_space *mapping, int err) in __filemap_set_wb_err() argument
681 errseq_t eseq = errseq_set(&mapping->wb_err, err); in __filemap_set_wb_err()
683 trace_filemap_set_wb_err(mapping, eseq); in __filemap_set_wb_err()
688 * file_check_and_advance_wb_err - report wb error (if any) that was previously
696 * Grab the wb_err from the mapping. If it matches what we have in the file,
699 * If it doesn't match, then take the mapping value, set the "seen" flag in
705 * While we handle mapping->wb_err with atomic operations, the f_wb_err
714 errseq_t old = READ_ONCE(file->f_wb_err); in file_check_and_advance_wb_err()
715 struct address_space *mapping = file->f_mapping; in file_check_and_advance_wb_err() local
718 if (errseq_check(&mapping->wb_err, old)) { in file_check_and_advance_wb_err()
720 spin_lock(&file->f_lock); in file_check_and_advance_wb_err()
721 old = file->f_wb_err; in file_check_and_advance_wb_err()
722 err = errseq_check_and_advance(&mapping->wb_err, in file_check_and_advance_wb_err()
723 &file->f_wb_err); in file_check_and_advance_wb_err()
725 spin_unlock(&file->f_lock); in file_check_and_advance_wb_err()
733 clear_bit(AS_EIO, &mapping->flags); in file_check_and_advance_wb_err()
734 clear_bit(AS_ENOSPC, &mapping->flags); in file_check_and_advance_wb_err()
740 * file_write_and_wait_range - write out & wait on a file range
745 * Write out and wait upon file offsets lstart->lend, inclusive.
748 * that this function can be used to write to the very end-of-file (end = -1).
750 * After writing out and waiting on the data, we check and advance the
758 struct address_space *mapping = file->f_mapping; in file_write_and_wait_range() local
760 if (mapping_needs_writeback(mapping)) { in file_write_and_wait_range()
761 err = __filemap_fdatawrite_range(mapping, lstart, lend, in file_write_and_wait_range()
764 if (err != -EIO) in file_write_and_wait_range()
765 __filemap_fdatawait_range(mapping, lstart, lend); in file_write_and_wait_range()
775 * replace_page_cache_page - replace a pagecache page with a new one
792 struct address_space *mapping = old->mapping; in replace_page_cache_page() local
793 void (*freepage)(struct page *) = mapping->a_ops->freepage; in replace_page_cache_page()
794 pgoff_t offset = old->index; in replace_page_cache_page()
795 XA_STATE(xas, &mapping->i_pages, offset); in replace_page_cache_page()
800 VM_BUG_ON_PAGE(new->mapping, new); in replace_page_cache_page()
803 new->mapping = mapping; in replace_page_cache_page()
804 new->index = offset; in replace_page_cache_page()
811 old->mapping = NULL; in replace_page_cache_page()
831 struct address_space *mapping, in __add_to_page_cache_locked() argument
835 XA_STATE(xas, &mapping->i_pages, offset); in __add_to_page_cache_locked()
841 mapping_set_update(&xas, mapping); in __add_to_page_cache_locked()
844 page->mapping = mapping; in __add_to_page_cache_locked()
845 page->index = offset; in __add_to_page_cache_locked()
848 error = mem_cgroup_charge(page, current->mm, gfp); in __add_to_page_cache_locked()
866 xas_set_err(&xas, -EEXIST); in __add_to_page_cache_locked()
887 mapping->nrexceptional--; in __add_to_page_cache_locked()
888 mapping->nrpages++; in __add_to_page_cache_locked()
905 page->mapping = NULL; in __add_to_page_cache_locked()
906 /* Leave page->index set: truncation relies upon it */ in __add_to_page_cache_locked()
913 * add_to_page_cache_locked - add a locked page to the pagecache
915 * @mapping: the page's address_space
924 int add_to_page_cache_locked(struct page *page, struct address_space *mapping, in add_to_page_cache_locked() argument
927 return __add_to_page_cache_locked(page, mapping, offset, in add_to_page_cache_locked()
932 int add_to_page_cache_lru(struct page *page, struct address_space *mapping, in add_to_page_cache_lru() argument
939 ret = __add_to_page_cache_locked(page, mapping, offset, in add_to_page_cache_lru()
949 * data from the working set, only to cache data that will in add_to_page_cache_lru()
1012 * The page wait code treats the "wait->flags" somewhat unusually, because
1059 flags = wait->flags; in wake_page_function()
1061 if (test_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1062 return -1; in wake_page_function()
1064 if (test_and_set_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1065 return -1; in wake_page_function()
1071 * We are holding the wait-queue lock, but the waiter that in wake_page_function()
1076 * afterwards to avoid any races. This store-release pairs in wake_page_function()
1077 * with the load-acquire in wait_on_page_bit_common(). in wake_page_function()
1079 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); in wake_page_function()
1080 wake_up_state(wait->private, mode); in wake_page_function()
1088 * After this list_del_init(&wait->entry) the wait entry in wake_page_function()
1089 * might be de-allocated and the process might even have in wake_page_function()
1092 list_del_init_careful(&wait->entry); in wake_page_function()
1112 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1122 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1124 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1130 * hash, so in that case check for a page match. That prevents a long- in wake_up_page_bit()
1147 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1179 if (wait->flags & WQ_FLAG_EXCLUSIVE) { in trylock_page_bit_common()
1180 if (test_and_set_bit(bit_nr, &page->flags)) in trylock_page_bit_common()
1182 } else if (test_bit(bit_nr, &page->flags)) in trylock_page_bit_common()
1185 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; in trylock_page_bit_common()
1213 wait->func = wake_page_function; in wait_on_page_bit_common()
1218 wait->flags = 0; in wait_on_page_bit_common()
1220 wait->flags = WQ_FLAG_EXCLUSIVE; in wait_on_page_bit_common()
1221 if (--unfairness < 0) in wait_on_page_bit_common()
1222 wait->flags |= WQ_FLAG_CUSTOM; in wait_on_page_bit_common()
1239 spin_lock_irq(&q->lock); in wait_on_page_bit_common()
1243 spin_unlock_irq(&q->lock); in wait_on_page_bit_common()
1259 * be very careful with the 'wait->flags', because in wait_on_page_bit_common()
1268 flags = smp_load_acquire(&wait->flags); in wait_on_page_bit_common()
1277 /* If we were non-exclusive, we're done */ in wait_on_page_bit_common()
1291 if (unlikely(test_and_set_bit(bit_nr, &page->flags))) in wait_on_page_bit_common()
1294 wait->flags |= WQ_FLAG_DONE; in wait_on_page_bit_common()
1300 * waiter from the wait-queues, but the PageWaiters bit will remain in wait_on_page_bit_common()
1313 * NOTE! The wait->flags weren't stable until we've done the in wait_on_page_bit_common()
1322 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive in wait_on_page_bit_common()
1326 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; in wait_on_page_bit_common()
1328 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; in wait_on_page_bit_common()
1351 wait->page = page; in __wait_on_page_locked_async()
1352 wait->bit_nr = PG_locked; in __wait_on_page_locked_async()
1354 spin_lock_irq(&q->lock); in __wait_on_page_locked_async()
1355 __add_wait_queue_entry_tail(q, &wait->wait); in __wait_on_page_locked_async()
1368 __remove_wait_queue(q, &wait->wait); in __wait_on_page_locked_async()
1370 ret = -EIOCBQUEUED; in __wait_on_page_locked_async()
1371 spin_unlock_irq(&q->lock); in __wait_on_page_locked_async()
1384 * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1403 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
1414 spin_lock_irqsave(&q->lock, flags); in add_page_wait_queue()
1417 spin_unlock_irqrestore(&q->lock, flags); in add_page_wait_queue()
1445 * unlock_page - unlock a locked page
1451 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1454 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1464 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags)) in unlock_page()
1470 * end_page_writeback - end writeback against a page
1519 struct address_space *mapping; in page_endio() local
1522 mapping = page_mapping(page); in page_endio()
1523 if (mapping) in page_endio()
1524 mapping_set_error(mapping, err); in page_endio()
1532 * __lock_page - get a lock on the page, assuming we need to sleep to get it
1560 * 1 - page is locked; mmap_lock is still held.
1561 * 0 - page is not locked.
1602 * page_cache_next_miss() - Find the next gap in the page cache.
1603 * @mapping: Mapping.
1607 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1617 * range specified (in which case 'return - index >= max_scan' will be true).
1618 * In the rare case of index wrap-around, 0 will be returned.
1620 pgoff_t page_cache_next_miss(struct address_space *mapping, in page_cache_next_miss() argument
1623 XA_STATE(xas, &mapping->i_pages, index); in page_cache_next_miss()
1625 while (max_scan--) { in page_cache_next_miss()
1638 * page_cache_prev_miss() - Find the previous gap in the page cache.
1639 * @mapping: Mapping.
1643 * Search the range [max(index - max_scan + 1, 0), index] for the
1653 * range specified (in which case 'index - return >= max_scan' will be true).
1654 * In the rare case of wrap-around, ULONG_MAX will be returned.
1656 pgoff_t page_cache_prev_miss(struct address_space *mapping, in page_cache_prev_miss() argument
1659 XA_STATE(xas, &mapping->i_pages, index); in page_cache_prev_miss()
1661 while (max_scan--) { in page_cache_prev_miss()
1674 * find_get_entry - find and get a page cache entry
1675 * @mapping: the address_space to search
1678 * Looks up the page cache slot at @mapping & @offset. If there is a
1686 struct page *find_get_entry(struct address_space *mapping, pgoff_t index) in find_get_entry() argument
1688 XA_STATE(xas, &mapping->i_pages, index); in find_get_entry()
1723 * find_lock_entry - Locate and lock a page cache entry.
1724 * @mapping: The address_space to search.
1727 * Looks up the page at @mapping & @index. If there is a page in the
1736 struct page *find_lock_entry(struct address_space *mapping, pgoff_t index) in find_lock_entry() argument
1741 page = find_get_entry(mapping, index); in find_lock_entry()
1745 if (unlikely(page->mapping != mapping)) { in find_lock_entry()
1756 * pagecache_get_page - Find and get a reference to a page.
1757 * @mapping: The address_space to search.
1762 * Looks up the page cache entry at @mapping & @index.
1766 * * %FGP_ACCESSED - The page will be marked accessed.
1767 * * %FGP_LOCK - The page is returned locked.
1768 * * %FGP_HEAD - If the page is present and a THP, return the head page
1770 * * %FGP_CREAT - If no page is present then a new page is allocated using
1773 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1776 * * %FGP_WRITE - The page will be written
1777 * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask
1778 * * %FGP_NOWAIT - Don't get blocked by page lock
1787 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, in pagecache_get_page() argument
1793 page = find_get_entry(mapping, index); in pagecache_get_page()
1810 if (unlikely(page->mapping != mapping)) { in pagecache_get_page()
1831 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) in pagecache_get_page()
1847 err = add_to_page_cache_lru(page, mapping, index, gfp_mask); in pagecache_get_page()
1851 if (err == -EEXIST) in pagecache_get_page()
1868 * find_get_entries - gang pagecache lookup
1869 * @mapping: The address_space to search
1876 * @nr_entries entries in the mapping. The entries are placed at
1880 * The search returns a group of mapping-contiguous page cache entries
1882 * not-present pages.
1894 unsigned find_get_entries(struct address_space *mapping, in find_get_entries() argument
1898 XA_STATE(xas, &mapping->i_pages, start); in find_get_entries()
1948 * find_get_pages_range - gang pagecache lookup
1949 * @mapping: The address_space to search
1956 * pages in the mapping starting at index @start and up to index @end
1960 * The search returns a group of mapping-contiguous pages with ascending
1961 * indexes. There may be holes in the indices due to not-present pages.
1968 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, in find_get_pages_range() argument
1972 XA_STATE(xas, &mapping->i_pages, *start); in find_get_pages_range()
2009 * breaks the iteration when there is a page at index -1 but that is in find_get_pages_range()
2012 if (end == (pgoff_t)-1) in find_get_pages_range()
2013 *start = (pgoff_t)-1; in find_get_pages_range()
2023 * find_get_pages_contig - gang contiguous pagecache lookup
2024 * @mapping: The address_space to search
2034 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, in find_get_pages_contig() argument
2037 XA_STATE(xas, &mapping->i_pages, index); in find_get_pages_contig()
2077 * find_get_pages_range_tag - find and return pages in given range matching @tag
2078 * @mapping: the address_space to search
2090 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, in find_get_pages_range_tag() argument
2094 XA_STATE(xas, &mapping->i_pages, *index); in find_get_pages_range_tag()
2135 * iteration when there is a page at index -1 but that is already in find_get_pages_range_tag()
2138 if (end == (pgoff_t)-1) in find_get_pages_range_tag()
2139 *index = (pgoff_t)-1; in find_get_pages_range_tag()
2153 * ---R__________________________________________B__________
2166 ra->ra_pages /= 4; in shrink_readahead_size_eio()
2170 * generic_file_buffered_read - generic file read routine
2172 * @iter: data destination
2176 * mapping->a_ops->readpage() function for the actual low-level stuff.
2188 struct file *filp = iocb->ki_filp; in generic_file_buffered_read()
2189 struct address_space *mapping = filp->f_mapping; in generic_file_buffered_read() local
2190 struct inode *inode = mapping->host; in generic_file_buffered_read()
2191 struct file_ra_state *ra = &filp->f_ra; in generic_file_buffered_read()
2192 loff_t *ppos = &iocb->ki_pos; in generic_file_buffered_read()
2200 if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) in generic_file_buffered_read()
2202 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); in generic_file_buffered_read()
2205 prev_index = ra->prev_pos >> PAGE_SHIFT; in generic_file_buffered_read()
2206 prev_offset = ra->prev_pos & (PAGE_SIZE-1); in generic_file_buffered_read()
2207 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; in generic_file_buffered_read()
2211 * If we've already successfully copied some data, then we in generic_file_buffered_read()
2212 * can no longer safely return -EIOCBQUEUED. Hence mark in generic_file_buffered_read()
2215 if (written && (iocb->ki_flags & IOCB_WAITQ)) in generic_file_buffered_read()
2216 iocb->ki_flags |= IOCB_NOWAIT; in generic_file_buffered_read()
2227 error = -EINTR; in generic_file_buffered_read()
2231 page = find_get_page(mapping, index); in generic_file_buffered_read()
2233 if (iocb->ki_flags & IOCB_NOIO) in generic_file_buffered_read()
2235 page_cache_sync_readahead(mapping, in generic_file_buffered_read()
2237 index, last_index - index); in generic_file_buffered_read()
2238 page = find_get_page(mapping, index); in generic_file_buffered_read()
2243 if (iocb->ki_flags & IOCB_NOIO) { in generic_file_buffered_read()
2247 page_cache_async_readahead(mapping, in generic_file_buffered_read()
2249 index, last_index - index); in generic_file_buffered_read()
2257 if (iocb->ki_flags & IOCB_WAITQ) { in generic_file_buffered_read()
2263 iocb->ki_waitq); in generic_file_buffered_read()
2265 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_buffered_read()
2276 if (inode->i_blkbits == PAGE_SHIFT || in generic_file_buffered_read()
2277 !mapping->a_ops->is_partially_uptodate) in generic_file_buffered_read()
2285 if (!page->mapping) in generic_file_buffered_read()
2287 if (!mapping->a_ops->is_partially_uptodate(page, in generic_file_buffered_read()
2288 offset, iter->count)) in generic_file_buffered_read()
2297 * the correct value for "nr", which means the zero-filled in generic_file_buffered_read()
2299 * another truncate extends the file - this is desired though). in generic_file_buffered_read()
2303 end_index = (isize - 1) >> PAGE_SHIFT; in generic_file_buffered_read()
2312 nr = ((isize - 1) & ~PAGE_MASK) + 1; in generic_file_buffered_read()
2318 nr = nr - offset; in generic_file_buffered_read()
2324 if (mapping_writably_mapped(mapping)) in generic_file_buffered_read()
2336 * Ok, we have the page, and it's up-to-date, so in generic_file_buffered_read()
2351 error = -EFAULT; in generic_file_buffered_read()
2358 if (iocb->ki_flags & IOCB_WAITQ) { in generic_file_buffered_read()
2363 error = lock_page_async(page, iocb->ki_waitq); in generic_file_buffered_read()
2372 if (!page->mapping) { in generic_file_buffered_read()
2385 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT)) { in generic_file_buffered_read()
2397 error = mapping->a_ops->readpage(filp, page); in generic_file_buffered_read()
2409 if (iocb->ki_flags & IOCB_WAITQ) { in generic_file_buffered_read()
2414 error = lock_page_async(page, iocb->ki_waitq); in generic_file_buffered_read()
2422 if (page->mapping == NULL) { in generic_file_buffered_read()
2432 error = -EIO; in generic_file_buffered_read()
2450 page = page_cache_alloc(mapping); in generic_file_buffered_read()
2452 error = -ENOMEM; in generic_file_buffered_read()
2455 error = add_to_page_cache_lru(page, mapping, index, in generic_file_buffered_read()
2456 mapping_gfp_constraint(mapping, GFP_KERNEL)); in generic_file_buffered_read()
2459 if (error == -EEXIST) { in generic_file_buffered_read()
2469 error = -EAGAIN; in generic_file_buffered_read()
2471 ra->prev_pos = prev_index; in generic_file_buffered_read()
2472 ra->prev_pos <<= PAGE_SHIFT; in generic_file_buffered_read()
2473 ra->prev_pos |= prev_offset; in generic_file_buffered_read()
2482 * generic_file_read_iter - generic filesystem read routine
2484 * @iter: destination for the data read
2489 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2490 * be returned when no data can be read without waiting for I/O requests
2493 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2494 * requests shall be made for the read or for readahead. When no data
2495 * can be read, -EAGAIN shall be returned. When readahead would be
2511 if (iocb->ki_flags & IOCB_DIRECT) { in generic_file_read_iter()
2512 struct file *file = iocb->ki_filp; in generic_file_read_iter()
2513 struct address_space *mapping = file->f_mapping; in generic_file_read_iter() local
2514 struct inode *inode = mapping->host; in generic_file_read_iter()
2518 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_read_iter()
2519 if (filemap_range_has_page(mapping, iocb->ki_pos, in generic_file_read_iter()
2520 iocb->ki_pos + count - 1)) in generic_file_read_iter()
2521 return -EAGAIN; in generic_file_read_iter()
2523 retval = filemap_write_and_wait_range(mapping, in generic_file_read_iter()
2524 iocb->ki_pos, in generic_file_read_iter()
2525 iocb->ki_pos + count - 1); in generic_file_read_iter()
2532 retval = mapping->a_ops->direct_IO(iocb, iter); in generic_file_read_iter()
2534 iocb->ki_pos += retval; in generic_file_read_iter()
2535 count -= retval; in generic_file_read_iter()
2537 iov_iter_revert(iter, count - iov_iter_count(iter)); in generic_file_read_iter()
2548 if (retval < 0 || !count || iocb->ki_pos >= size || in generic_file_read_iter()
2562 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2563 * @vmf - the vm_fault for this fault.
2564 * @page - the page to lock.
2565 * @fpin - the pointer to the file we may pin (or is already pinned).
2583 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in lock_page_maybe_drop_mmap()
2587 if (vmf->flags & FAULT_FLAG_KILLABLE) { in lock_page_maybe_drop_mmap()
2596 mmap_read_unlock(vmf->vma->vm_mm); in lock_page_maybe_drop_mmap()
2614 struct file *file = vmf->vma->vm_file; in do_sync_mmap_readahead()
2615 struct file_ra_state *ra = &file->f_ra; in do_sync_mmap_readahead()
2616 struct address_space *mapping = file->f_mapping; in do_sync_mmap_readahead() local
2617 DEFINE_READAHEAD(ractl, file, mapping, vmf->pgoff); in do_sync_mmap_readahead()
2621 /* If we don't want any read-ahead, don't bother */ in do_sync_mmap_readahead()
2622 if (vmf->vma->vm_flags & VM_RAND_READ) in do_sync_mmap_readahead()
2624 if (!ra->ra_pages) in do_sync_mmap_readahead()
2627 if (vmf->vma->vm_flags & VM_SEQ_READ) { in do_sync_mmap_readahead()
2629 page_cache_sync_ra(&ractl, ra, ra->ra_pages); in do_sync_mmap_readahead()
2634 mmap_miss = READ_ONCE(ra->mmap_miss); in do_sync_mmap_readahead()
2636 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); in do_sync_mmap_readahead()
2640 * stop bothering with read-ahead. It will only hurt. in do_sync_mmap_readahead()
2646 * mmap read-around in do_sync_mmap_readahead()
2649 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); in do_sync_mmap_readahead()
2650 ra->size = ra->ra_pages; in do_sync_mmap_readahead()
2651 ra->async_size = ra->ra_pages / 4; in do_sync_mmap_readahead()
2652 ractl._index = ra->start; in do_sync_mmap_readahead()
2653 do_page_cache_ra(&ractl, ra->size, ra->async_size); in do_sync_mmap_readahead()
2665 struct file *file = vmf->vma->vm_file; in do_async_mmap_readahead()
2666 struct file_ra_state *ra = &file->f_ra; in do_async_mmap_readahead()
2667 struct address_space *mapping = file->f_mapping; in do_async_mmap_readahead() local
2670 pgoff_t offset = vmf->pgoff; in do_async_mmap_readahead()
2672 /* If we don't want any read-ahead, don't bother */ in do_async_mmap_readahead()
2673 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) in do_async_mmap_readahead()
2675 mmap_miss = READ_ONCE(ra->mmap_miss); in do_async_mmap_readahead()
2677 WRITE_ONCE(ra->mmap_miss, --mmap_miss); in do_async_mmap_readahead()
2680 page_cache_async_readahead(mapping, ra, file, in do_async_mmap_readahead()
2681 page, offset, ra->ra_pages); in do_async_mmap_readahead()
2687 * filemap_fault - read in file data for page fault handling
2691 * mapped memory region to read in file data during a page fault.
2697 * vma->vm_mm->mmap_lock must be held on entry.
2707 * Return: bitwise-OR of %VM_FAULT_ codes.
2712 struct file *file = vmf->vma->vm_file; in filemap_fault()
2714 struct address_space *mapping = file->f_mapping; in filemap_fault() local
2715 struct file_ra_state *ra = &file->f_ra; in filemap_fault()
2716 struct inode *inode = mapping->host; in filemap_fault()
2717 pgoff_t offset = vmf->pgoff; in filemap_fault()
2729 page = find_get_page(mapping, offset); in filemap_fault()
2730 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { in filemap_fault()
2739 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); in filemap_fault()
2743 page = pagecache_get_page(mapping, offset, in filemap_fault()
2745 vmf->gfp_mask); in filemap_fault()
2757 if (unlikely(compound_head(page)->mapping != mapping)) { in filemap_fault()
2766 * that it's up-to-date. If not, it is going to be due to an error. in filemap_fault()
2773 * time to return to the upper layer and have it re-find the vma and in filemap_fault()
2792 vmf->page = page; in filemap_fault()
2797 * Umm, take care of errors if the page isn't up-to-date. in filemap_fault()
2798 * Try to re-read it _once_. We do this synchronously, in filemap_fault()
2804 error = mapping->a_ops->readpage(file, page); in filemap_fault()
2808 error = -EIO; in filemap_fault()
2823 * re-find the vma and come back and find our hopefully still populated in filemap_fault()
2837 struct file *file = vmf->vma->vm_file; in filemap_map_pages()
2838 struct address_space *mapping = file->f_mapping; in filemap_map_pages() local
2841 XA_STATE(xas, &mapping->i_pages, start_pgoff); in filemap_map_pages()
2843 unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); in filemap_map_pages()
2873 if (head->mapping != mapping || !PageUptodate(head)) in filemap_map_pages()
2876 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in filemap_map_pages()
2881 mmap_miss--; in filemap_map_pages()
2883 vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT; in filemap_map_pages()
2884 if (vmf->pte) in filemap_map_pages()
2885 vmf->pte += xas.xa_index - last_pgoff; in filemap_map_pages()
2897 if (pmd_trans_huge(*vmf->pmd)) in filemap_map_pages()
2901 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); in filemap_map_pages()
2907 struct page *page = vmf->page; in filemap_page_mkwrite()
2908 struct inode *inode = file_inode(vmf->vma->vm_file); in filemap_page_mkwrite()
2911 sb_start_pagefault(inode->i_sb); in filemap_page_mkwrite()
2912 file_update_time(vmf->vma->vm_file); in filemap_page_mkwrite()
2914 if (page->mapping != inode->i_mapping) { in filemap_page_mkwrite()
2927 sb_end_pagefault(inode->i_sb); in filemap_page_mkwrite()
2941 struct address_space *mapping = file->f_mapping; in generic_file_mmap() local
2943 if (!mapping->a_ops->readpage) in generic_file_mmap()
2944 return -ENOEXEC; in generic_file_mmap()
2946 vma->vm_ops = &generic_file_vm_ops; in generic_file_mmap()
2951 * This is for filesystems which do not implement ->writepage.
2955 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) in generic_file_readonly_mmap()
2956 return -EINVAL; in generic_file_readonly_mmap()
2966 return -ENOSYS; in generic_file_mmap()
2970 return -ENOSYS; in generic_file_readonly_mmap()
2984 page = ERR_PTR(-EIO); in wait_on_page_read()
2990 static struct page *do_read_cache_page(struct address_space *mapping, in do_read_cache_page() argument
2993 void *data, in do_read_cache_page() argument
2999 page = find_get_page(mapping, index); in do_read_cache_page()
3003 return ERR_PTR(-ENOMEM); in do_read_cache_page()
3004 err = add_to_page_cache_lru(page, mapping, index, gfp); in do_read_cache_page()
3007 if (err == -EEXIST) in do_read_cache_page()
3015 err = filler(data, page); in do_read_cache_page()
3017 err = mapping->a_ops->readpage(data, page); in do_read_cache_page()
3042 * page is truncated, the data is still valid if PageUptodate as in do_read_cache_page()
3045 * Case c, the page may be truncated but in itself, the data may still in do_read_cache_page()
3048 * otherwise serialising on page lock to stabilise the mapping gives in do_read_cache_page()
3052 * will be a race with remove_mapping that determines if the mapping in do_read_cache_page()
3053 * is valid on unlock but otherwise the data is valid and there is in do_read_cache_page()
3071 if (!page->mapping) { in do_read_cache_page()
3098 * read_cache_page - read into page cache, fill it if needed
3099 * @mapping: the page's address_space
3102 * @data: first arg to filler(data, page) function, often left as NULL
3107 * If the page does not get brought uptodate, return -EIO.
3111 struct page *read_cache_page(struct address_space *mapping, in read_cache_page() argument
3114 void *data) in read_cache_page() argument
3116 return do_read_cache_page(mapping, index, filler, data, in read_cache_page()
3117 mapping_gfp_mask(mapping)); in read_cache_page()
3122 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3123 * @mapping: the page's address_space
3127 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3130 * If the page does not get brought uptodate, return -EIO.
3134 struct page *read_cache_page_gfp(struct address_space *mapping, in read_cache_page_gfp() argument
3138 return do_read_cache_page(mapping, index, NULL, NULL, gfp); in read_cache_page_gfp()
3142 int pagecache_write_begin(struct file *file, struct address_space *mapping, in pagecache_write_begin() argument
3146 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_begin()
3148 return aops->write_begin(file, mapping, pos, len, flags, in pagecache_write_begin()
3153 int pagecache_write_end(struct file *file, struct address_space *mapping, in pagecache_write_end() argument
3157 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_end()
3159 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); in pagecache_write_end()
3173 errseq_set(&inode->i_mapping->wb_err, -EIO); in dio_warn_stale_pagecache()
3178 …pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision… in dio_warn_stale_pagecache()
3179 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, in dio_warn_stale_pagecache()
3180 current->comm); in dio_warn_stale_pagecache()
3187 struct file *file = iocb->ki_filp; in generic_file_direct_write()
3188 struct address_space *mapping = file->f_mapping; in generic_file_direct_write() local
3189 struct inode *inode = mapping->host; in generic_file_direct_write()
3190 loff_t pos = iocb->ki_pos; in generic_file_direct_write()
3196 end = (pos + write_len - 1) >> PAGE_SHIFT; in generic_file_direct_write()
3198 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_direct_write()
3200 if (filemap_range_has_page(inode->i_mapping, pos, in generic_file_direct_write()
3201 pos + write_len - 1)) in generic_file_direct_write()
3202 return -EAGAIN; in generic_file_direct_write()
3204 written = filemap_write_and_wait_range(mapping, pos, in generic_file_direct_write()
3205 pos + write_len - 1); in generic_file_direct_write()
3212 * the new data. We invalidate clean cached page from the region we're in generic_file_direct_write()
3214 * without clobbering -EIOCBQUEUED from ->direct_IO(). in generic_file_direct_write()
3216 written = invalidate_inode_pages2_range(mapping, in generic_file_direct_write()
3223 if (written == -EBUSY) in generic_file_direct_write()
3228 written = mapping->a_ops->direct_IO(iocb, from); in generic_file_direct_write()
3232 * cached by non-direct readahead, or faulted in by get_user_pages() in generic_file_direct_write()
3245 * Skip invalidation for async writes or if mapping has no pages. in generic_file_direct_write()
3247 if (written > 0 && mapping->nrpages && in generic_file_direct_write()
3248 invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end)) in generic_file_direct_write()
3253 write_len -= written; in generic_file_direct_write()
3254 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { in generic_file_direct_write()
3258 iocb->ki_pos = pos; in generic_file_direct_write()
3260 iov_iter_revert(from, write_len - iov_iter_count(from)); in generic_file_direct_write()
3270 struct page *grab_cache_page_write_begin(struct address_space *mapping, in grab_cache_page_write_begin() argument
3279 page = pagecache_get_page(mapping, index, fgp_flags, in grab_cache_page_write_begin()
3280 mapping_gfp_mask(mapping)); in grab_cache_page_write_begin()
3291 struct address_space *mapping = file->f_mapping; in generic_perform_write() local
3292 const struct address_space_operations *a_ops = mapping->a_ops; in generic_perform_write()
3304 offset = (pos & (PAGE_SIZE - 1)); in generic_perform_write()
3305 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3313 * up-to-date. in generic_perform_write()
3320 status = -EFAULT; in generic_perform_write()
3325 status = -EINTR; in generic_perform_write()
3329 status = a_ops->write_begin(file, mapping, pos, bytes, flags, in generic_perform_write()
3334 if (mapping_writably_mapped(mapping)) in generic_perform_write()
3340 status = a_ops->write_end(file, mapping, pos, bytes, copied, in generic_perform_write()
3351 * If we were unable to copy any data at all, we must in generic_perform_write()
3358 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3365 balance_dirty_pages_ratelimited(mapping); in generic_perform_write()
3373 * __generic_file_write_iter - write data to a file
3375 * @from: iov_iter with data to write
3377 * This function does all the work needed for actually writing data to a
3385 * This function does *not* take care of syncing data in case of O_SYNC write.
3391 * * negative error code if no data has been written at all
3395 struct file *file = iocb->ki_filp; in __generic_file_write_iter()
3396 struct address_space * mapping = file->f_mapping; in __generic_file_write_iter() local
3397 struct inode *inode = mapping->host; in __generic_file_write_iter()
3403 current->backing_dev_info = inode_to_bdi(inode); in __generic_file_write_iter()
3412 if (iocb->ki_flags & IOCB_DIRECT) { in __generic_file_write_iter()
3421 * page-cache pages correctly). in __generic_file_write_iter()
3426 status = generic_perform_write(file, from, pos = iocb->ki_pos); in __generic_file_write_iter()
3430 * direct-written, or the error code if that was zero. Note in __generic_file_write_iter()
3431 * that this differs from normal direct-io semantics, which in __generic_file_write_iter()
3432 * will return -EFOO even if some bytes were written. in __generic_file_write_iter()
3443 endbyte = pos + status - 1; in __generic_file_write_iter()
3444 err = filemap_write_and_wait_range(mapping, pos, endbyte); in __generic_file_write_iter()
3446 iocb->ki_pos = endbyte + 1; in __generic_file_write_iter()
3448 invalidate_mapping_pages(mapping, in __generic_file_write_iter()
3454 * the number of bytes which were direct-written in __generic_file_write_iter()
3458 written = generic_perform_write(file, from, iocb->ki_pos); in __generic_file_write_iter()
3460 iocb->ki_pos += written; in __generic_file_write_iter()
3463 current->backing_dev_info = NULL; in __generic_file_write_iter()
3469 * generic_file_write_iter - write data to a file
3471 * @from: iov_iter with data to write
3477 * * negative error code if no data has been written at all of
3483 struct file *file = iocb->ki_filp; in generic_file_write_iter()
3484 struct inode *inode = file->f_mapping->host; in generic_file_write_iter()
3500 * try_to_release_page() - release old fs-specific metadata on a page
3505 * The address_space is to try to release any data against the page
3506 * (presumably at page->private).
3518 struct address_space * const mapping = page->mapping; in try_to_release_page() local
3524 if (mapping && mapping->a_ops->releasepage) in try_to_release_page()
3525 return mapping->a_ops->releasepage(page, gfp_mask); in try_to_release_page()