31 static inline void __clear_shadow_entry(struct address_space *mapping,  in __clear_shadow_entry()  argument
34 	XA_STATE(xas, &mapping->i_pages, index);  in __clear_shadow_entry()
42 static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,  in clear_shadow_entry()  argument
45 	spin_lock(&mapping->host->i_lock);  in clear_shadow_entry()
46 	xa_lock_irq(&mapping->i_pages);  in clear_shadow_entry()
47 	__clear_shadow_entry(mapping, index, entry);  in clear_shadow_entry()
48 	xa_unlock_irq(&mapping->i_pages);  in clear_shadow_entry()
49 	if (mapping_shrinkable(mapping))  in clear_shadow_entry()
50 		inode_add_lru(mapping->host);  in clear_shadow_entry()
51 	spin_unlock(&mapping->host->i_lock);  in clear_shadow_entry()
59 static void truncate_folio_batch_exceptionals(struct address_space *mapping,  in truncate_folio_batch_exceptionals()  argument
66 	if (shmem_mapping(mapping))  in truncate_folio_batch_exceptionals()
76 	dax = dax_mapping(mapping);  in truncate_folio_batch_exceptionals()
78 		spin_lock(&mapping->host->i_lock);  in truncate_folio_batch_exceptionals()
79 		xa_lock_irq(&mapping->i_pages);  in truncate_folio_batch_exceptionals()
92 			dax_delete_mapping_entry(mapping, index);  in truncate_folio_batch_exceptionals()
96 		__clear_shadow_entry(mapping, index, folio);  in truncate_folio_batch_exceptionals()
100 		xa_unlock_irq(&mapping->i_pages);  in truncate_folio_batch_exceptionals()
101 		if (mapping_shrinkable(mapping))  in truncate_folio_batch_exceptionals()
102 			inode_add_lru(mapping->host);  in truncate_folio_batch_exceptionals()
103 		spin_unlock(&mapping->host->i_lock);  in truncate_folio_batch_exceptionals()
112 static int invalidate_exceptional_entry(struct address_space *mapping,  in invalidate_exceptional_entry()  argument
116 	if (shmem_mapping(mapping) || dax_mapping(mapping))  in invalidate_exceptional_entry()
118 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry()
126 static int invalidate_exceptional_entry2(struct address_space *mapping,  in invalidate_exceptional_entry2()  argument
130 	if (shmem_mapping(mapping))  in invalidate_exceptional_entry2()
132 	if (dax_mapping(mapping))  in invalidate_exceptional_entry2()
133 		return dax_invalidate_mapping_entry_sync(mapping, index);  in invalidate_exceptional_entry2()
134 	clear_shadow_entry(mapping, index, entry);  in invalidate_exceptional_entry2()
155 	const struct address_space_operations *aops = folio->mapping->a_ops;  in folio_invalidate()
167  * We need to bail out if page->mapping is no longer equal to the original
168  * mapping.  This happens a) when the VM reclaimed the page while we waited on
189 int truncate_inode_folio(struct address_space *mapping, struct folio *folio)  in truncate_inode_folio()  argument
191 	if (folio->mapping != mapping)  in truncate_inode_folio()
227 		truncate_inode_folio(folio->mapping, folio);  in truncate_inode_partial_folio()
246 	truncate_inode_folio(folio->mapping, folio);  in truncate_inode_partial_folio()
253 int generic_error_remove_folio(struct address_space *mapping,  in generic_error_remove_folio()  argument
256 	if (!mapping)  in generic_error_remove_folio()
262 	if (!S_ISREG(mapping->host->i_mode))  in generic_error_remove_folio()
264 	return truncate_inode_folio(mapping, folio);  in generic_error_remove_folio()
270  * @mapping: The mapping this folio belongs to.
279 long mapping_evict_folio(struct address_space *mapping, struct folio *folio)  in mapping_evict_folio()  argument
282 	if (!mapping)  in mapping_evict_folio()
293 	return remove_mapping(mapping, folio);  in mapping_evict_folio()
298  * @mapping: mapping to truncate
313  * mapping is large, it is probably the case that the final pages are the most
320 void truncate_inode_pages_range(struct address_space *mapping,  in truncate_inode_pages_range()  argument
332 	if (mapping_empty(mapping))  in truncate_inode_pages_range()
354 	while (index < end && find_lock_entries(mapping, &index, end - 1,  in truncate_inode_pages_range()
356 		truncate_folio_batch_exceptionals(mapping, &fbatch, indices);  in truncate_inode_pages_range()
359 		delete_from_page_cache_batch(mapping, &fbatch);  in truncate_inode_pages_range()
367 	folio = __filemap_get_folio(mapping, lstart >> PAGE_SHIFT, FGP_LOCK, 0);  in truncate_inode_pages_range()
381 		folio = __filemap_get_folio(mapping, lend >> PAGE_SHIFT,  in truncate_inode_pages_range()
394 		if (!find_get_entries(mapping, &index, end - 1, &fbatch,  in truncate_inode_pages_range()
415 			truncate_inode_folio(mapping, folio);  in truncate_inode_pages_range()
418 		truncate_folio_batch_exceptionals(mapping, &fbatch, indices);  in truncate_inode_pages_range()
426  * @mapping: mapping to truncate
430  * mapping->invalidate_lock.
434  * mapping->nrpages can be non-zero when this function returns even after
435  * truncation of the whole mapping.
437 void truncate_inode_pages(struct address_space *mapping, loff_t lstart)  in truncate_inode_pages()  argument
439 	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);  in truncate_inode_pages()
445  * @mapping: mapping to truncate
452 void truncate_inode_pages_final(struct address_space *mapping)  in truncate_inode_pages_final()  argument
461 	mapping_set_exiting(mapping);  in truncate_inode_pages_final()
463 	if (!mapping_empty(mapping)) {  in truncate_inode_pages_final()
470 		xa_lock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
471 		xa_unlock_irq(&mapping->i_pages);  in truncate_inode_pages_final()
474 	truncate_inode_pages(mapping, 0);  in truncate_inode_pages_final()
480  * @mapping: the address_space which holds the folios to invalidate
488 unsigned long mapping_try_invalidate(struct address_space *mapping,  in mapping_try_invalidate()  argument
499 	while (find_lock_entries(mapping, &index, end, &fbatch, indices)) {  in mapping_try_invalidate()
506 				count += invalidate_exceptional_entry(mapping,  in mapping_try_invalidate()
511 			ret = mapping_evict_folio(mapping, folio);  in mapping_try_invalidate()
534  * @mapping: the address_space which holds the cache to invalidate
546 unsigned long invalidate_mapping_pages(struct address_space *mapping,  in invalidate_mapping_pages()  argument
549 	return mapping_try_invalidate(mapping, start, end, NULL);  in invalidate_mapping_pages()
560 static int invalidate_complete_folio2(struct address_space *mapping,  in invalidate_complete_folio2()  argument
563 	if (folio->mapping != mapping)  in invalidate_complete_folio2()
569 	spin_lock(&mapping->host->i_lock);  in invalidate_complete_folio2()
570 	xa_lock_irq(&mapping->i_pages);  in invalidate_complete_folio2()
576 	xa_unlock_irq(&mapping->i_pages);  in invalidate_complete_folio2()
577 	if (mapping_shrinkable(mapping))  in invalidate_complete_folio2()
578 		inode_add_lru(mapping->host);  in invalidate_complete_folio2()
579 	spin_unlock(&mapping->host->i_lock);  in invalidate_complete_folio2()
581 	filemap_free_folio(mapping, folio);  in invalidate_complete_folio2()
584 	xa_unlock_irq(&mapping->i_pages);  in invalidate_complete_folio2()
585 	spin_unlock(&mapping->host->i_lock);  in invalidate_complete_folio2()
589 static int folio_launder(struct address_space *mapping, struct folio *folio)  in folio_launder()  argument
593 	if (folio->mapping != mapping || mapping->a_ops->launder_folio == NULL)  in folio_launder()
595 	return mapping->a_ops->launder_folio(folio);  in folio_launder()
600  * @mapping: the address_space
609 int invalidate_inode_pages2_range(struct address_space *mapping,  in invalidate_inode_pages2_range()  argument
620 	if (mapping_empty(mapping))  in invalidate_inode_pages2_range()
625 	while (find_get_entries(mapping, &index, end, &fbatch, indices)) {  in invalidate_inode_pages2_range()
632 				if (!invalidate_exceptional_entry2(mapping,  in invalidate_inode_pages2_range()
643 				unmap_mapping_pages(mapping, indices[i],  in invalidate_inode_pages2_range()
649 			if (unlikely(folio->mapping != mapping)) {  in invalidate_inode_pages2_range()
660 			ret2 = folio_launder(mapping, folio);  in invalidate_inode_pages2_range()
662 				if (!invalidate_complete_folio2(mapping, folio))  in invalidate_inode_pages2_range()
680 	if (dax_mapping(mapping)) {  in invalidate_inode_pages2_range()
681 		unmap_mapping_pages(mapping, start, end - start + 1, false);  in invalidate_inode_pages2_range()
689  * @mapping: the address_space
696 int invalidate_inode_pages2(struct address_space *mapping)  in invalidate_inode_pages2()  argument
698 	return invalidate_inode_pages2_range(mapping, 0, -1);  in invalidate_inode_pages2()
719 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache()  local
731 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
732 	truncate_inode_pages(mapping, newsize);  in truncate_pagecache()
733 	unmap_mapping_range(mapping, holebegin, 0, 1);  in truncate_pagecache()
827 	struct address_space *mapping = inode->i_mapping;  in truncate_pagecache_range()  local
844 		unmap_mapping_range(mapping, unmap_start,  in truncate_pagecache_range()
846 	truncate_inode_pages_range(mapping, lstart, lend);  in truncate_pagecache_range()