1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/readahead.c - address_space-level file readahead.
17  * folio is present but not up-to-date, readahead will not try to read
18  * it. In that case a simple ->read_folio() will be requested.
29  * contains ->size being the total number of pages, and ->async_size
37  * to be determined: the start of the region to read, the size of the
40  * The start of the region is simply the first page address at or after
46  * this would be less than zero - then zero is used.  NOTE THIS
47  * CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED
57  * page to the start of the new readahead.  In these cases, the size of
65  * larger than the current request, and it is not scaled up, unless it
66  * is at the start of file.
68  * In general readahead is accelerated at the start of the file, as
77  * Readahead requests are sent to the filesystem using the ->readahead()
79  * implementation.  ->readahead() should normally initiate reads on all
81  * error.  The page cache reading code will issue a ->read_folio() request
82  * for any folio which ->readahead() did not read, and only an error
85  * ->readahead() will generally call readahead_folio() repeatedly to get
103  * considered to be important and ->readahead() should not fail them due
113  * ->read_folio() which may be less efficient.
121 #include <linux/backing-dev.h>
128 #include <linux/blk-cgroup.h>
141 	ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;  in file_ra_state_init()
142 	ra->prev_pos = -1;  in file_ra_state_init()
148 	const struct address_space_operations *aops = rac->mapping->a_ops;  in read_pages()
155 	if (unlikely(rac->_workingset))  in read_pages()
156 		psi_memstall_enter(&rac->_pflags);  in read_pages()
159 	if (aops->readahead) {  in read_pages()
160 		aops->readahead(rac);  in read_pages()
162 		 * Clean up the remaining folios.  The sizes in ->ra  in read_pages()
170 			rac->ra->size -= nr;  in read_pages()
171 			if (rac->ra->async_size >= nr) {  in read_pages()
172 				rac->ra->async_size -= nr;  in read_pages()
180 			aops->read_folio(rac->file, folio);  in read_pages()
184 	if (unlikely(rac->_workingset))  in read_pages()
185 		psi_memstall_leave(&rac->_pflags);  in read_pages()
186 	rac->_workingset = false;  in read_pages()
192  * page_cache_ra_unbounded - Start unchecked readahead.
195  * @lookahead_size: Where to start the next readahead.
197  * This function is for filesystems to call when they want to start
208 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
219 	 * touch file-backed pages, preventing a deadlock.  Most (all?)  in page_cache_ra_unbounded()
230 		struct folio *folio = xa_load(&mapping->i_pages, index + i);  in page_cache_ra_unbounded()
242 			ractl->_index++;  in page_cache_ra_unbounded()
243 			i = ractl->_index + ractl->_nr_pages - index - 1;  in page_cache_ra_unbounded()
254 			ractl->_index++;  in page_cache_ra_unbounded()
255 			i = ractl->_index + ractl->_nr_pages - index - 1;  in page_cache_ra_unbounded()
258 		if (i == nr_to_read - lookahead_size)  in page_cache_ra_unbounded()
260 		ractl->_workingset |= folio_test_workingset(folio);  in page_cache_ra_unbounded()
261 		ractl->_nr_pages++;  in page_cache_ra_unbounded()
265 	 * Now start the IO.  We ignore I/O errors - if the folio is not  in page_cache_ra_unbounded()
284 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
292 	end_index = (isize - 1) >> PAGE_SHIFT;  in do_page_cache_ra()
296 	if (nr_to_read > end_index - index)  in do_page_cache_ra()
297 		nr_to_read = end_index - index + 1;  in do_page_cache_ra()
309 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
310 	struct file_ra_state *ra = ractl->ra;  in force_page_cache_ra()
311 	struct backing_dev_info *bdi = inode_to_bdi(mapping->host);  in force_page_cache_ra()
314 	if (unlikely(!mapping->a_ops->read_folio && !mapping->a_ops->readahead))  in force_page_cache_ra()
319 	 * be up to the optimal hardware IO size  in force_page_cache_ra()
322 	max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);  in force_page_cache_ra()
329 		ractl->_index = index;  in force_page_cache_ra()
333 		nr_to_read -= this_chunk;  in force_page_cache_ra()
341  * 1-2 page = 16k, 3-4 page 32k, 5-8 page = 64k, > 8 page = 128k initial
358  *  Get the previous window size, ramp it up, and
364 	unsigned long cur = ra->size;  in get_next_ra_size()
374  * On-demand readahead design.
376  * The fields in struct file_ra_state represent the most-recently-executed
379  *                        |<----- async_size ---------|
380  *     |------------------- size -------------------->|
382  *     ^start             ^page marked with PG_readahead
393  * page at (start+size-async_size) with PG_readahead, and use it as readahead
395  * readahead-for-nothing fuss, saving pointless page cache lookups.
403  * There is a special-case: if the first page which the application tries to
408  * The code ramps up the readahead size aggressively at first, but slow down as
413  * Count contiguously cached pages from @index-1 to @index-@max,
415  * 	- length of the sequential read sequence, or
416  * 	- thrashing threshold in memory tight systems
424 	head = page_cache_prev_miss(mapping, index - 1, max);  in count_history_pages()
427 	return index - 1 - head;  in count_history_pages()
452 	 * it is a strong indication of long-run stream (or whole-file-read)  in try_context_readahead()
457 	ra->start = index;  in try_context_readahead()
458 	ra->size = min(size + req_size, max);  in try_context_readahead()
459 	ra->async_size = 1;  in try_context_readahead()
471 		return -ENOMEM;  in ra_alloc_folio()
475 	err = filemap_add_folio(ractl->mapping, folio, index, gfp);  in ra_alloc_folio()
481 	ractl->_nr_pages += 1UL << order;  in ra_alloc_folio()
482 	ractl->_workingset |= folio_test_workingset(folio);  in ra_alloc_folio()
489 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_order()
491 	pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;  in page_cache_ra_order()
492 	pgoff_t mark = index + ra->size - ra->async_size;  in page_cache_ra_order()
496 	if (!mapping_large_folio_support(mapping) || ra->size < 4)  in page_cache_ra_order()
499 	limit = min(limit, index + ra->size - 1);  in page_cache_ra_order()
505 		while ((1 << new_order) > ra->size)  in page_cache_ra_order()
506 			new_order--;  in page_cache_ra_order()
514 		if (index & ((1UL << order) - 1))  in page_cache_ra_order()
517 		while (index + (1UL << order) - 1 > limit)  in page_cache_ra_order()
518 			order--;  in page_cache_ra_order()
519 		/* THP machinery does not support order-1 */  in page_cache_ra_order()
529 		ra->size += index - limit - 1;  in page_cache_ra_order()
530 		ra->async_size += index - limit - 1;  in page_cache_ra_order()
544 	do_page_cache_ra(ractl, ra->size, ra->async_size);  in page_cache_ra_order()
553 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ondemand_readahead()
554 	struct file_ra_state *ra = ractl->ra;  in ondemand_readahead()
555 	unsigned long max_pages = ra->ra_pages;  in ondemand_readahead()
563 	 * be up to the optimal hardware IO size  in ondemand_readahead()
565 	if (req_size > max_pages && bdi->io_pages > max_pages)  in ondemand_readahead()
566 		max_pages = min(req_size, bdi->io_pages);  in ondemand_readahead()
569 	 * start of file  in ondemand_readahead()
576 	 * Ramp up sizes, and push forward the readahead window.  in ondemand_readahead()
578 	expected = round_down(ra->start + ra->size - ra->async_size,  in ondemand_readahead()
580 	if (index == expected || index == (ra->start + ra->size)) {  in ondemand_readahead()
581 		ra->start += ra->size;  in ondemand_readahead()
582 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
583 		ra->async_size = ra->size;  in ondemand_readahead()
591 	 * readahead size. Ramp it up and use it as the new readahead size.  in ondemand_readahead()
594 		pgoff_t start;  in ondemand_readahead()  local
597 		start = page_cache_next_miss(ractl->mapping, index + 1,  in ondemand_readahead()
601 		if (!start || start - index > max_pages)  in ondemand_readahead()
604 		ra->start = start;  in ondemand_readahead()
605 		ra->size = start - index;	/* old async_size */  in ondemand_readahead()
606 		ra->size += req_size;  in ondemand_readahead()
607 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
608 		ra->async_size = ra->size;  in ondemand_readahead()
620 	 * trivial case: (index - prev_index) == 1  in ondemand_readahead()
621 	 * unaligned reads: (index - prev_index) == 0  in ondemand_readahead()
623 	prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;  in ondemand_readahead()
624 	if (index - prev_index <= 1UL)  in ondemand_readahead()
631 	if (try_context_readahead(ractl->mapping, ra, index, req_size,  in ondemand_readahead()
643 	ra->start = index;  in ondemand_readahead()
644 	ra->size = get_init_ra_size(req_size, max_pages);  in ondemand_readahead()
645 	ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;  in ondemand_readahead()
654 	if (index == ra->start && ra->size == ra->async_size) {  in ondemand_readahead()
656 		if (ra->size + add_pages <= max_pages) {  in ondemand_readahead()
657 			ra->async_size = add_pages;  in ondemand_readahead()
658 			ra->size += add_pages;  in ondemand_readahead()
660 			ra->size = max_pages;  in ondemand_readahead()
661 			ra->async_size = max_pages >> 1;  in ondemand_readahead()
665 	ractl->_index = ra->start;  in ondemand_readahead()
672 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
680 	if (!ractl->ra->ra_pages || blk_cgroup_congested()) {  in page_cache_sync_ra()
681 		if (!ractl->file)  in page_cache_sync_ra()
701 	if (!ractl->ra->ra_pages)  in page_cache_async_ra()
724 	ret = -EBADF;  in ksys_readahead()
726 	if (!f.file || !(f.file->f_mode & FMODE_READ))  in ksys_readahead()
732 	 * on this file, then we must return -EINVAL.  in ksys_readahead()
734 	ret = -EINVAL;  in ksys_readahead()
735 	if (!f.file->f_mapping || !f.file->f_mapping->a_ops ||  in ksys_readahead()
736 	    (!S_ISREG(file_inode(f.file)->i_mode) &&  in ksys_readahead()
737 	    !S_ISBLK(file_inode(f.file)->i_mode)))  in ksys_readahead()
759  * readahead_expand - Expand a readahead request
761  * @new_start: The revised start
779 	struct address_space *mapping = ractl->mapping;  in readahead_expand()
780 	struct file_ra_state *ra = ractl->ra;  in readahead_expand()
787 	while (ractl->_index > new_index) {  in readahead_expand()
788 		unsigned long index = ractl->_index - 1;  in readahead_expand()
789 		struct folio *folio = xa_load(&mapping->i_pages, index);  in readahead_expand()
802 				!ractl->_workingset) {  in readahead_expand()
803 			ractl->_workingset = true;  in readahead_expand()
804 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
806 		ractl->_nr_pages++;  in readahead_expand()
807 		ractl->_index = folio->index;  in readahead_expand()
810 	new_len += new_start - readahead_pos(ractl);  in readahead_expand()
814 	while (ractl->_nr_pages < new_nr_pages) {  in readahead_expand()
815 		unsigned long index = ractl->_index + ractl->_nr_pages;  in readahead_expand()
816 		struct folio *folio = xa_load(&mapping->i_pages, index);  in readahead_expand()
829 				!ractl->_workingset) {  in readahead_expand()
830 			ractl->_workingset = true;  in readahead_expand()
831 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
833 		ractl->_nr_pages++;  in readahead_expand()
835 			ra->size++;  in readahead_expand()
836 			ra->async_size++;  in readahead_expand()