1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/readahead.c - address_space-level file readahead.
16 #include <linux/backing-dev.h>
23 #include <linux/blk-cgroup.h>
36 	ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;  in file_ra_state_init()
37 	ra->prev_pos = -1;  in file_ra_state_init()
43  * - the caller of read_cache_pages() may have set PG_private or PG_fscache
45  *   on disk, thus we need to give the fs a chance to clean up in the event of
54 		page->mapping = mapping;  in read_cache_pages_invalidate_page()
56 		page->mapping = NULL;  in read_cache_pages_invalidate_page()
72 		list_del(&victim->lru);  in read_cache_pages_invalidate_pages()
78  * read_cache_pages - populate an address space with some pages & start reads against them
81  *   pages have their ->index populated and are otherwise uninitialised.
97 		list_del(&page->lru);  in read_cache_pages()
98 		if (add_to_page_cache_lru(page, mapping, page->index,  in read_cache_pages()
120 	const struct address_space_operations *aops = rac->mapping->a_ops;  in read_pages()
129 	if (aops->readahead) {  in read_pages()
130 		aops->readahead(rac);  in read_pages()
131 		/* Clean up the remaining pages */  in read_pages()
136 	} else if (aops->readpages) {  in read_pages()
137 		aops->readpages(rac->file, rac->mapping, pages,  in read_pages()
139 		/* Clean up the remaining pages */  in read_pages()
141 		rac->_index += rac->_nr_pages;  in read_pages()
142 		rac->_nr_pages = 0;  in read_pages()
145 			aops->readpage(rac->file, page);  in read_pages()
157 		rac->_index++;  in read_pages()
161  * page_cache_ra_unbounded - Start unchecked readahead.
164  * @lookahead_size: Where to start the next readahead.
166  * This function is for filesystems to call when they want to start
177 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
189 	 * touch file-backed pages, preventing a deadlock.  Most (all?)  in page_cache_ra_unbounded()
199 		struct page *page = xa_load(&mapping->i_pages, index + i);  in page_cache_ra_unbounded()
201 		BUG_ON(index + i != ractl->_index + ractl->_nr_pages);  in page_cache_ra_unbounded()
219 		if (mapping->a_ops->readpages) {  in page_cache_ra_unbounded()
220 			page->index = index + i;  in page_cache_ra_unbounded()
221 			list_add(&page->lru, &page_pool);  in page_cache_ra_unbounded()
228 		if (i == nr_to_read - lookahead_size)  in page_cache_ra_unbounded()
230 		ractl->_nr_pages++;  in page_cache_ra_unbounded()
234 	 * Now start the IO.  We ignore I/O errors - if the page is not  in page_cache_ra_unbounded()
252 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
260 	end_index = (isize - 1) >> PAGE_SHIFT;  in do_page_cache_ra()
264 	if (nr_to_read > end_index - index)  in do_page_cache_ra()
265 		nr_to_read = end_index - index + 1;  in do_page_cache_ra()
277 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
278 	struct backing_dev_info *bdi = inode_to_bdi(mapping->host);  in force_page_cache_ra()
281 	if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages &&  in force_page_cache_ra()
282 			!mapping->a_ops->readahead))  in force_page_cache_ra()
287 	 * be up to the optimal hardware IO size  in force_page_cache_ra()
290 	max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);  in force_page_cache_ra()
297 		ractl->_index = index;  in force_page_cache_ra()
301 		nr_to_read -= this_chunk;  in force_page_cache_ra()
309  * 1-8 page = 32k initial, > 8 page = 128k initial
326  *  Get the previous window size, ramp it up, and
332 	unsigned long cur = ra->size;  in get_next_ra_size()
342  * On-demand readahead design.
344  * The fields in struct file_ra_state represent the most-recently-executed
347  *                        |<----- async_size ---------|
348  *     |------------------- size -------------------->|
350  *     ^start             ^page marked with PG_readahead
361  * page at (start+size-async_size) with PG_readahead, and use it as readahead
363  * readahead-for-nothing fuss, saving pointless page cache lookups.
371  * There is a special-case: if the first page which the application tries to
376  * The code ramps up the readahead size aggressively at first, but slow down as
381  * Count contiguously cached pages from @index-1 to @index-@max,
383  * 	- length of the sequential read sequence, or
384  * 	- thrashing threshold in memory tight systems
392 	head = page_cache_prev_miss(mapping, index - 1, max);  in count_history_pages()
395 	return index - 1 - head;  in count_history_pages()
399  * page cache context based read-ahead
420 	 * it is a strong indication of long-run stream (or whole-file-read)  in try_context_readahead()
425 	ra->start = index;  in try_context_readahead()
426 	ra->size = min(size + req_size, max);  in try_context_readahead()
427 	ra->async_size = 1;  in try_context_readahead()
439 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ondemand_readahead()
440 	unsigned long max_pages = ra->ra_pages;  in ondemand_readahead()
447 	 * be up to the optimal hardware IO size  in ondemand_readahead()
449 	if (req_size > max_pages && bdi->io_pages > max_pages)  in ondemand_readahead()
450 		max_pages = min(req_size, bdi->io_pages);  in ondemand_readahead()
453 	 * start of file  in ondemand_readahead()
460 	 * Ramp up sizes, and push forward the readahead window.  in ondemand_readahead()
462 	if ((index == (ra->start + ra->size - ra->async_size) ||  in ondemand_readahead()
463 	     index == (ra->start + ra->size))) {  in ondemand_readahead()
464 		ra->start += ra->size;  in ondemand_readahead()
465 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
466 		ra->async_size = ra->size;  in ondemand_readahead()
474 	 * readahead size. Ramp it up and use it as the new readahead size.  in ondemand_readahead()
477 		pgoff_t start;  in ondemand_readahead()  local
480 		start = page_cache_next_miss(ractl->mapping, index + 1,  in ondemand_readahead()
484 		if (!start || start - index > max_pages)  in ondemand_readahead()
487 		ra->start = start;  in ondemand_readahead()
488 		ra->size = start - index;	/* old async_size */  in ondemand_readahead()
489 		ra->size += req_size;  in ondemand_readahead()
490 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
491 		ra->async_size = ra->size;  in ondemand_readahead()
503 	 * trivial case: (index - prev_index) == 1  in ondemand_readahead()
504 	 * unaligned reads: (index - prev_index) == 0  in ondemand_readahead()
506 	prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;  in ondemand_readahead()
507 	if (index - prev_index <= 1UL)  in ondemand_readahead()
514 	if (try_context_readahead(ractl->mapping, ra, index, req_size,  in ondemand_readahead()
526 	ra->start = index;  in ondemand_readahead()
527 	ra->size = get_init_ra_size(req_size, max_pages);  in ondemand_readahead()
528 	ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;  in ondemand_readahead()
537 	if (index == ra->start && ra->size == ra->async_size) {  in ondemand_readahead()
539 		if (ra->size + add_pages <= max_pages) {  in ondemand_readahead()
540 			ra->async_size = add_pages;  in ondemand_readahead()
541 			ra->size += add_pages;  in ondemand_readahead()
543 			ra->size = max_pages;  in ondemand_readahead()
544 			ra->async_size = max_pages >> 1;  in ondemand_readahead()
548 	ractl->_index = ra->start;  in ondemand_readahead()
549 	do_page_cache_ra(ractl, ra->size, ra->async_size);  in ondemand_readahead()
555 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
558 	 * Even if read-ahead is disabled, issue this request as read-ahead  in page_cache_sync_ra()
560 	 * read-ahead will do the right thing and limit the read to just the  in page_cache_sync_ra()
563 	if (!ra->ra_pages || blk_cgroup_congested()) {  in page_cache_sync_ra()
564 		if (!ractl->file)  in page_cache_sync_ra()
576 	/* do read-ahead */  in page_cache_sync_ra()
585 	/* no read-ahead */  in page_cache_async_ra()
586 	if (!ra->ra_pages)  in page_cache_async_ra()
598 	 * Defer asynchronous read-ahead on IO congestion.  in page_cache_async_ra()
600 	if (inode_read_congested(ractl->mapping->host))  in page_cache_async_ra()
606 	/* do read-ahead */  in page_cache_async_ra()
616 	ret = -EBADF;  in ksys_readahead()
618 	if (!f.file || !(f.file->f_mode & FMODE_READ))  in ksys_readahead()
624 	 * on this file, then we must return -EINVAL.  in ksys_readahead()
626 	ret = -EINVAL;  in ksys_readahead()
627 	if (!f.file->f_mapping || !f.file->f_mapping->a_ops ||  in ksys_readahead()
628 	    !S_ISREG(file_inode(f.file)->i_mode))  in ksys_readahead()