1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/readahead.c - address_space-level file readahead.
16 #include <linux/backing-dev.h>
23 #include <linux/blk-cgroup.h>
36 	ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;  in file_ra_state_init()
37 	ra->prev_pos = -1;  in file_ra_state_init()
42  * see if a page needs releasing upon read_cache_pages() failure
43  * - the caller of read_cache_pages() may have set PG_private or PG_fscache
49 					     struct page *page)  in read_cache_pages_invalidate_page()  argument
51 	if (page_has_private(page)) {  in read_cache_pages_invalidate_page()
52 		if (!trylock_page(page))  in read_cache_pages_invalidate_page()
54 		page->mapping = mapping;  in read_cache_pages_invalidate_page()
55 		do_invalidatepage(page, 0, PAGE_SIZE);  in read_cache_pages_invalidate_page()
56 		page->mapping = NULL;  in read_cache_pages_invalidate_page()
57 		unlock_page(page);  in read_cache_pages_invalidate_page()
59 	put_page(page);  in read_cache_pages_invalidate_page()
68 	struct page *victim;  in read_cache_pages_invalidate_pages()
72 		list_del(&victim->lru);  in read_cache_pages_invalidate_pages()
78  * read_cache_pages - populate an address space with some pages & start reads against them
81  *   pages have their ->index populated and are otherwise uninitialised.
82  * @filler: callback routine for filling a single page.
90 			int (*filler)(void *, struct page *), void *data)  in read_cache_pages()  argument
92 	struct page *page;  in read_cache_pages()  local
96 		page = lru_to_page(pages);  in read_cache_pages()
97 		list_del(&page->lru);  in read_cache_pages()
98 		if (add_to_page_cache_lru(page, mapping, page->index,  in read_cache_pages()
100 			read_cache_pages_invalidate_page(mapping, page);  in read_cache_pages()
103 		put_page(page);  in read_cache_pages()
105 		ret = filler(data, page);  in read_cache_pages()
120 	const struct address_space_operations *aops = rac->mapping->a_ops;  in read_pages()
121 	struct page *page;  in read_pages()  local
129 	if (aops->readahead) {  in read_pages()
130 		aops->readahead(rac);  in read_pages()
132 		while ((page = readahead_page(rac))) {  in read_pages()
133 			unlock_page(page);  in read_pages()
134 			put_page(page);  in read_pages()
136 	} else if (aops->readpages) {  in read_pages()
137 		aops->readpages(rac->file, rac->mapping, pages,  in read_pages()
141 		rac->_index += rac->_nr_pages;  in read_pages()
142 		rac->_nr_pages = 0;  in read_pages()
144 		while ((page = readahead_page(rac))) {  in read_pages()
145 			aops->readpage(rac->file, page);  in read_pages()
146 			put_page(page);  in read_pages()
157 		rac->_index++;  in read_pages()
161  * page_cache_ra_unbounded - Start unchecked readahead.
177 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
185 	 * locked pages to the page cache, but will not yet have submitted  in page_cache_ra_unbounded()
186 	 * them for I/O.  Adding another page may need to allocate memory,  in page_cache_ra_unbounded()
189 	 * touch file-backed pages, preventing a deadlock.  Most (all?)  in page_cache_ra_unbounded()
199 		struct page *page = xa_load(&mapping->i_pages, index + i);  in page_cache_ra_unbounded()  local
201 		BUG_ON(index + i != ractl->_index + ractl->_nr_pages);  in page_cache_ra_unbounded()
203 		if (page && !xa_is_value(page)) {  in page_cache_ra_unbounded()
205 			 * Page already present?  Kick off the current batch  in page_cache_ra_unbounded()
207 			 * next batch.  This page may be the one we would  in page_cache_ra_unbounded()
209 			 * have a stable reference to this page, and it's  in page_cache_ra_unbounded()
216 		page = __page_cache_alloc(gfp_mask);  in page_cache_ra_unbounded()
217 		if (!page)  in page_cache_ra_unbounded()
219 		if (mapping->a_ops->readpages) {  in page_cache_ra_unbounded()
220 			page->index = index + i;  in page_cache_ra_unbounded()
221 			list_add(&page->lru, &page_pool);  in page_cache_ra_unbounded()
222 		} else if (add_to_page_cache_lru(page, mapping, index + i,  in page_cache_ra_unbounded()
224 			put_page(page);  in page_cache_ra_unbounded()
228 		if (i == nr_to_read - lookahead_size)  in page_cache_ra_unbounded()
229 			SetPageReadahead(page);  in page_cache_ra_unbounded()
230 		ractl->_nr_pages++;  in page_cache_ra_unbounded()
234 	 * Now start the IO.  We ignore I/O errors - if the page is not  in page_cache_ra_unbounded()
246  * behaviour which would occur if page allocations are causing VM writeback.
252 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
255 	pgoff_t end_index;	/* The last page we want to read */  in do_page_cache_ra()
260 	end_index = (isize - 1) >> PAGE_SHIFT;  in do_page_cache_ra()
263 	/* Don't read past the page containing the last byte of the file */  in do_page_cache_ra()
264 	if (nr_to_read > end_index - index)  in do_page_cache_ra()
265 		nr_to_read = end_index - index + 1;  in do_page_cache_ra()
277 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
278 	struct backing_dev_info *bdi = inode_to_bdi(mapping->host);  in force_page_cache_ra()
281 	if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages &&  in force_page_cache_ra()
282 			!mapping->a_ops->readahead))  in force_page_cache_ra()
287 	 * be up to the optimal hardware IO size  in force_page_cache_ra()
290 	max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);  in force_page_cache_ra()
297 		ractl->_index = index;  in force_page_cache_ra()
301 		nr_to_read -= this_chunk;  in force_page_cache_ra()
306  * Set the initial window size, round to next power of 2 and square
307  * for small size, x 4 for medium, and x 2 for large
308  * for 128k (32 page) max ra
309  * 1-8 page = 32k initial, > 8 page = 128k initial
311 static unsigned long get_init_ra_size(unsigned long size, unsigned long max)  in get_init_ra_size()  argument
313 	unsigned long newsize = roundup_pow_of_two(size);  in get_init_ra_size()
326  *  Get the previous window size, ramp it up, and
327  *  return it as the new window size.
332 	unsigned long cur = ra->size;  in get_next_ra_size()
342  * On-demand readahead design.
344  * The fields in struct file_ra_state represent the most-recently-executed
347  *                        |<----- async_size ---------|
348  *     |------------------- size -------------------->|
350  *     ^start             ^page marked with PG_readahead
354  * readahead pages and stalled on the missing page at readahead_index;
357  * will be equal to size, for maximum pipelining.
361  * page at (start+size-async_size) with PG_readahead, and use it as readahead
363  * readahead-for-nothing fuss, saving pointless page cache lookups.
371  * There is a special-case: if the first page which the application tries to
372  * read happens to be the first page of the file, it is assumed that a linear
373  * read is about to happen and the window is immediately set to the initial size
374  * based on I/O request size and the max_readahead.
376  * The code ramps up the readahead size aggressively at first, but slow down as
381  * Count contiguously cached pages from @index-1 to @index-@max,
383  * 	- length of the sequential read sequence, or
384  * 	- thrashing threshold in memory tight systems
392 	head = page_cache_prev_miss(mapping, index - 1, max);  in count_history_pages()
395 	return index - 1 - head;  in count_history_pages()
399  * page cache context based read-ahead
407 	pgoff_t size;  in try_context_readahead()  local
409 	size = count_history_pages(mapping, index, max);  in try_context_readahead()
415 	if (size <= req_size)  in try_context_readahead()
420 	 * it is a strong indication of long-run stream (or whole-file-read)  in try_context_readahead()
422 	if (size >= index)  in try_context_readahead()
423 		size *= 2;  in try_context_readahead()
425 	ra->start = index;  in try_context_readahead()
426 	ra->size = min(size + req_size, max);  in try_context_readahead()
427 	ra->async_size = 1;  in try_context_readahead()
439 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ondemand_readahead()
440 	unsigned long max_pages = ra->ra_pages;  in ondemand_readahead()
447 	 * be up to the optimal hardware IO size  in ondemand_readahead()
449 	if (req_size > max_pages && bdi->io_pages > max_pages)  in ondemand_readahead()
450 		max_pages = min(req_size, bdi->io_pages);  in ondemand_readahead()
462 	if ((index == (ra->start + ra->size - ra->async_size) ||  in ondemand_readahead()
463 	     index == (ra->start + ra->size))) {  in ondemand_readahead()
464 		ra->start += ra->size;  in ondemand_readahead()
465 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
466 		ra->async_size = ra->size;  in ondemand_readahead()
471 	 * Hit a marked page without valid readahead state.  in ondemand_readahead()
474 	 * readahead size. Ramp it up and use it as the new readahead size.  in ondemand_readahead()
480 		start = page_cache_next_miss(ractl->mapping, index + 1,  in ondemand_readahead()
484 		if (!start || start - index > max_pages)  in ondemand_readahead()
487 		ra->start = start;  in ondemand_readahead()
488 		ra->size = start - index;	/* old async_size */  in ondemand_readahead()
489 		ra->size += req_size;  in ondemand_readahead()
490 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
491 		ra->async_size = ra->size;  in ondemand_readahead()
503 	 * trivial case: (index - prev_index) == 1  in ondemand_readahead()
504 	 * unaligned reads: (index - prev_index) == 0  in ondemand_readahead()
506 	prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;  in ondemand_readahead()
507 	if (index - prev_index <= 1UL)  in ondemand_readahead()
511 	 * Query the page cache and look for the traces(cached history pages)  in ondemand_readahead()
514 	if (try_context_readahead(ractl->mapping, ra, index, req_size,  in ondemand_readahead()
526 	ra->start = index;  in ondemand_readahead()
527 	ra->size = get_init_ra_size(req_size, max_pages);  in ondemand_readahead()
528 	ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;  in ondemand_readahead()
537 	if (index == ra->start && ra->size == ra->async_size) {  in ondemand_readahead()
539 		if (ra->size + add_pages <= max_pages) {  in ondemand_readahead()
540 			ra->async_size = add_pages;  in ondemand_readahead()
541 			ra->size += add_pages;  in ondemand_readahead()
543 			ra->size = max_pages;  in ondemand_readahead()
544 			ra->async_size = max_pages >> 1;  in ondemand_readahead()
548 	ractl->_index = ra->start;  in ondemand_readahead()
549 	do_page_cache_ra(ractl, ra->size, ra->async_size);  in ondemand_readahead()
555 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
558 	 * Even if read-ahead is disabled, issue this request as read-ahead  in page_cache_sync_ra()
560 	 * read-ahead will do the right thing and limit the read to just the  in page_cache_sync_ra()
561 	 * requested range, which we'll set to 1 page for this case.  in page_cache_sync_ra()
563 	if (!ra->ra_pages || blk_cgroup_congested()) {  in page_cache_sync_ra()
564 		if (!ractl->file)  in page_cache_sync_ra()
576 	/* do read-ahead */  in page_cache_sync_ra()
582 		struct file_ra_state *ra, struct page *page,  in page_cache_async_ra()  argument
585 	/* no read-ahead */  in page_cache_async_ra()
586 	if (!ra->ra_pages)  in page_cache_async_ra()
592 	if (PageWriteback(page))  in page_cache_async_ra()
595 	ClearPageReadahead(page);  in page_cache_async_ra()
598 	 * Defer asynchronous read-ahead on IO congestion.  in page_cache_async_ra()
600 	if (inode_read_congested(ractl->mapping->host))  in page_cache_async_ra()
606 	/* do read-ahead */  in page_cache_async_ra()
616 	ret = -EBADF;  in ksys_readahead()
618 	if (!f.file || !(f.file->f_mode & FMODE_READ))  in ksys_readahead()
624 	 * on this file, then we must return -EINVAL.  in ksys_readahead()
626 	ret = -EINVAL;  in ksys_readahead()
627 	if (!f.file->f_mapping || !f.file->f_mapping->a_ops ||  in ksys_readahead()
628 	    !S_ISREG(file_inode(f.file)->i_mode))  in ksys_readahead()