10  *  Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
30 				struct page *page, bio_end_io_t end_io)  in get_swap_bio()  argument
38 		bio->bi_iter.bi_sector = map_swap_page(page, &bdev);  in get_swap_bio()
43 		bio_add_page(bio, page, thp_size(page), 0);  in get_swap_bio()
50 	struct page *page = bio_first_page_all(bio);  in end_swap_bio_write()  local
53 		SetPageError(page);  in end_swap_bio_write()
55 		 * We failed to write the page out to swap-space.  in end_swap_bio_write()
56 		 * Re-dirty the page in order to avoid it being reclaimed.  in end_swap_bio_write()
62 		set_page_dirty(page);  in end_swap_bio_write()
66 		ClearPageReclaim(page);  in end_swap_bio_write()
68 	end_page_writeback(page);  in end_swap_bio_write()
72 static void swap_slot_free_notify(struct page *page)  in swap_slot_free_notify()  argument
79 	 * There is no guarantee that the page is in swap cache - the software  in swap_slot_free_notify()
81 	 * swapcache page.  So we must check PG_swapcache before proceeding with  in swap_slot_free_notify()
84 	if (unlikely(!PageSwapCache(page)))  in swap_slot_free_notify()
87 	sis = page_swap_info(page);  in swap_slot_free_notify()
93 	 * expecting that the page will be swapped out again.  in swap_slot_free_notify()
94 	 * So we can avoid an unnecessary write if the page  in swap_slot_free_notify()
103 	 * and make the VM-owned decompressed page *dirty*,  in swap_slot_free_notify()
104 	 * so the page should be swapped out somewhere again if  in swap_slot_free_notify()
108 	entry.val = page_private(page);  in swap_slot_free_notify()
114 		SetPageDirty(page);  in swap_slot_free_notify()
122 	struct page *page = bio_first_page_all(bio);  in end_swap_bio_read()  local
126 		SetPageError(page);  in end_swap_bio_read()
127 		ClearPageUptodate(page);  in end_swap_bio_read()
134 	SetPageUptodate(page);  in end_swap_bio_read()
135 	swap_slot_free_notify(page);  in end_swap_bio_read()
137 	unlock_page(page);  in end_swap_bio_read()
209 		if (page_no) {	/* exclude the header page */  in generic_swapfile_activate()
247 int swap_writepage(struct page *page, struct writeback_control *wbc)  in swap_writepage()  argument
251 	if (try_to_free_swap(page)) {  in swap_writepage()
252 		unlock_page(page);  in swap_writepage()
256 	 * Arch code may have to preserve more data than just the page  in swap_writepage()
259 	ret = arch_prepare_to_swap(page);  in swap_writepage()
261 		set_page_dirty(page);  in swap_writepage()
262 		unlock_page(page);  in swap_writepage()
265 	if (frontswap_store(page) == 0) {  in swap_writepage()
266 		set_page_writeback(page);  in swap_writepage()
267 		unlock_page(page);  in swap_writepage()
268 		end_page_writeback(page);  in swap_writepage()
271 	ret = __swap_writepage(page, wbc, end_swap_bio_write);  in swap_writepage()
276 static sector_t swap_page_sector(struct page *page)  in swap_page_sector()  argument
278 	return (sector_t)__page_file_index(page) << (PAGE_SHIFT - 9);  in swap_page_sector()
281 static inline void count_swpout_vm_event(struct page *page)  in count_swpout_vm_event()  argument
284 	if (unlikely(PageTransHuge(page)))  in count_swpout_vm_event()
287 	count_vm_events(PSWPOUT, thp_nr_pages(page));  in count_swpout_vm_event()
291 static void bio_associate_blkg_from_page(struct bio *bio, struct page *page)  in bio_associate_blkg_from_page()  argument
295 	if (!page->mem_cgroup)  in bio_associate_blkg_from_page()
299 	css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);  in bio_associate_blkg_from_page()
304 #define bio_associate_blkg_from_page(bio, page)		do { } while (0)  argument
307 int __swap_writepage(struct page *page, struct writeback_control *wbc,  in __swap_writepage()  argument
312 	struct swap_info_struct *sis = page_swap_info(page);  in __swap_writepage()
314 	VM_BUG_ON_PAGE(!PageSwapCache(page), page);  in __swap_writepage()
320 			.bv_page = page,  in __swap_writepage()
328 		kiocb.ki_pos = page_file_offset(page);  in __swap_writepage()
330 		set_page_writeback(page);  in __swap_writepage()
331 		unlock_page(page);  in __swap_writepage()
341 			 * Mark the page dirty and avoid  in __swap_writepage()
347 			set_page_dirty(page);  in __swap_writepage()
348 			ClearPageReclaim(page);  in __swap_writepage()
350 					   page_file_offset(page));  in __swap_writepage()
352 		end_page_writeback(page);  in __swap_writepage()
356 	ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);  in __swap_writepage()
358 		count_swpout_vm_event(page);  in __swap_writepage()
362 	bio = get_swap_bio(GFP_NOIO, page, end_write_func);  in __swap_writepage()
364 		set_page_dirty(page);  in __swap_writepage()
365 		unlock_page(page);  in __swap_writepage()
369 	bio_associate_blkg_from_page(bio, page);  in __swap_writepage()
370 	count_swpout_vm_event(page);  in __swap_writepage()
371 	set_page_writeback(page);  in __swap_writepage()
372 	unlock_page(page);  in __swap_writepage()
378 int swap_readpage(struct page *page, bool synchronous)  in swap_readpage()  argument
382 	struct swap_info_struct *sis = page_swap_info(page);  in swap_readpage()
387 	VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);  in swap_readpage()
388 	VM_BUG_ON_PAGE(!PageLocked(page), page);  in swap_readpage()
389 	VM_BUG_ON_PAGE(PageUptodate(page), page);  in swap_readpage()
398 	if (frontswap_load(page) == 0) {  in swap_readpage()
399 		SetPageUptodate(page);  in swap_readpage()
400 		unlock_page(page);  in swap_readpage()
408 		ret = mapping->a_ops->readpage(swap_file, page);  in swap_readpage()
415 		ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);  in swap_readpage()
417 			if (trylock_page(page)) {  in swap_readpage()
418 				swap_slot_free_notify(page);  in swap_readpage()
419 				unlock_page(page);  in swap_readpage()
428 	bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);  in swap_readpage()
430 		unlock_page(page);  in swap_readpage()
437 	 * attempt to access it in the page fault retry time check.  in swap_readpage()
464 int swap_set_page_dirty(struct page *page)  in swap_set_page_dirty()  argument
466 	struct swap_info_struct *sis = page_swap_info(page);  in swap_set_page_dirty()
471 		VM_BUG_ON_PAGE(!PageSwapCache(page), page);  in swap_set_page_dirty()
472 		return mapping->a_ops->set_page_dirty(page);  in swap_set_page_dirty()
474 		return __set_page_dirty_no_writeback(page);  in swap_set_page_dirty()