7 #include <linux/page-isolation.h>
28  * Returns a page without holding a reference. If the caller wants to
29  * dereference that page (e.g., dumping), it has to make sure that it
33 static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long end_pfn,  in has_unmovable_pages()
36 	struct page *page = pfn_to_page(start_pfn);  in has_unmovable_pages()  local
37 	struct zone *zone = page_zone(page);  in has_unmovable_pages()
43 	if (is_migrate_cma_page(page)) {  in has_unmovable_pages()
52 		return page;  in has_unmovable_pages()
56 		page = pfn_to_page(pfn);  in has_unmovable_pages()
64 		if (PageReserved(page))  in has_unmovable_pages()
65 			return page;  in has_unmovable_pages()
79 		 * handle each tail page individually in migration.  in has_unmovable_pages()
81 		if (PageHuge(page) || PageTransCompound(page)) {  in has_unmovable_pages()
82 			struct folio *folio = page_folio(page);  in has_unmovable_pages()
85 			if (PageHuge(page)) {  in has_unmovable_pages()
89 				 * The huge page may be freed so can not  in has_unmovable_pages()
94 					return page;  in has_unmovable_pages()
96 				return page;  in has_unmovable_pages()
99 			skip_pages = folio_nr_pages(folio) - folio_page_idx(folio, page);  in has_unmovable_pages()
105 		 * We can't use page_count without pin a page  in has_unmovable_pages()
106 		 * because another CPU can free compound page.  in has_unmovable_pages()
108 		 * because their page->_refcount is zero at all time.  in has_unmovable_pages()
110 		if (!page_ref_count(page)) {  in has_unmovable_pages()
111 			if (PageBuddy(page))  in has_unmovable_pages()
112 				pfn += (1 << buddy_order(page)) - 1;  in has_unmovable_pages()
117 		 * The HWPoisoned page may be not in buddy system, and  in has_unmovable_pages()
120 		if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))  in has_unmovable_pages()
133 		if ((flags & MEMORY_OFFLINE) && PageOffline(page))  in has_unmovable_pages()
136 		if (__PageMovable(page) || PageLRU(page))  in has_unmovable_pages()
144 		return page;  in has_unmovable_pages()
150  * This function set pageblock migratetype to isolate if no unmovable page is
154 static int set_migratetype_isolate(struct page *page, int migratetype, int isol_flags,  in set_migratetype_isolate()  argument
157 	struct zone *zone = page_zone(page);  in set_migratetype_isolate()
158 	struct page *unmovable;  in set_migratetype_isolate()
162 	if (PageUnaccepted(page))  in set_migratetype_isolate()
163 		accept_page(page);  in set_migratetype_isolate()
172 	if (is_migrate_isolate_page(page)) {  in set_migratetype_isolate()
181 	 * Pass the intersection of [start_pfn, end_pfn) and the page's pageblock  in set_migratetype_isolate()
184 	check_unmovable_start = max(page_to_pfn(page), start_pfn);  in set_migratetype_isolate()
185 	check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),  in set_migratetype_isolate()
191 		if (!move_freepages_block_isolate(zone, page, MIGRATE_ISOLATE)) {  in set_migratetype_isolate()
206 		dump_page(unmovable, "unmovable page");  in set_migratetype_isolate()
212 static void unset_migratetype_isolate(struct page *page, int migratetype)  in unset_migratetype_isolate()  argument
218 	struct page *buddy;  in unset_migratetype_isolate()
220 	zone = page_zone(page);  in unset_migratetype_isolate()
222 	if (!is_migrate_isolate_page(page))  in unset_migratetype_isolate()
228 	 * it is possible that there is free buddy page.  in unset_migratetype_isolate()
233 	if (PageBuddy(page)) {  in unset_migratetype_isolate()
234 		order = buddy_order(page);  in unset_migratetype_isolate()
236 			buddy = find_buddy_page_pfn(page, page_to_pfn(page),  in unset_migratetype_isolate()
239 				isolated_page = !!__isolate_free_page(page, order);  in unset_migratetype_isolate()
241 				 * Isolating a free page in an isolated pageblock  in unset_migratetype_isolate()
265 		WARN_ON_ONCE(!move_freepages_block_isolate(zone, page, migratetype));  in unset_migratetype_isolate()
267 		set_pageblock_migratetype(page, migratetype);  in unset_migratetype_isolate()
268 		__putback_isolated_page(page, order, migratetype);  in unset_migratetype_isolate()
275 static inline struct page *
281 		struct page *page;  in __first_valid_page()  local
283 		page = pfn_to_online_page(pfn + i);  in __first_valid_page()
284 		if (!page)  in __first_valid_page()
286 		return page;  in __first_valid_page()
293  * within a free or in-use page.
294  * @boundary_pfn:		pageblock-aligned pfn that a page might cross
302  * pageblock. When not all pageblocks within a page are isolated at the same
303  * time, free page accounting can go wrong. For example, in the case of
304  * MAX_PAGE_ORDER = pageblock_order + 1, a MAX_PAGE_ORDER page has two
308  * When either pageblock is isolated, if it is a free page, the page is not
310  * in-use page and freed later, __free_one_page() does not split the free page
311  * either. The function handles this by splitting the free page or migrating
312  * the in-use page then splitting the free page.
333 	 * free or in-use page. Also make sure all to-be-isolated pageblocks  in isolate_single_pageblock()
354 	 * a free or in-use page across boundary_pfn:  in isolate_single_pageblock()
356 	 * 1. isolate before boundary_pfn: the page after is not online  in isolate_single_pageblock()
357 	 * 2. isolate after boundary_pfn: the page before is not online  in isolate_single_pageblock()
373 		struct page *page = __first_valid_page(pfn, boundary_pfn - pfn);  in isolate_single_pageblock()  local
375 		VM_BUG_ON(!page);  in isolate_single_pageblock()
376 		pfn = page_to_pfn(page);  in isolate_single_pageblock()
378 		if (PageUnaccepted(page)) {  in isolate_single_pageblock()
383 		if (PageBuddy(page)) {  in isolate_single_pageblock()
384 			int order = buddy_order(page);  in isolate_single_pageblock()
394 		 * If a compound page is straddling our block, attempt  in isolate_single_pageblock()
398 		 * free page that straddles into our block: gigantic  in isolate_single_pageblock()
406 		if (PageCompound(page)) {  in isolate_single_pageblock()
407 			struct page *head = compound_head(page);  in isolate_single_pageblock()
412 			    PageHuge(page)) {  in isolate_single_pageblock()
424 			VM_WARN_ON_ONCE_PAGE(PageLRU(page), page);  in isolate_single_pageblock()
425 			VM_WARN_ON_ONCE_PAGE(__PageMovable(page), page);  in isolate_single_pageblock()
441  * start_isolate_page_range() - mark page range MIGRATE_ISOLATE
453  * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
472  * Please note that there is no strong synchronization with the page allocator
473  * either. Pages might be freed while their page blocks are marked ISOLATED.
487 	struct page *page;  in start_isolate_page_range()  local
488 	/* isolation is done at page block granularity */  in start_isolate_page_range()
515 		page = __first_valid_page(pfn, pageblock_nr_pages);  in start_isolate_page_range()
516 		if (page && set_migratetype_isolate(page, migratetype, flags,  in start_isolate_page_range()
534  * This finds every MIGRATE_ISOLATE page block in the given range
541 	struct page *page;  in undo_isolate_page_range()  local
548 		page = __first_valid_page(pfn, pageblock_nr_pages);  in undo_isolate_page_range()
549 		if (!page || !is_migrate_isolate_page(page))  in undo_isolate_page_range()
551 		unset_migratetype_isolate(page, migratetype);  in undo_isolate_page_range()
565 	struct page *page;  in __test_page_isolated_in_pageblock()  local
568 		page = pfn_to_page(pfn);  in __test_page_isolated_in_pageblock()
569 		if (PageBuddy(page))  in __test_page_isolated_in_pageblock()
571 			 * If the page is on a free list, it has to be on  in __test_page_isolated_in_pageblock()
575 			pfn += 1 << buddy_order(page);  in __test_page_isolated_in_pageblock()
576 		else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))  in __test_page_isolated_in_pageblock()
577 			/* A HWPoisoned page cannot be also PageBuddy */  in __test_page_isolated_in_pageblock()
579 		else if ((flags & MEMORY_OFFLINE) && PageOffline(page) &&  in __test_page_isolated_in_pageblock()
580 			 !page_count(page))  in __test_page_isolated_in_pageblock()
613 	struct page *page;  in test_pages_isolated()  local
633 		page = __first_valid_page(pfn, pageblock_nr_pages);  in test_pages_isolated()
634 		if (page && !is_migrate_isolate_page(page))  in test_pages_isolated()
637 	page = __first_valid_page(start_pfn, end_pfn - start_pfn);  in test_pages_isolated()
638 	if ((pfn < end_pfn) || !page) {  in test_pages_isolated()
644 	zone = page_zone(page);  in test_pages_isolated()