1 /* SPDX-License-Identifier: GPL-2.0 */
3  * Macros for manipulating and testing page->flags
18  * Various page->flags bits:
20  * PG_reserved is set for special pages. The "struct page" of such a page
22  * Pages marked as PG_reserved include:
23  * - Pages part of the kernel image (including vDSO) and similar (e.g. BIOS,
25  * - Pages reserved or allocated early during boot (before the page allocator
30  * - Pages falling into physical memory gaps - not IORESOURCE_SYSRAM. Trying
31  *   to read/write these pages might end badly. Don't touch!
32  * - The zero page(s)
33  * - Pages allocated in the context of kexec/kdump (loaded kernel image,
34  *   control pages, vmcoreinfo)
35  * - MMIO/DMA pages. Some architectures don't allow to ioremap pages that are
36  *   not marked PG_reserved (as they might be in use by somebody else who does
38  * - MCA pages on ia64
39  * - Pages holding CPU notes for POWER Firmware Assisted Dump
40  * - Device memory (e.g. PMEM, DAX, HMM)
41  * Some PG_reserved pages will be excluded from the hibernation image.
45  * the zero page, the vDSO, MMIO pages or device memory.
47  * The PG_private bitflag is set on pagecache pages if they contain filesystem
48  * specific data (which is normally at page->private). It can be used by
62  * usually PageAnon or shmem pages but please note that even anonymous pages
67  * file-backed pagecache (see mm/vmscan.c).
79  * Don't use the pageflags directly.  Use the PageFoo macros.
86  *  N-1           ^       0
104 	PG_owner_priv_1,	/* Owner use. If pagecache, fs may use */
105 	PG_owner_2,		/* Owner use. If pagecache, fs may use */
108 	PG_private,		/* If pagecache, has fs-private data */
113 	PG_dropbehind,		/* drop pages on IO completion */
141 	 * table (e.g., single PMD/PUD/CONT of the head page vs. PTE-mapped
143 	 * tail pages of an anonymous folio. For now, we only expect it to be
144 	 * set on tail pages for PTE-mapped THP.
150 	 * Filesystems which do not use BHs can use it for their own purpose.
154 	/* Two page bits are conscripted by FS-Cache to maintain local caching
155 	 * state.  These bits are set on pages belonging to the netfs's inodes
161 	/* Pinned in Xen as a read-only pagetable page. */
167 	/* Remapped by swiotlb-xen. */
170 	/* non-lru isolated movable page */
173 	/* Only valid for buddy pages. Used to track pages that are reported */
177 	/* For self-hosted memmap pages */
182 	 * Flags only valid for compound pages.  Stored in first tail page's
183 	 * flags word.  Cannot use the first 8 flags or any flag marked as
189 	PG_large_rmappable = PG_workingset, /* anon or file-backed */
193 #define PAGEFLAGS_MASK		((1UL << NR_PAGEFLAGS) - 1)
216 	    test_bit(PG_head, &page->flags)) {  in page_fixed_fake_head()
220 		 * two contiguous pages.  in page_fixed_fake_head()
225 			return (const struct page *)(head - 1);  in page_fixed_fake_head()
236 	 * The refcount check is ordered before the fake-head check to prevent  in page_count_writable()
250 	 *                                   atomic_add_unless(&page->_refcount)  in page_count_writable()
254 	 * tail pages that are not fake heads.  in page_count_writable()
256 	if (atomic_read_acquire(&page->_refcount) == u)  in page_count_writable()
280 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
283 		return head - 1;  in _compound_head()
290  * page_folio - Converts from page to folio.
298  * it should re-check the folio still contains this page after gaining
307  * folio_page - Return a page from a folio.
315 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
319 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);  in PageTail()
324 	return test_bit(PG_head, &page->flags) ||  in PageCompound()
325 	       READ_ONCE(page->compound_head) & 1;  in PageCompound()
328 #define	PAGE_POISON_PATTERN	-1l
331 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;  in PagePoisoned()
345 	const struct page *page = &folio->page;  in const_folio_flags()
347 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in const_folio_flags()
348 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in const_folio_flags()
354 	struct page *page = &folio->page;  in folio_flags()
356 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in folio_flags()
357 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in folio_flags()
362  * Page flags policies wrt compound pages
368  *     the page flag is relevant for small, head and tail pages.
375  *     modifications of the page flag must be done on small or head pages,
376  *     checks can be done on tail pages too.
379  *     the page flag is not relevant for compound pages.
448 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
453 { set_bit(PG_##lname, &policy(page, 1)->flags); }
458 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
463 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
468 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
473 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
478 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
581  * - PG_private and PG_private_2 cause release_folio() and co to be invoked  in PAGEFLAG()
586 /* owner_2 can be set on tail pages for anon memory */  in PAGEFLAG()
590  * Only test-and-set exist for PG_writeback.  The unconditional operators are  in PAGEFLAG()
609  * Must use a macro here due to header dependency issues. page_zone() is not  in PAGEFLAG()
678  * PageReported() is used to track reported free pages within the Buddy  in FOLIO_SET_FLAG()
679  * allocator. We can use the non-atomic version of the test and set  in FOLIO_SET_FLAG()
693  * folio->mapping points to its anon_vma, not to a struct address_space;  in FOLIO_SET_FLAG()
698  * bit; and then folio->mapping points, not to an anon_vma, but to a private  in FOLIO_SET_FLAG()
701  * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is used for non-lru movable  in FOLIO_SET_FLAG()
702  * page and then folio->mapping points to a struct movable_operations.  in FOLIO_SET_FLAG()
708  * For slab pages, since slab reuses the bits in struct page to store its  in FOLIO_SET_FLAG()
709  * internal states, the folio->mapping does not exist as such, nor do  in FOLIO_SET_FLAG()
710  * these flags below.  So in order to avoid testing non-existent bits,  in FOLIO_SET_FLAG()
722 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0;  in FOLIO_SET_FLAG()
727 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in PageMappingFlags()
732 	return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0;  in folio_test_anon()
737 	unsigned long flags = (unsigned long)page_folio(page)->mapping;  in PageAnonNotKsm()
749 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==  in __folio_test_movable()
755 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
761  * A KSM page is one of those write-protected "shared pages" or "merged pages"
768 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==  in folio_test_ksm()
778  * folio_xor_flags_has_waiters - Change some folio flags.
783  * lock held.  For example, it is unsafe to use for PG_dirty as that
785  * on flags which are in the range 0-6 as some of the implementations
797  * folio_test_uptodate - Is this folio up to date?
811 	 * _after_ we've loaded folio->flags to check the uptodate bit.  in folio_test_uptodate()
873 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);  in PageHead()
881  * folio_test_large() - Does this folio contain more than one page?  in __SETPAGEFLAG()
893 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in set_compound_head()
898 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
918  * PageHuge() only returns true for hugetlbfs pages, but not for  in FOLIO_FLAG()
919  * normal or transparent huge pages.  in FOLIO_FLAG()
922  * hugetlbfs pages, but not normal pages. PageTransHuge() can only be  in FOLIO_FLAG()
923  * called only in the core VM paths where hugetlbfs pages can't exist.  in FOLIO_FLAG()
932  * PageTransCompound returns true for both transparent huge pages
933  * and hugetlbfs pages, so it should only be called when it's known
934  * that hugetlbfs pages aren't involved.
958  * For pages that do not use mapcount, page_type may be used.
964 	/* 0x00-0x7f are positive numbers, ie mapcount */
965 	/* Reserve 0x80-0xef for mapcount overflow. */
984 /* This takes a mapcount which is one more than page->_mapcount */
987 	return page_type_has_type(mapcount - 1);  in page_mapcount_is_type()
992 	return page_mapcount_is_type(data_race(page->page_type));  in page_has_type()
998 	return data_race(folio->page.page_type >> 24) == PGTY_##lname;	\
1004 	VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX,	\
1006 	folio->page.page_type = (unsigned int)PGTY_##lname << 24;	\
1010 	if (folio->page.page_type == UINT_MAX)				\
1013 	folio->page.page_type = UINT_MAX;				\
1020 	return data_race(page->page_type >> 24) == PGTY_##lname;	\
1026 	VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page);	\
1027 	page->page_type = (unsigned int)PGTY_##lname << 24;		\
1031 	if (page->page_type == UINT_MAX)				\
1034 	page->page_type = UINT_MAX;					\
1047  * The content of these pages is effectively stale. Such pages should not
1050  * When a memory block gets onlined, all pages are initialized with a
1054  * If a driver wants to allow to offline unmovable PageOffline() pages without
1058  * pages (now with a reference count of zero) are treated like free (unmanaged)
1059  * pages, allowing the containing memory block to get offlined. A driver that
1060  * relies on this feature is aware that re-onlining the memory block will
1064  * PageOffline() pages, treating them like they were never exposed to the
1069  * pages should check PageOffline() and synchronize with such drivers using
1080  * Marks pages in use as page tables.
1087 PAGE_TYPE_OPS(Guard, guard, guard)  in PAGE_TYPE_OPS()
1092  * PageSlab - Determine if the page belongs to the slab allocator  in PAGE_TYPE_OPS()
1096  * Return: True for slab pages, false for any other kind of page.  in PAGE_TYPE_OPS()
1112  * Mark pages that has to be accepted before touched for the first time.  in FOLIO_TYPE_OPS()
1120  * PageHuge - Determine if the page belongs to hugetlbfs  in FOLIO_TYPE_OPS()
1124  * Return: True for hugetlbfs pages, false for anon pages or pages  in FOLIO_TYPE_OPS()
1144 	return folio_test_hugetlb(folio) && PageHWPoison(&folio->page);  in is_page_hwpoison()
1166 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in PageAnonExclusive()
1173 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in SetPageAnonExclusive()
1180 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in ClearPageAnonExclusive()
1187 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in __ClearPageAnonExclusive()
1197  * Flags checked when a page is freed.  Pages being freed should not have
1209  * Pages being prepped should not have these flags set.  If they are set,
1213  * alloc-free cycle to prevent from reusing the page.
1229  * folio_has_private - Determine if folio has private stuff
1237 	return !!(folio->flags & PAGE_FLAGS_PRIVATE);  in folio_has_private()
1242 	return test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids);  in folio_test_large_maybe_mapped_shared()