Lines Matching full:page
3 * Macros for manipulating and testing page->flags
18 * Various page->flags bits:
20 * PG_reserved is set for special pages. The "struct page" of such a page
25 * - Pages reserved or allocated early during boot (before the page allocator
27 * initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
29 * be given to the page allocator.
32 * - The zero page(s)
33 * - Pages not added to the page allocator when onlining a section because
49 * Consequently, PG_reserved for a page mapped into user space can indicate
50 * the zero page, the vDSO, MMIO pages or device memory.
53 * specific data (which is normally at page->private). It can be used by
60 * PG_locked also pins a page in pagecache, and blocks truncation of the file
63 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
66 * PG_swapbacked is set when a page uses swap as a backing storage. This are
71 * PG_uptodate tells whether the page's contents is valid. When a read
72 * completes, the page becomes uptodate, unless a disk I/O error happened.
74 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
77 * PG_error is set to indicate that an I/O error occurred on this page.
79 * PG_arch_1 is an architecture specific page state bit. The generic code
80 * guarantees that this bit is cleared for a page when it first is entered into
81 * the page cache.
83 * PG_hwpoison indicates that a page got corrupted in hardware and contains
90 * locked- and dirty-page accounting.
92 * The page flags field is split into two parts, the main flags area
105 PG_locked, /* Page is locked. Don't touch. */
112 …PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_…
120 PG_writeback, /* Page is under writeback */
121 PG_head, /* A head page */
124 PG_swapbacked, /* Page is backed by RAM/swap */
125 PG_unevictable, /* Page is "unevictable" */
127 PG_mlocked, /* Page is vma mlocked */
130 PG_uncached, /* Page has been mapped as uncached */
133 PG_hwpoison, /* hardware poisoned page. Don't touch */
148 PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */
150 /* Two page bits are conscripted by FS-Cache to maintain local caching
154 PG_fscache = PG_private_2, /* page backed by cache */
157 /* Pinned in Xen as a read-only pagetable page. */
161 /* Has a grant mapping of another (foreign) domain's page. */
169 /* Compound pages. Stored in first tail page's flags */
172 /* non-lru isolated movable page */
181 struct page; /* forward declaration */
183 static inline struct page *compound_head(struct page *page) in compound_head() argument
185 unsigned long head = READ_ONCE(page->compound_head); in compound_head()
188 return (struct page *) (head - 1); in compound_head()
189 return page; in compound_head()
192 static __always_inline int PageTail(struct page *page) in PageTail() argument
194 return READ_ONCE(page->compound_head) & 1; in PageTail()
197 static __always_inline int PageCompound(struct page *page) in PageCompound() argument
199 return test_bit(PG_head, &page->flags) || PageTail(page); in PageCompound()
203 static inline int PagePoisoned(const struct page *page) in PagePoisoned() argument
205 return page->flags == PAGE_POISON_PATTERN; in PagePoisoned()
209 void page_init_poison(struct page *page, size_t size);
211 static inline void page_init_poison(struct page *page, size_t size) in page_init_poison() argument
217 * Page flags policies wrt compound pages
220 * check if this struct page poisoned/uninitialized
223 * the page flag is relevant for small, head and tail pages.
226 * for compound page all operations related to the page flag applied to
227 * head page.
230 * for compound page, callers only ever operate on the head page.
233 * modifications of the page flag must be done on small or head pages,
237 * the page flag is not relevant for compound pages.
240 * the page flag is stored in the first tail page.
242 #define PF_POISONED_CHECK(page) ({ \ argument
243 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
244 page; })
245 #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) argument
246 #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) argument
247 #define PF_ONLY_HEAD(page, enforce) ({ \ argument
248 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
249 PF_POISONED_CHECK(page); })
250 #define PF_NO_TAIL(page, enforce) ({ \ argument
251 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
252 PF_POISONED_CHECK(compound_head(page)); })
253 #define PF_NO_COMPOUND(page, enforce) ({ \ argument
254 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
255 PF_POISONED_CHECK(page); })
256 #define PF_SECOND(page, enforce) ({ \ argument
257 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
258 PF_POISONED_CHECK(&page[1]); })
261 * Macros to create function definitions for page flags
264 static __always_inline int Page##uname(struct page *page) \
265 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
268 static __always_inline void SetPage##uname(struct page *page) \
269 { set_bit(PG_##lname, &policy(page, 1)->flags); }
272 static __always_inline void ClearPage##uname(struct page *page) \
273 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
276 static __always_inline void __SetPage##uname(struct page *page) \
277 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
280 static __always_inline void __ClearPage##uname(struct page *page) \
281 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
284 static __always_inline int TestSetPage##uname(struct page *page) \
285 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
288 static __always_inline int TestClearPage##uname(struct page *page) \
289 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
306 static inline int Page##uname(const struct page *page) { return 0; }
309 static inline void SetPage##uname(struct page *page) { }
312 static inline void ClearPage##uname(struct page *page) { }
315 static inline void __ClearPage##uname(struct page *page) { }
318 static inline int TestSetPage##uname(struct page *page) { return 0; }
321 static inline int TestClearPage##uname(struct page *page) { return 0; }
362 * Private page markings that may be used by the filesystem that owns the page in PAGEFLAG()
374 * risky: they bypass page accounting. in PAGEFLAG()
397 static __always_inline int PageSwapCache(struct page *page) in PAGEFLAG()
400 page = compound_head(page); in PAGEFLAG()
402 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); in PAGEFLAG()
434 extern bool take_page_off_buddy(struct page *page);
456 * On an anonymous page mapped into a user virtual memory area, in TESTPAGEFLAG()
457 * page->mapping points to its anon_vma, not to a struct address_space; in TESTPAGEFLAG()
460 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled, in TESTPAGEFLAG()
462 * bit; and then page->mapping points, not to an anon_vma, but to a private in TESTPAGEFLAG()
463 * structure which KSM associates with that merged page. See ksm.h. in TESTPAGEFLAG()
466 * page and then page->mapping points a struct address_space. in TESTPAGEFLAG()
469 * address_space which maps the page from disk; whereas "page_mapped" in TESTPAGEFLAG()
470 * refers to user virtual address space into which the page is mapped. in TESTPAGEFLAG()
477 static __always_inline int PageMappingFlags(struct page *page) in TESTPAGEFLAG()
479 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; in TESTPAGEFLAG()
482 static __always_inline int PageAnon(struct page *page) in PageAnon() argument
484 page = compound_head(page); in PageAnon()
485 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0; in PageAnon()
488 static __always_inline int __PageMovable(struct page *page) in __PageMovable() argument
490 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in __PageMovable()
496 * A KSM page is one of those write-protected "shared pages" or "merged pages"
497 * which KSM maps into multiple mms, wherever identical anonymous page content
498 * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
499 * anon_vma, but to that page's node of the stable tree.
501 static __always_inline int PageKsm(struct page *page) in PageKsm() argument
503 page = compound_head(page); in PageKsm()
504 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in PageKsm()
511 u64 stable_page_flags(struct page *page);
513 static inline int PageUptodate(struct page *page) in PageUptodate() argument
516 page = compound_head(page); in PageUptodate()
517 ret = test_bit(PG_uptodate, &(page)->flags); in PageUptodate()
519 * Must ensure that the data we read out of the page is loaded in PageUptodate()
520 * _after_ we've loaded page->flags to check for PageUptodate. in PageUptodate()
521 * We can skip the barrier if the page is not uptodate, because in PageUptodate()
532 static __always_inline void __SetPageUptodate(struct page *page) in __SetPageUptodate() argument
534 VM_BUG_ON_PAGE(PageTail(page), page); in __SetPageUptodate()
536 __set_bit(PG_uptodate, &page->flags); in __SetPageUptodate()
539 static __always_inline void SetPageUptodate(struct page *page) in SetPageUptodate() argument
541 VM_BUG_ON_PAGE(PageTail(page), page); in SetPageUptodate()
544 * so that all previous stores issued in order to bring the page in SetPageUptodate()
548 set_bit(PG_uptodate, &page->flags); in SetPageUptodate()
553 int test_clear_page_writeback(struct page *page);
554 int __test_set_page_writeback(struct page *page, bool keep_write);
556 #define test_set_page_writeback(page) \ argument
557 __test_set_page_writeback(page, false)
558 #define test_set_page_writeback_keepwrite(page) \ argument
559 __test_set_page_writeback(page, true)
561 static inline void set_page_writeback(struct page *page) in set_page_writeback() argument
563 test_set_page_writeback(page); in set_page_writeback()
566 static inline void set_page_writeback_keepwrite(struct page *page) in set_page_writeback_keepwrite() argument
568 test_set_page_writeback_keepwrite(page); in set_page_writeback_keepwrite()
573 static __always_inline void set_compound_head(struct page *page, struct page *head) in __PAGEFLAG()
575 WRITE_ONCE(page->compound_head, (unsigned long)head + 1); in __PAGEFLAG()
578 static __always_inline void clear_compound_head(struct page *page) in clear_compound_head() argument
580 WRITE_ONCE(page->compound_head, 0); in clear_compound_head()
584 static inline void ClearPageCompound(struct page *page) in ClearPageCompound() argument
586 BUG_ON(!PageHead(page)); in ClearPageCompound()
587 ClearPageHead(page); in ClearPageCompound()
594 int PageHuge(struct page *page);
595 int PageHeadHuge(struct page *page);
596 bool page_huge_active(struct page *page);
601 static inline bool page_huge_active(struct page *page) in TESTPAGEFLAG_FALSE()
617 static inline int PageTransHuge(struct page *page) in PageTransHuge() argument
619 VM_BUG_ON_PAGE(PageTail(page), page); in PageTransHuge()
620 return PageHead(page); in PageTransHuge()
628 static inline int PageTransCompound(struct page *page) in PageTransCompound() argument
630 return PageCompound(page); in PageTransCompound()
635 * guarantees the primary MMU has the entire compound page mapped
637 * can also map the entire compound page. This allows the secondary
638 * MMUs to call get_user_pages() only once for each compound page and
639 * to immediately map the entire compound page with a single secondary
646 * MMU notifier, otherwise it may result in page->_mapcount check false
649 * We have to treat page cache THP differently since every subpage of it
654 static inline int PageTransCompoundMap(struct page *page) in PageTransCompoundMap() argument
656 struct page *head; in PageTransCompoundMap()
658 if (!PageTransCompound(page)) in PageTransCompoundMap()
661 if (PageAnon(page)) in PageTransCompoundMap()
662 return atomic_read(&page->_mapcount) < 0; in PageTransCompoundMap()
664 head = compound_head(page); in PageTransCompoundMap()
666 return atomic_read(&page->_mapcount) == in PageTransCompoundMap()
675 static inline int PageTransTail(struct page *page) in PageTransTail() argument
677 return PageTail(page); in PageTransTail()
681 * PageDoubleMap indicates that the compound page is mapped with PTEs as well
685 * per small page mapcount accounting (and its overhead from atomic operations)
688 * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
710 * mistaken for a page type value. in PAGEFLAG()
722 #define PageType(page, flag) \ in PAGEFLAG() argument
723 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE) in PAGEFLAG()
725 static inline int page_has_type(struct page *page) in PAGEFLAG()
727 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE; in PAGEFLAG()
731 static __always_inline int Page##uname(struct page *page) \
733 return PageType(page, PG_##lname); \
735 static __always_inline void __SetPage##uname(struct page *page) \
737 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
738 page->page_type &= ~PG_##lname; \
740 static __always_inline void __ClearPage##uname(struct page *page) \
742 VM_BUG_ON_PAGE(!Page##uname(page), page); \
743 page->page_type |= PG_##lname; \
747 * PageBuddy() indicates that the page is free and in the buddy system
753 * PageOffline() indicates that the page is logically offline although the
773 * pages allocated with __GFP_ACCOUNT. It gets cleared on page free.
778 * Marks pages in use as page tables.
787 extern bool is_free_buddy_page(struct page *page);
795 static inline int PageSlabPfmemalloc(struct page *page) in PageSlabPfmemalloc() argument
797 VM_BUG_ON_PAGE(!PageSlab(page), page); in PageSlabPfmemalloc()
798 return PageActive(page); in PageSlabPfmemalloc()
801 static inline void SetPageSlabPfmemalloc(struct page *page) in SetPageSlabPfmemalloc() argument
803 VM_BUG_ON_PAGE(!PageSlab(page), page); in SetPageSlabPfmemalloc()
804 SetPageActive(page); in SetPageSlabPfmemalloc()
807 static inline void __ClearPageSlabPfmemalloc(struct page *page) in __ClearPageSlabPfmemalloc() argument
809 VM_BUG_ON_PAGE(!PageSlab(page), page); in __ClearPageSlabPfmemalloc()
810 __ClearPageActive(page); in __ClearPageSlabPfmemalloc()
813 static inline void ClearPageSlabPfmemalloc(struct page *page) in ClearPageSlabPfmemalloc() argument
815 VM_BUG_ON_PAGE(!PageSlab(page), page); in ClearPageSlabPfmemalloc()
816 ClearPageActive(page); in ClearPageSlabPfmemalloc()
826 * Flags checked when a page is freed. Pages being freed should not have
837 * Flags checked when a page is prepped for return by the page allocator.
839 * there has been a kernel bug or struct page corruption.
841 * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
842 * alloc-free cycle to prevent from reusing the page.
850 * page_has_private - Determine if page has private stuff
851 * @page: The page to be checked
853 * Determine if a page has private stuff, indicating that release routines
856 static inline int page_has_private(struct page *page) in page_has_private() argument
858 return !!(page->flags & PAGE_FLAGS_PRIVATE); in page_has_private()