Lines Matching full:page
27 #include <linux/page-flags.h>
100 #include <asm/page.h>
128 * a zero page mapping on a read fault.
131 * related to the physical page in case of virtualization.
144 /* This function must be updated when the size of struct page grows above 80
151 static inline void __mm_zero_struct_page(struct page *page) in __mm_zero_struct_page() argument
153 unsigned long *_pp = (void *)page; in __mm_zero_struct_page()
155 /* Check that struct page is either 56, 64, 72, or 80 bytes */ in __mm_zero_struct_page()
156 BUILD_BUG_ON(sizeof(struct page) & 7); in __mm_zero_struct_page()
157 BUILD_BUG_ON(sizeof(struct page) < 56); in __mm_zero_struct_page()
158 BUILD_BUG_ON(sizeof(struct page) > 80); in __mm_zero_struct_page()
160 switch (sizeof(struct page)) { in __mm_zero_struct_page()
181 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
219 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) argument
221 /* to align the pointer to the (next) page boundary */
227 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
268 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
284 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
285 #define VM_SYNC 0x00800000 /* Synchronous page faults */
296 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
339 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
412 * low four bits) to a page protection mask..
431 * whether we would allow page faults to retry by specifying these two
434 * (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and
437 * (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and
440 * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry
443 * be used. Note that page faults can be allowed to retry for multiple times,
446 * signals before a retry to make sure the continuous page faults can still be
462 * arch-specific page fault handlers.
477 * Return: true if the page fault allows retry and this is the first
503 * MM layer fills up gfp_mask for page allocations but fault handler might
512 pgoff_t pgoff; /* Logical page offset based on vma */
521 struct page *cow_page; /* Page handler may use for COW fault */
522 struct page *page; /* ->fault handlers should return a member
523 * page here, unless VM_FAULT_NOPAGE
529 * the 'address'. NULL if the page
532 spinlock_t *ptl; /* Page table lock.
533 * Protects pte page table if 'pte'
536 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
540 * page table to avoid allocation from
545 /* page entry size for vm->huge_fault() */
555 * to the functions called when a no-page or a wp-page exception occurs.
569 /* notification that a previously read-only page is about to become
612 * page for @addr. This is useful if the default behavior
613 * (using pte_page()) would not find the correct page.
615 struct page *(*find_special_page)(struct vm_area_struct *vma,
690 * Methods to modify the page usage count.
692 * What counts for a page usage:
693 * - cache mapping (page->mapping)
694 * - private data (page->private)
695 * - page mapped in a task's page tables, each mapping
698 * Also, many kernel routines increase the page count before a critical
699 * routine so they can be sure the page doesn't go away from under them.
703 * Drop a ref, return true if the refcount fell to zero (the page has no users)
705 static inline int put_page_testzero(struct page *page) in put_page_testzero() argument
707 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); in put_page_testzero()
708 return page_ref_dec_and_test(page); in put_page_testzero()
712 * Try to grab a ref unless the page has a refcount of zero, return false if
717 static inline int get_page_unless_zero(struct page *page) in get_page_unless_zero() argument
719 return page_ref_add_unless(page, 1, 0); in get_page_unless_zero()
734 struct page *vmalloc_to_page(const void *addr);
794 static inline int head_compound_mapcount(struct page *head) in head_compound_mapcount()
800 * Mapcount of compound page as a whole, does not include mapped sub-pages.
804 static inline int compound_mapcount(struct page *page) in compound_mapcount() argument
806 VM_BUG_ON_PAGE(!PageCompound(page), page); in compound_mapcount()
807 page = compound_head(page); in compound_mapcount()
808 return head_compound_mapcount(page); in compound_mapcount()
812 * The atomic page->_mapcount, starts from -1: so that transitions
816 static inline void page_mapcount_reset(struct page *page) in page_mapcount_reset() argument
818 atomic_set(&(page)->_mapcount, -1); in page_mapcount_reset()
821 int __page_mapcount(struct page *page);
824 * Mapcount of 0-order page; when compound sub-page, includes
829 * They use this place in struct page differently.
831 static inline int page_mapcount(struct page *page) in page_mapcount() argument
833 if (unlikely(PageCompound(page))) in page_mapcount()
834 return __page_mapcount(page); in page_mapcount()
835 return atomic_read(&page->_mapcount) + 1; in page_mapcount()
839 int total_mapcount(struct page *page);
840 int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
842 static inline int total_mapcount(struct page *page) in total_mapcount() argument
844 return page_mapcount(page); in total_mapcount()
846 static inline int page_trans_huge_mapcount(struct page *page, in page_trans_huge_mapcount() argument
849 int mapcount = page_mapcount(page); in page_trans_huge_mapcount()
856 static inline struct page *virt_to_head_page(const void *x) in virt_to_head_page()
858 struct page *page = virt_to_page(x); in virt_to_head_page() local
860 return compound_head(page); in virt_to_head_page()
863 void __put_page(struct page *page);
867 void split_page(struct page *page, unsigned int order);
872 * These are _only_ valid on the head of a compound page.
874 typedef void compound_page_dtor(struct page *);
890 static inline void set_compound_page_dtor(struct page *page, in set_compound_page_dtor() argument
893 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page); in set_compound_page_dtor()
894 page[1].compound_dtor = compound_dtor; in set_compound_page_dtor()
897 static inline void destroy_compound_page(struct page *page) in destroy_compound_page() argument
899 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page); in destroy_compound_page()
900 compound_page_dtors[page[1].compound_dtor](page); in destroy_compound_page()
903 static inline unsigned int compound_order(struct page *page) in compound_order() argument
905 if (!PageHead(page)) in compound_order()
907 return page[1].compound_order; in compound_order()
910 static inline bool hpage_pincount_available(struct page *page) in hpage_pincount_available() argument
913 * Can the page->hpage_pinned_refcount field be used? That field is in in hpage_pincount_available()
914 * the 3rd page of the compound page, so the smallest (2-page) compound in hpage_pincount_available()
917 page = compound_head(page); in hpage_pincount_available()
918 return PageCompound(page) && compound_order(page) > 1; in hpage_pincount_available()
921 static inline int head_compound_pincount(struct page *head) in head_compound_pincount()
926 static inline int compound_pincount(struct page *page) in compound_pincount() argument
928 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); in compound_pincount()
929 page = compound_head(page); in compound_pincount()
930 return head_compound_pincount(page); in compound_pincount()
933 static inline void set_compound_order(struct page *page, unsigned int order) in set_compound_order() argument
935 page[1].compound_order = order; in set_compound_order()
936 page[1].compound_nr = 1U << order; in set_compound_order()
939 /* Returns the number of pages in this potentially compound page. */
940 static inline unsigned long compound_nr(struct page *page) in compound_nr() argument
942 if (!PageHead(page)) in compound_nr()
944 return page[1].compound_nr; in compound_nr()
947 /* Returns the number of bytes in this potentially compound page. */
948 static inline unsigned long page_size(struct page *page) in page_size() argument
950 return PAGE_SIZE << compound_order(page); in page_size()
953 /* Returns the number of bits needed for the number of bytes in a page */
954 static inline unsigned int page_shift(struct page *page) in page_shift() argument
956 return PAGE_SHIFT + compound_order(page); in page_shift()
959 void free_compound_page(struct page *page);
975 vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page);
981 * Multiple processes may "see" the same page. E.g. for untouched
982 * mappings of /dev/null, all processes see the same page full of
986 * For the non-reserved pages, page_count(page) denotes a reference count.
987 * page_count() == 0 means the page is free. page->lru is then used for
989 * page_count() > 0 means the page has been allocated.
993 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
997 * A page may be used by anyone else who does a __get_free_page().
999 * be used through the normal accessor functions. The top bits of page->flags
1000 * and page->virtual store page management information, but all other fields
1002 * page is the responsibility of the one who allocated it, and those who have
1009 * A pagecache page contains an opaque `private' member, which belongs to the
1010 * page's address_space. Usually, this is the address of a circular list of
1011 * the page's disk buffers. PG_private must be set to tell the VM to call
1014 * A page may belong to an inode's memory mapping. In this case, page->mapping
1015 * is the pointer to the inode, and page->index is the file offset of the page,
1020 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1023 * reference to the page. Setting PG_private should also increment the
1024 * refcount. The each user mapping also has a reference to the page.
1045 /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
1083 static inline enum zone_type page_zonenum(const struct page *page) in page_zonenum() argument
1085 ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT); in page_zonenum()
1086 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; in page_zonenum()
1090 static inline bool is_zone_device_page(const struct page *page) in is_zone_device_page() argument
1092 return page_zonenum(page) == ZONE_DEVICE; in is_zone_device_page()
1097 static inline bool is_zone_device_page(const struct page *page) in is_zone_device_page() argument
1104 void free_devmap_managed_page(struct page *page);
1107 static inline bool page_is_devmap_managed(struct page *page) in page_is_devmap_managed() argument
1111 if (!is_zone_device_page(page)) in page_is_devmap_managed()
1113 switch (page->pgmap->type) { in page_is_devmap_managed()
1123 void put_devmap_managed_page(struct page *page);
1126 static inline bool page_is_devmap_managed(struct page *page) in page_is_devmap_managed() argument
1131 static inline void put_devmap_managed_page(struct page *page) in put_devmap_managed_page() argument
1136 static inline bool is_device_private_page(const struct page *page) in is_device_private_page() argument
1140 is_zone_device_page(page) && in is_device_private_page()
1141 page->pgmap->type == MEMORY_DEVICE_PRIVATE; in is_device_private_page()
1144 static inline bool is_pci_p2pdma_page(const struct page *page) in is_pci_p2pdma_page() argument
1148 is_zone_device_page(page) && in is_pci_p2pdma_page()
1149 page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; in is_pci_p2pdma_page()
1153 #define page_ref_zero_or_close_to_overflow(page) \ argument
1154 ((unsigned int) page_ref_count(page) + 127u <= 127u)
1156 static inline void get_page(struct page *page) in get_page() argument
1158 page = compound_head(page); in get_page()
1160 * Getting a normal page or the head of a compound page in get_page()
1161 * requires to already have an elevated page->_refcount. in get_page()
1163 VM_BUG_ON_PAGE(page_ref_zero_or_close_to_overflow(page), page); in get_page()
1164 page_ref_inc(page); in get_page()
1167 bool __must_check try_grab_page(struct page *page, unsigned int flags);
1169 static inline __must_check bool try_get_page(struct page *page) in try_get_page() argument
1171 page = compound_head(page); in try_get_page()
1172 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) in try_get_page()
1174 page_ref_inc(page); in try_get_page()
1178 static inline void put_page(struct page *page) in put_page() argument
1180 page = compound_head(page); in put_page()
1184 * 2 to 1, when refcount reach one it means the page is free and we in put_page()
1188 if (page_is_devmap_managed(page)) { in put_page()
1189 put_devmap_managed_page(page); in put_page()
1193 if (put_page_testzero(page)) in put_page()
1194 __put_page(page); in put_page()
1199 * the page's refcount so that two separate items are tracked: the original page
1201 * made against the page. ("gup-pinned" is another term for the latter).
1209 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
1211 * simpler, due to the fact that adding an even power of two to the page
1220 * applications that don't have huge page reference counts, this won't be an
1225 * get_user_pages and page_mkclean and other calls that race to set up page
1230 void unpin_user_page(struct page *page);
1231 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1233 void unpin_user_pages(struct page **pages, unsigned long npages);
1236 * page_maybe_dma_pinned() - report if a page is pinned for DMA.
1238 * This function checks if a page has been pinned via a call to
1244 * GUP_PIN_COUNTING_BIAS worth of normal page references".
1246 * False positives are OK, because: a) it's unlikely for a page to get that many
1251 * more tracking data available: the 3rd struct page in the compound page is
1257 * @page: pointer to page to be queried.
1258 * @Return: True, if it is likely that the page has been "dma-pinned".
1259 * False, if the page is definitely not dma-pinned.
1261 static inline bool page_maybe_dma_pinned(struct page *page) in page_maybe_dma_pinned() argument
1263 if (hpage_pincount_available(page)) in page_maybe_dma_pinned()
1264 return compound_pincount(page) > 0; in page_maybe_dma_pinned()
1274 return ((unsigned int)page_ref_count(compound_head(page))) >= in page_maybe_dma_pinned()
1286 * node id available in page flags.
1290 static inline int page_zone_id(struct page *page) in page_zone_id() argument
1292 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; in page_zone_id()
1296 extern int page_to_nid(const struct page *page);
1298 static inline int page_to_nid(const struct page *page) in page_to_nid() argument
1300 struct page *p = (struct page *)page; in page_to_nid()
1344 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1346 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK); in page_cpupid_xchg_last()
1349 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1351 return page->_last_cpupid; in page_cpupid_last()
1353 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1355 page->_last_cpupid = -1 & LAST_CPUPID_MASK; in page_cpupid_reset_last()
1358 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1360 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; in page_cpupid_last()
1363 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1365 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1367 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; in page_cpupid_reset_last()
1371 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1373 return page_to_nid(page); /* XXX */ in page_cpupid_xchg_last()
1376 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1378 return page_to_nid(page); /* XXX */ in page_cpupid_last()
1406 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1417 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1419 return (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; in page_kasan_tag()
1422 static inline void page_kasan_tag_set(struct page *page, u8 tag) in page_kasan_tag_set() argument
1424 page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); in page_kasan_tag_set()
1425 page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; in page_kasan_tag_set()
1428 static inline void page_kasan_tag_reset(struct page *page) in page_kasan_tag_reset() argument
1430 page_kasan_tag_set(page, 0xff); in page_kasan_tag_reset()
1433 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1438 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } in page_kasan_tag_set() argument
1439 static inline void page_kasan_tag_reset(struct page *page) { } in page_kasan_tag_reset() argument
1442 static inline struct zone *page_zone(const struct page *page) in page_zone() argument
1444 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; in page_zone()
1447 static inline pg_data_t *page_pgdat(const struct page *page) in page_pgdat() argument
1449 return NODE_DATA(page_to_nid(page)); in page_pgdat()
1453 static inline void set_page_section(struct page *page, unsigned long section) in set_page_section() argument
1455 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); in set_page_section()
1456 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; in set_page_section()
1459 static inline unsigned long page_to_section(const struct page *page) in page_to_section() argument
1461 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; in page_to_section()
1465 static inline void set_page_zone(struct page *page, enum zone_type zone) in set_page_zone() argument
1467 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); in set_page_zone()
1468 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; in set_page_zone()
1471 static inline void set_page_node(struct page *page, unsigned long node) in set_page_node() argument
1473 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); in set_page_node()
1474 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; in set_page_node()
1477 static inline void set_page_links(struct page *page, enum zone_type zone, in set_page_links() argument
1480 set_page_zone(page, zone); in set_page_links()
1481 set_page_node(page, node); in set_page_links()
1483 set_page_section(page, pfn_to_section_nr(pfn)); in set_page_links()
1488 static inline struct mem_cgroup *page_memcg(struct page *page) in page_memcg() argument
1490 return page->mem_cgroup; in page_memcg()
1492 static inline struct mem_cgroup *page_memcg_rcu(struct page *page) in page_memcg_rcu() argument
1495 return READ_ONCE(page->mem_cgroup); in page_memcg_rcu()
1498 static inline struct mem_cgroup *page_memcg(struct page *page) in page_memcg() argument
1502 static inline struct mem_cgroup *page_memcg_rcu(struct page *page) in page_memcg_rcu() argument
1514 static __always_inline void *lowmem_page_address(const struct page *page) in lowmem_page_address() argument
1516 return page_to_virt(page); in lowmem_page_address()
1524 static inline void *page_address(const struct page *page) in page_address() argument
1526 return page->virtual; in page_address()
1528 static inline void set_page_address(struct page *page, void *address) in set_page_address() argument
1530 page->virtual = address; in set_page_address()
1536 void *page_address(const struct page *page);
1537 void set_page_address(struct page *page, void *virtual);
1542 #define page_address(page) lowmem_page_address(page) argument
1543 #define set_page_address(page, address) do { } while(0) argument
1547 extern void *page_rmapping(struct page *page);
1548 extern struct anon_vma *page_anon_vma(struct page *page);
1549 extern struct address_space *page_mapping(struct page *page);
1551 extern struct address_space *__page_file_mapping(struct page *);
1554 struct address_space *page_file_mapping(struct page *page) in page_file_mapping() argument
1556 if (unlikely(PageSwapCache(page))) in page_file_mapping()
1557 return __page_file_mapping(page); in page_file_mapping()
1559 return page->mapping; in page_file_mapping()
1562 extern pgoff_t __page_file_index(struct page *page);
1565 * Return the pagecache index of the passed page. Regular pagecache pages
1568 static inline pgoff_t page_index(struct page *page) in page_index() argument
1570 if (unlikely(PageSwapCache(page))) in page_index()
1571 return __page_file_index(page); in page_index()
1572 return page->index; in page_index()
1575 bool page_mapped(struct page *page);
1576 struct address_space *page_mapping(struct page *page);
1577 struct address_space *page_mapping_file(struct page *page);
1580 * Return true only if the page has been allocated with
1584 static inline bool page_is_pfmemalloc(struct page *page) in page_is_pfmemalloc() argument
1587 * Page index cannot be this large so this must be in page_is_pfmemalloc()
1588 * a pfmemalloc page. in page_is_pfmemalloc()
1590 return page->index == -1UL; in page_is_pfmemalloc()
1594 * Only to be called by the page allocator on a freshly allocated
1595 * page.
1597 static inline void set_page_pfmemalloc(struct page *page) in set_page_pfmemalloc() argument
1599 page->index = -1UL; in set_page_pfmemalloc()
1602 static inline void clear_page_pfmemalloc(struct page *page) in clear_page_pfmemalloc() argument
1604 page->index = 0; in clear_page_pfmemalloc()
1613 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) argument
1635 struct address_space *check_mapping; /* Check page->mapping if set */
1636 pgoff_t first_index; /* Lowest page->index to unmap */
1637 pgoff_t last_index; /* Highest page->index to unmap */
1640 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1642 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1672 int truncate_inode_page(struct address_space *mapping, struct page *page);
1673 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1674 int invalidate_inode_page(struct page *page);
1724 unsigned int gup_flags, struct page **pages,
1728 unsigned int gup_flags, struct page **pages,
1731 unsigned int gup_flags, struct page **pages,
1734 unsigned int gup_flags, struct page **pages,
1737 unsigned int gup_flags, struct page **pages, int *locked);
1739 unsigned int gup_flags, struct page **pages, int *locked);
1741 struct page **pages, unsigned int gup_flags);
1743 struct page **pages, unsigned int gup_flags);
1746 unsigned int gup_flags, struct page **pages);
1748 unsigned int gup_flags, struct page **pages);
1758 bool got_ref; /* Did we pin pages by getting page ref? */
1778 static inline struct page **frame_vector_pages(struct frame_vector *vec) in frame_vector_pages()
1786 return (struct page **)(vec->ptrs); in frame_vector_pages()
1798 struct page **pages);
1799 int get_kernel_page(unsigned long start, int write, struct page **pages);
1800 struct page *get_dump_page(unsigned long addr);
1802 extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1803 extern void do_invalidatepage(struct page *page, unsigned int offset,
1806 void __set_page_dirty(struct page *, struct address_space *, int warn);
1807 int __set_page_dirty_nobuffers(struct page *page);
1808 int __set_page_dirty_no_writeback(struct page *page);
1810 struct page *page);
1811 void account_page_dirtied(struct page *page, struct address_space *mapping);
1812 void account_page_cleaned(struct page *page, struct address_space *mapping,
1814 int set_page_dirty(struct page *page);
1815 int set_page_dirty_lock(struct page *page);
1816 void __cancel_dirty_page(struct page *page);
1817 static inline void cancel_dirty_page(struct page *page) in cancel_dirty_page() argument
1820 if (PageDirty(page)) in cancel_dirty_page()
1821 __cancel_dirty_page(page); in cancel_dirty_page()
1823 int clear_page_dirty_for_io(struct page *page);
1859 unsigned int gup_flags, struct page **pages);
1861 unsigned int gup_flags, struct page **pages);
1864 unsigned int gup_flags, struct page **pagep) in get_user_page_fast_only()
1909 /* Optimized variant when page is already known not to be PageAnon */
1910 static inline int mm_counter_file(struct page *page) in mm_counter_file() argument
1912 if (PageSwapBacked(page)) in mm_counter_file()
1917 static inline int mm_counter(struct page *page) in mm_counter() argument
1919 if (PageAnon(page)) in mm_counter()
1921 return mm_counter_file(page); in mm_counter()
2134 extern bool ptlock_alloc(struct page *page);
2135 extern void ptlock_free(struct page *page);
2137 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2139 return page->ptl; in ptlock_ptr()
2146 static inline bool ptlock_alloc(struct page *page) in ptlock_alloc() argument
2151 static inline void ptlock_free(struct page *page) in ptlock_free() argument
2155 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2157 return &page->ptl; in ptlock_ptr()
2166 static inline bool ptlock_init(struct page *page) in ptlock_init() argument
2169 * prep_new_page() initialize page->private (and therefore page->ptl) in ptlock_init()
2172 * It can happen if arch try to use slab for page table allocation: in ptlock_init()
2173 * slab code uses page->slab_cache, which share storage with page->ptl. in ptlock_init()
2175 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page); in ptlock_init()
2176 if (!ptlock_alloc(page)) in ptlock_init()
2178 spin_lock_init(ptlock_ptr(page)); in ptlock_init()
2191 static inline bool ptlock_init(struct page *page) { return true; } in ptlock_init() argument
2192 static inline void ptlock_free(struct page *page) {} in ptlock_free() argument
2201 static inline bool pgtable_pte_page_ctor(struct page *page) in pgtable_pte_page_ctor() argument
2203 if (!ptlock_init(page)) in pgtable_pte_page_ctor()
2205 __SetPageTable(page); in pgtable_pte_page_ctor()
2206 inc_zone_page_state(page, NR_PAGETABLE); in pgtable_pte_page_ctor()
2210 static inline void pgtable_pte_page_dtor(struct page *page) in pgtable_pte_page_dtor() argument
2212 ptlock_free(page); in pgtable_pte_page_dtor()
2213 __ClearPageTable(page); in pgtable_pte_page_dtor()
2214 dec_zone_page_state(page, NR_PAGETABLE); in pgtable_pte_page_dtor()
2246 static struct page *pmd_to_page(pmd_t *pmd) in pmd_to_page()
2257 static inline bool pmd_ptlock_init(struct page *page) in pmd_ptlock_init() argument
2260 page->pmd_huge_pte = NULL; in pmd_ptlock_init()
2262 return ptlock_init(page); in pmd_ptlock_init()
2265 static inline void pmd_ptlock_free(struct page *page) in pmd_ptlock_free() argument
2268 VM_BUG_ON_PAGE(page->pmd_huge_pte, page); in pmd_ptlock_free()
2270 ptlock_free(page); in pmd_ptlock_free()
2282 static inline bool pmd_ptlock_init(struct page *page) { return true; } in pmd_ptlock_init() argument
2283 static inline void pmd_ptlock_free(struct page *page) {} in pmd_ptlock_free() argument
2296 static inline bool pgtable_pmd_page_ctor(struct page *page) in pgtable_pmd_page_ctor() argument
2298 if (!pmd_ptlock_init(page)) in pgtable_pmd_page_ctor()
2300 __SetPageTable(page); in pgtable_pmd_page_ctor()
2301 inc_zone_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_ctor()
2305 static inline void pgtable_pmd_page_dtor(struct page *page) in pgtable_pmd_page_dtor() argument
2307 pmd_ptlock_free(page); in pgtable_pmd_page_dtor()
2308 __ClearPageTable(page); in pgtable_pmd_page_dtor()
2309 dec_zone_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_dtor()
2346 * Free a highmem page into the buddy system, adjusting totalhigh_pages
2349 extern void free_highmem_page(struct page *page);
2352 extern void adjust_managed_page_count(struct page *page, long count);
2357 /* Free the reserved page into the buddy system, so it gets managed. */
2358 static inline void __free_reserved_page(struct page *page) in __free_reserved_page() argument
2360 ClearPageReserved(page); in __free_reserved_page()
2361 init_page_count(page); in __free_reserved_page()
2362 __free_page(page); in __free_reserved_page()
2365 static inline void free_reserved_page(struct page *page) in free_reserved_page() argument
2367 __free_reserved_page(page); in free_reserved_page()
2368 adjust_managed_page_count(page, 1); in free_reserved_page()
2371 static inline void mark_page_reserved(struct page *page) in mark_page_reserved() argument
2373 SetPageReserved(page); in mark_page_reserved()
2374 adjust_managed_page_count(page, -1); in mark_page_reserved()
2407 * An architecture is expected to register range of page frames backed by
2566 unsigned long flags, struct page **pages);
2628 /* mm/page-writeback.c */
2629 int __must_check write_one_page(struct page *page);
2730 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2732 struct page **pages, unsigned long *num);
2733 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2735 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2750 unsigned long addr, struct page *page) in vmf_insert_page() argument
2752 int err = vm_insert_page(vma, addr, page); in vmf_insert_page()
2778 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
2782 #define FOLL_TOUCH 0x02 /* mark page accessed */
2783 #define FOLL_GET 0x04 /* do get_page on page */
2788 #define FOLL_POPULATE 0x40 /* fault in page */
2790 #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2791 #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2792 #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2807 * FOLL_LONGTERM indicates that the page will be held for an indefinite time
2815 * pin, delaying writeback, bounce buffer page writeback, etc. As FS DAX was
2827 * that region. And so, CMA attempts to migrate the page before pinning, when
2830 * FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
2832 * anything that gets a page reference and then touches page data (for example,
2879 extern void kernel_poison_pages(struct page *page, int numpages, int enable);
2882 static inline void kernel_poison_pages(struct page *page, int numpages, in kernel_poison_pages() argument
2937 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2944 kernel_map_pages(struct page *page, int numpages, int enable) in kernel_map_pages() argument
2946 __kernel_map_pages(page, numpages, enable); in kernel_map_pages()
2949 extern bool kernel_page_present(struct page *page);
2953 kernel_map_pages(struct page *page, int numpages, int enable) {} in kernel_map_pages() argument
2955 static inline bool kernel_page_present(struct page *page) { return true; } in kernel_page_present() argument
3002 struct page * __populate_section_memmap(unsigned long pfn,
3024 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
3039 extern void shake_page(struct page *p, int access);
3081 extern void clear_huge_page(struct page *page,
3084 extern void copy_user_huge_page(struct page *dst, struct page *src,
3088 extern long copy_huge_page_from_user(struct page *dst_page,
3094 * vma_is_special_huge - Are transhuge page-table entries considered special?
3097 * Whether transhuge page-table entries are considered "special" following
3100 * Return: true if transhuge page-table entries should be considered special,
3125 static inline bool page_is_guard(struct page *page) in page_is_guard() argument
3130 return PageGuard(page); in page_is_guard()
3135 static inline bool page_is_guard(struct page *page) { return false; } in page_is_guard() argument
3144 extern int memcmp_pages(struct page *page1, struct page *page2);
3146 static inline int pages_identical(struct page *page1, struct page *page2) in pages_identical()