7 #include <linux/page-flags.h>
29 extern void __page_ref_set(struct page *page, int v);
30 extern void __page_ref_mod(struct page *page, int v);
31 extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
32 extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
33 extern void __page_ref_mod_unless(struct page *page, int v, int u);
34 extern void __page_ref_freeze(struct page *page, int v, int ret);
35 extern void __page_ref_unfreeze(struct page *page, int v);
41 static inline void __page_ref_set(struct page *page, int v)  in __page_ref_set()  argument
44 static inline void __page_ref_mod(struct page *page, int v)  in __page_ref_mod()  argument
47 static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)  in __page_ref_mod_and_test()  argument
50 static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)  in __page_ref_mod_and_return()  argument
53 static inline void __page_ref_mod_unless(struct page *page, int v, int u)  in __page_ref_mod_unless()  argument
56 static inline void __page_ref_freeze(struct page *page, int v, int ret)  in __page_ref_freeze()  argument
59 static inline void __page_ref_unfreeze(struct page *page, int v)  in __page_ref_unfreeze()  argument
65 static inline int page_ref_count(const struct page *page)  in page_ref_count()  argument
67 	return atomic_read(&page->_refcount);  in page_ref_count()
78  * - Each reference from a page table
79  * - The page cache
83  * - Direct IO which references this page in the process address space
89 	return page_ref_count(&folio->page);  in folio_ref_count()
92 static inline int page_count(const struct page *page)  in page_count()  argument
94 	return folio_ref_count(page_folio(page));  in page_count()
97 static inline void set_page_count(struct page *page, int v)  in set_page_count()  argument
99 	atomic_set(&page->_refcount, v);  in set_page_count()
101 		__page_ref_set(page, v);  in set_page_count()
106 	set_page_count(&folio->page, v);  in folio_set_count()
110  * Setup the page count before being freed into the page allocator for
113 static inline void init_page_count(struct page *page)  in init_page_count()  argument
115 	set_page_count(page, 1);  in init_page_count()
118 static inline void page_ref_add(struct page *page, int nr)  in page_ref_add()  argument
120 	atomic_add(nr, &page->_refcount);  in page_ref_add()
122 		__page_ref_mod(page, nr);  in page_ref_add()
127 	page_ref_add(&folio->page, nr);  in folio_ref_add()
130 static inline void page_ref_sub(struct page *page, int nr)  in page_ref_sub()  argument
132 	atomic_sub(nr, &page->_refcount);  in page_ref_sub()
134 		__page_ref_mod(page, -nr);  in page_ref_sub()
139 	page_ref_sub(&folio->page, nr);  in folio_ref_sub()
142 static inline int page_ref_sub_return(struct page *page, int nr)  in page_ref_sub_return()  argument
144 	int ret = atomic_sub_return(nr, &page->_refcount);  in page_ref_sub_return()
147 		__page_ref_mod_and_return(page, -nr, ret);  in page_ref_sub_return()
153 	return page_ref_sub_return(&folio->page, nr);  in folio_ref_sub_return()
156 static inline void page_ref_inc(struct page *page)  in page_ref_inc()  argument
158 	atomic_inc(&page->_refcount);  in page_ref_inc()
160 		__page_ref_mod(page, 1);  in page_ref_inc()
165 	page_ref_inc(&folio->page);  in folio_ref_inc()
168 static inline void page_ref_dec(struct page *page)  in page_ref_dec()  argument
170 	atomic_dec(&page->_refcount);  in page_ref_dec()
172 		__page_ref_mod(page, -1);  in page_ref_dec()
177 	page_ref_dec(&folio->page);  in folio_ref_dec()
180 static inline int page_ref_sub_and_test(struct page *page, int nr)  in page_ref_sub_and_test()  argument
182 	int ret = atomic_sub_and_test(nr, &page->_refcount);  in page_ref_sub_and_test()
185 		__page_ref_mod_and_test(page, -nr, ret);  in page_ref_sub_and_test()
191 	return page_ref_sub_and_test(&folio->page, nr);  in folio_ref_sub_and_test()
194 static inline int page_ref_inc_return(struct page *page)  in page_ref_inc_return()  argument
196 	int ret = atomic_inc_return(&page->_refcount);  in page_ref_inc_return()
199 		__page_ref_mod_and_return(page, 1, ret);  in page_ref_inc_return()
205 	return page_ref_inc_return(&folio->page);  in folio_ref_inc_return()
208 static inline int page_ref_dec_and_test(struct page *page)  in page_ref_dec_and_test()  argument
210 	int ret = atomic_dec_and_test(&page->_refcount);  in page_ref_dec_and_test()
213 		__page_ref_mod_and_test(page, -1, ret);  in page_ref_dec_and_test()
219 	return page_ref_dec_and_test(&folio->page);  in folio_ref_dec_and_test()
222 static inline int page_ref_dec_return(struct page *page)  in page_ref_dec_return()  argument
224 	int ret = atomic_dec_return(&page->_refcount);  in page_ref_dec_return()
227 		__page_ref_mod_and_return(page, -1, ret);  in page_ref_dec_return()
233 	return page_ref_dec_return(&folio->page);  in folio_ref_dec_return()
236 static inline bool page_ref_add_unless(struct page *page, int nr, int u)  in page_ref_add_unless()  argument
238 	bool ret = atomic_add_unless(&page->_refcount, nr, u);  in page_ref_add_unless()
241 		__page_ref_mod_unless(page, nr, ret);  in page_ref_add_unless()
247 	return page_ref_add_unless(&folio->page, nr, u);  in folio_ref_add_unless()
294  * page and know that the page cannot have its refcount decreased to
298  * from interrupt context) and the page cache lookups (as pages are not
303  * pages being frozen & removed; eg the i_pages lock for the page cache
304  * or the mmap_lock or page table lock for page tables.  In this case,
316 static inline int page_ref_freeze(struct page *page, int count)  in page_ref_freeze()  argument
318 	int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);  in page_ref_freeze()
321 		__page_ref_freeze(page, count, ret);  in page_ref_freeze()
327 	return page_ref_freeze(&folio->page, count);  in folio_ref_freeze()
330 static inline void page_ref_unfreeze(struct page *page, int count)  in page_ref_unfreeze()  argument
332 	VM_BUG_ON_PAGE(page_count(page) != 0, page);  in page_ref_unfreeze()
335 	atomic_set_release(&page->_refcount, count);  in page_ref_unfreeze()
337 		__page_ref_unfreeze(page, count);  in page_ref_unfreeze()
342 	page_ref_unfreeze(&folio->page, count);  in folio_ref_unfreeze()