374 static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem);
376 static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)  in page_pool_refill_alloc_cache()
379 	netmem_ref netmem;  in page_pool_refill_alloc_cache()
400 		netmem = (__force netmem_ref)__ptr_ring_consume(r);  in page_pool_refill_alloc_cache()
429 static netmem_ref __page_pool_get_cached(struct page_pool *pool)  in __page_pool_get_cached()
431 	netmem_ref netmem;  in __page_pool_get_cached()
446 					    netmem_ref netmem,  in __page_pool_dma_sync_for_device()
460 			      netmem_ref netmem,  in page_pool_dma_sync_for_device()
473 static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp)  in page_pool_dma_map()
547 static noinline netmem_ref __page_pool_alloc_netmems_slow(struct page_pool *pool,  in __page_pool_alloc_netmems_slow()
553 	netmem_ref netmem;  in __page_pool_alloc_netmems_slow()
605 netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp)  in page_pool_alloc_netmems()
607 	netmem_ref netmem;  in page_pool_alloc_netmems()
654 void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)  in page_pool_set_pp_info()
670 void page_pool_clear_pp_info(netmem_ref netmem)  in page_pool_clear_pp_info()
677 							   netmem_ref netmem)  in __page_pool_release_netmem_dma()
715 static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem)  in page_pool_return_netmem()
742 static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)  in page_pool_recycle_in_ring()
761 static bool page_pool_recycle_in_cache(netmem_ref netmem,  in page_pool_recycle_in_cache()
775 static bool __page_pool_page_can_be_recycled(netmem_ref netmem)  in __page_pool_page_can_be_recycled()
788 static __always_inline netmem_ref
789 __page_pool_put_page(struct page_pool *pool, netmem_ref netmem,  in __page_pool_put_page()
861 void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,  in page_pool_put_unrefed_netmem()
886 					netmem_ref *bulk,  in page_pool_recycle_ring_bulk()
932 void page_pool_put_netmem_bulk(netmem_ref *data, u32 count)  in page_pool_put_netmem_bulk()
937 		netmem_ref netmem = netmem_compound_head(data[i]);  in page_pool_put_netmem_bulk()
945 		netmem_ref bulk[XDP_BULK_QUEUE_SIZE];  in page_pool_put_netmem_bulk()
954 			netmem_ref netmem = data[i];  in page_pool_put_netmem_bulk()
984 static netmem_ref page_pool_drain_frag(struct page_pool *pool,  in page_pool_drain_frag()
985 				       netmem_ref netmem)  in page_pool_drain_frag()
1005 	netmem_ref netmem = pool->frag_page;  in page_pool_free_frag()
1015 netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,  in page_pool_alloc_frag_netmem()
1020 	netmem_ref netmem = pool->frag_page;  in page_pool_alloc_frag_netmem()
1070 	netmem_ref netmem;  in page_pool_empty_ring()
1073 	while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) {  in page_pool_empty_ring()
1101 	netmem_ref netmem;  in page_pool_empty_alloc_cache_once()
1277 	netmem_ref netmem;  in page_pool_update_nid()
1300 	netmem_ref netmem = net_iov_to_netmem(niov);  in net_mp_niov_set_page_pool()
1313 	netmem_ref netmem = net_iov_to_netmem(niov);  in net_mp_niov_clear_page_pool()