Lines Matching full:pool
46 #define alloc_stat_inc(pool, __stat) (pool->alloc_stats.__stat++) argument
48 #define recycle_stat_inc(pool, __stat) \ argument
50 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
54 #define recycle_stat_add(pool, __stat, val) \ argument
56 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
75 * page_pool_get_stats() - fetch page pool stats
76 * @pool: pool from which page was allocated
85 bool page_pool_get_stats(const struct page_pool *pool, in page_pool_get_stats() argument
94 stats->alloc_stats.fast += pool->alloc_stats.fast; in page_pool_get_stats()
95 stats->alloc_stats.slow += pool->alloc_stats.slow; in page_pool_get_stats()
96 stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order; in page_pool_get_stats()
97 stats->alloc_stats.empty += pool->alloc_stats.empty; in page_pool_get_stats()
98 stats->alloc_stats.refill += pool->alloc_stats.refill; in page_pool_get_stats()
99 stats->alloc_stats.waive += pool->alloc_stats.waive; in page_pool_get_stats()
103 per_cpu_ptr(pool->recycle_stats, cpu); in page_pool_get_stats()
156 #define alloc_stat_inc(pool, __stat) argument
157 #define recycle_stat_inc(pool, __stat) argument
158 #define recycle_stat_add(pool, __stat, val) argument
161 static bool page_pool_producer_lock(struct page_pool *pool) in page_pool_producer_lock() argument
162 __acquires(&pool->ring.producer_lock) in page_pool_producer_lock()
167 spin_lock(&pool->ring.producer_lock); in page_pool_producer_lock()
169 spin_lock_bh(&pool->ring.producer_lock); in page_pool_producer_lock()
174 static void page_pool_producer_unlock(struct page_pool *pool, in page_pool_producer_unlock() argument
176 __releases(&pool->ring.producer_lock) in page_pool_producer_unlock()
179 spin_unlock(&pool->ring.producer_lock); in page_pool_producer_unlock()
181 spin_unlock_bh(&pool->ring.producer_lock); in page_pool_producer_unlock()
193 static int page_pool_init(struct page_pool *pool, in page_pool_init() argument
203 memcpy(&pool->p, ¶ms->fast, sizeof(pool->p)); in page_pool_init()
204 memcpy(&pool->slow, ¶ms->slow, sizeof(pool->slow)); in page_pool_init()
206 pool->cpuid = cpuid; in page_pool_init()
207 pool->dma_sync_for_cpu = true; in page_pool_init()
210 if (pool->slow.flags & ~PP_FLAG_ALL) in page_pool_init()
213 if (pool->p.pool_size) in page_pool_init()
214 ring_qsize = pool->p.pool_size; in page_pool_init()
224 if (pool->slow.flags & PP_FLAG_DMA_MAP) { in page_pool_init()
225 if ((pool->p.dma_dir != DMA_FROM_DEVICE) && in page_pool_init()
226 (pool->p.dma_dir != DMA_BIDIRECTIONAL)) in page_pool_init()
229 pool->dma_map = true; in page_pool_init()
232 if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) { in page_pool_init()
236 if (!(pool->slow.flags & PP_FLAG_DMA_MAP)) in page_pool_init()
239 if (!pool->p.max_len) in page_pool_init()
242 pool->dma_sync = true; in page_pool_init()
244 /* pool->p.offset has to be set according to the address in page_pool_init()
249 pool->has_init_callback = !!pool->slow.init_callback; in page_pool_init()
252 if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) { in page_pool_init()
253 pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats); in page_pool_init()
254 if (!pool->recycle_stats) in page_pool_init()
257 /* For system page pool instance we use a singular stats object in page_pool_init()
259 * (also percpu) page pool instance. in page_pool_init()
261 pool->recycle_stats = &pp_system_recycle_stats; in page_pool_init()
262 pool->system = true; in page_pool_init()
266 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) { in page_pool_init()
268 if (!pool->system) in page_pool_init()
269 free_percpu(pool->recycle_stats); in page_pool_init()
274 atomic_set(&pool->pages_state_release_cnt, 0); in page_pool_init()
277 refcount_set(&pool->user_cnt, 1); in page_pool_init()
279 if (pool->dma_map) in page_pool_init()
280 get_device(pool->p.dev); in page_pool_init()
282 if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) { in page_pool_init()
283 netdev_assert_locked(pool->slow.netdev); in page_pool_init()
284 rxq = __netif_get_rx_queue(pool->slow.netdev, in page_pool_init()
285 pool->slow.queue_idx); in page_pool_init()
286 pool->mp_priv = rxq->mp_params.mp_priv; in page_pool_init()
287 pool->mp_ops = rxq->mp_params.mp_ops; in page_pool_init()
290 if (pool->mp_ops) { in page_pool_init()
291 if (!pool->dma_map || !pool->dma_sync) in page_pool_init()
294 if (WARN_ON(!is_kernel_rodata((unsigned long)pool->mp_ops))) { in page_pool_init()
299 err = pool->mp_ops->init(pool); in page_pool_init()
312 ptr_ring_cleanup(&pool->ring, NULL); in page_pool_init()
314 if (!pool->system) in page_pool_init()
315 free_percpu(pool->recycle_stats); in page_pool_init()
320 static void page_pool_uninit(struct page_pool *pool) in page_pool_uninit() argument
322 ptr_ring_cleanup(&pool->ring, NULL); in page_pool_uninit()
324 if (pool->dma_map) in page_pool_uninit()
325 put_device(pool->p.dev); in page_pool_uninit()
328 if (!pool->system) in page_pool_uninit()
329 free_percpu(pool->recycle_stats); in page_pool_uninit()
334 * page_pool_create_percpu() - create a page pool for a given cpu.
341 struct page_pool *pool; in page_pool_create_percpu() local
344 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid); in page_pool_create_percpu()
345 if (!pool) in page_pool_create_percpu()
348 err = page_pool_init(pool, params, cpuid); in page_pool_create_percpu()
352 err = page_pool_list(pool); in page_pool_create_percpu()
356 return pool; in page_pool_create_percpu()
359 page_pool_uninit(pool); in page_pool_create_percpu()
362 kfree(pool); in page_pool_create_percpu()
368 * page_pool_create() - create a page pool
377 static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem);
379 static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) in page_pool_refill_alloc_cache() argument
381 struct ptr_ring *r = &pool->ring; in page_pool_refill_alloc_cache()
387 alloc_stat_inc(pool, empty); in page_pool_refill_alloc_cache()
395 pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid; in page_pool_refill_alloc_cache()
397 /* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */ in page_pool_refill_alloc_cache()
408 pool->alloc.cache[pool->alloc.count++] = netmem; in page_pool_refill_alloc_cache()
415 page_pool_return_page(pool, netmem); in page_pool_refill_alloc_cache()
416 alloc_stat_inc(pool, waive); in page_pool_refill_alloc_cache()
420 } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); in page_pool_refill_alloc_cache()
423 if (likely(pool->alloc.count > 0)) { in page_pool_refill_alloc_cache()
424 netmem = pool->alloc.cache[--pool->alloc.count]; in page_pool_refill_alloc_cache()
425 alloc_stat_inc(pool, refill); in page_pool_refill_alloc_cache()
432 static netmem_ref __page_pool_get_cached(struct page_pool *pool) in __page_pool_get_cached() argument
437 if (likely(pool->alloc.count)) { in __page_pool_get_cached()
439 netmem = pool->alloc.cache[--pool->alloc.count]; in __page_pool_get_cached()
440 alloc_stat_inc(pool, fast); in __page_pool_get_cached()
442 netmem = page_pool_refill_alloc_cache(pool); in __page_pool_get_cached()
448 static void __page_pool_dma_sync_for_device(const struct page_pool *pool, in __page_pool_dma_sync_for_device() argument
455 dma_sync_size = min(dma_sync_size, pool->p.max_len); in __page_pool_dma_sync_for_device()
456 __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset, in __page_pool_dma_sync_for_device()
457 dma_sync_size, pool->p.dma_dir); in __page_pool_dma_sync_for_device()
462 page_pool_dma_sync_for_device(const struct page_pool *pool, in page_pool_dma_sync_for_device() argument
466 if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) in page_pool_dma_sync_for_device()
467 __page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); in page_pool_dma_sync_for_device()
470 static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) in page_pool_dma_map() argument
477 * This mapping is kept for lifetime of page, until leaving pool. in page_pool_dma_map()
479 dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0, in page_pool_dma_map()
480 (PAGE_SIZE << pool->p.order), pool->p.dma_dir, in page_pool_dma_map()
483 if (dma_mapping_error(pool->p.dev, dma)) in page_pool_dma_map()
489 page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len); in page_pool_dma_map()
495 dma_unmap_page_attrs(pool->p.dev, dma, in page_pool_dma_map()
496 PAGE_SIZE << pool->p.order, pool->p.dma_dir, in page_pool_dma_map()
501 static struct page *__page_pool_alloc_page_order(struct page_pool *pool, in __page_pool_alloc_page_order() argument
507 page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); in __page_pool_alloc_page_order()
511 if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) { in __page_pool_alloc_page_order()
516 alloc_stat_inc(pool, slow_high_order); in __page_pool_alloc_page_order()
517 page_pool_set_pp_info(pool, page_to_netmem(page)); in __page_pool_alloc_page_order()
520 pool->pages_state_hold_cnt++; in __page_pool_alloc_page_order()
521 trace_page_pool_state_hold(pool, page_to_netmem(page), in __page_pool_alloc_page_order()
522 pool->pages_state_hold_cnt); in __page_pool_alloc_page_order()
527 static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, in __page_pool_alloc_pages_slow() argument
531 unsigned int pp_order = pool->p.order; in __page_pool_alloc_pages_slow()
532 bool dma_map = pool->dma_map; in __page_pool_alloc_pages_slow()
538 return page_to_netmem(__page_pool_alloc_page_order(pool, gfp)); in __page_pool_alloc_pages_slow()
541 if (unlikely(pool->alloc.count > 0)) in __page_pool_alloc_pages_slow()
542 return pool->alloc.cache[--pool->alloc.count]; in __page_pool_alloc_pages_slow()
545 memset(&pool->alloc.cache, 0, sizeof(void *) * bulk); in __page_pool_alloc_pages_slow()
547 nr_pages = alloc_pages_bulk_node(gfp, pool->p.nid, bulk, in __page_pool_alloc_pages_slow()
548 (struct page **)pool->alloc.cache); in __page_pool_alloc_pages_slow()
556 netmem = pool->alloc.cache[i]; in __page_pool_alloc_pages_slow()
557 if (dma_map && unlikely(!page_pool_dma_map(pool, netmem))) { in __page_pool_alloc_pages_slow()
562 page_pool_set_pp_info(pool, netmem); in __page_pool_alloc_pages_slow()
563 pool->alloc.cache[pool->alloc.count++] = netmem; in __page_pool_alloc_pages_slow()
565 pool->pages_state_hold_cnt++; in __page_pool_alloc_pages_slow()
566 trace_page_pool_state_hold(pool, netmem, in __page_pool_alloc_pages_slow()
567 pool->pages_state_hold_cnt); in __page_pool_alloc_pages_slow()
571 if (likely(pool->alloc.count > 0)) { in __page_pool_alloc_pages_slow()
572 netmem = pool->alloc.cache[--pool->alloc.count]; in __page_pool_alloc_pages_slow()
573 alloc_stat_inc(pool, slow); in __page_pool_alloc_pages_slow()
585 netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_netmems() argument
590 netmem = __page_pool_get_cached(pool); in page_pool_alloc_netmems()
595 if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops) in page_pool_alloc_netmems()
596 netmem = pool->mp_ops->alloc_netmems(pool, gfp); in page_pool_alloc_netmems()
598 netmem = __page_pool_alloc_pages_slow(pool, gfp); in page_pool_alloc_netmems()
604 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_pages() argument
606 return netmem_to_page(page_pool_alloc_netmems(pool, gfp)); in page_pool_alloc_pages()
615 s32 page_pool_inflight(const struct page_pool *pool, bool strict) in page_pool_inflight() argument
617 u32 release_cnt = atomic_read(&pool->pages_state_release_cnt); in page_pool_inflight()
618 u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt); in page_pool_inflight()
624 trace_page_pool_release(pool, inflight, hold_cnt, release_cnt); in page_pool_inflight()
634 void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) in page_pool_set_pp_info() argument
636 netmem_set_pp(netmem, pool); in page_pool_set_pp_info()
646 if (pool->has_init_callback) in page_pool_set_pp_info()
647 pool->slow.init_callback(netmem, pool->slow.init_arg); in page_pool_set_pp_info()
656 static __always_inline void __page_pool_release_page_dma(struct page_pool *pool, in __page_pool_release_page_dma() argument
661 if (!pool->dma_map) in __page_pool_release_page_dma()
669 /* When page is unmapped, it cannot be returned to our pool */ in __page_pool_release_page_dma()
670 dma_unmap_page_attrs(pool->p.dev, dma, in __page_pool_release_page_dma()
671 PAGE_SIZE << pool->p.order, pool->p.dma_dir, in __page_pool_release_page_dma()
681 void page_pool_return_page(struct page_pool *pool, netmem_ref netmem) in page_pool_return_page() argument
687 if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops) in page_pool_return_page()
688 put = pool->mp_ops->release_netmem(pool, netmem); in page_pool_return_page()
690 __page_pool_release_page_dma(pool, netmem); in page_pool_return_page()
692 /* This may be the last page returned, releasing the pool, so in page_pool_return_page()
693 * it is not safe to reference pool afterwards. in page_pool_return_page()
695 count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); in page_pool_return_page()
696 trace_page_pool_state_release(pool, netmem, count); in page_pool_return_page()
702 /* An optimization would be to call __free_pages(page, pool->p.order) in page_pool_return_page()
708 static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem) in page_pool_recycle_in_ring() argument
713 ret = ptr_ring_produce(&pool->ring, (__force void *)netmem); in page_pool_recycle_in_ring()
715 ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem); in page_pool_recycle_in_ring()
718 recycle_stat_inc(pool, ring); in page_pool_recycle_in_ring()
731 struct page_pool *pool) in page_pool_recycle_in_cache() argument
733 if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) { in page_pool_recycle_in_cache()
734 recycle_stat_inc(pool, cache_full); in page_pool_recycle_in_cache()
739 pool->alloc.cache[pool->alloc.count++] = netmem; in page_pool_recycle_in_cache()
740 recycle_stat_inc(pool, cached); in page_pool_recycle_in_cache()
752 * If pool->dma_sync is set, we'll try to sync the DMA area for
753 * the configured size min(dma_sync_size, pool->max_len).
758 __page_pool_put_page(struct page_pool *pool, netmem_ref netmem, in __page_pool_put_page() argument
775 page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); in __page_pool_put_page()
777 if (allow_direct && page_pool_recycle_in_cache(netmem, pool)) in __page_pool_put_page()
797 recycle_stat_inc(pool, released_refcnt); in __page_pool_put_page()
798 page_pool_return_page(pool, netmem); in __page_pool_put_page()
803 static bool page_pool_napi_local(const struct page_pool *pool) in page_pool_napi_local() argument
818 if (READ_ONCE(pool->cpuid) == cpuid) in page_pool_napi_local()
821 napi = READ_ONCE(pool->p.napi); in page_pool_napi_local()
826 void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, in page_pool_put_unrefed_netmem() argument
830 allow_direct = page_pool_napi_local(pool); in page_pool_put_unrefed_netmem()
833 __page_pool_put_page(pool, netmem, dma_sync_size, allow_direct); in page_pool_put_unrefed_netmem()
834 if (netmem && !page_pool_recycle_in_ring(pool, netmem)) { in page_pool_put_unrefed_netmem()
836 recycle_stat_inc(pool, ring_full); in page_pool_put_unrefed_netmem()
837 page_pool_return_page(pool, netmem); in page_pool_put_unrefed_netmem()
842 void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, in page_pool_put_unrefed_page() argument
845 page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size, in page_pool_put_unrefed_page()
850 static void page_pool_recycle_ring_bulk(struct page_pool *pool, in page_pool_recycle_ring_bulk() argument
858 in_softirq = page_pool_producer_lock(pool); in page_pool_recycle_ring_bulk()
861 if (__ptr_ring_produce(&pool->ring, (__force void *)bulk[i])) { in page_pool_recycle_ring_bulk()
863 recycle_stat_inc(pool, ring_full); in page_pool_recycle_ring_bulk()
868 page_pool_producer_unlock(pool, in_softirq); in page_pool_recycle_ring_bulk()
869 recycle_stat_add(pool, ring, i); in page_pool_recycle_ring_bulk()
880 page_pool_return_page(pool, bulk[i]); in page_pool_recycle_ring_bulk()
911 struct page_pool *pool = NULL; in page_pool_put_netmem_bulk() local
922 if (unlikely(!pool)) { in page_pool_put_netmem_bulk()
923 pool = netmem_pp; in page_pool_put_netmem_bulk()
924 allow_direct = page_pool_napi_local(pool); in page_pool_put_netmem_bulk()
925 } else if (netmem_pp != pool) { in page_pool_put_netmem_bulk()
934 netmem = __page_pool_put_page(pool, netmem, -1, in page_pool_put_netmem_bulk()
942 page_pool_recycle_ring_bulk(pool, bulk, bulk_len); in page_pool_put_netmem_bulk()
949 static netmem_ref page_pool_drain_frag(struct page_pool *pool, in page_pool_drain_frag() argument
952 long drain_count = BIAS_MAX - pool->frag_users; in page_pool_drain_frag()
959 page_pool_dma_sync_for_device(pool, netmem, -1); in page_pool_drain_frag()
963 page_pool_return_page(pool, netmem); in page_pool_drain_frag()
967 static void page_pool_free_frag(struct page_pool *pool) in page_pool_free_frag() argument
969 long drain_count = BIAS_MAX - pool->frag_users; in page_pool_free_frag()
970 netmem_ref netmem = pool->frag_page; in page_pool_free_frag()
972 pool->frag_page = 0; in page_pool_free_frag()
977 page_pool_return_page(pool, netmem); in page_pool_free_frag()
980 netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, in page_pool_alloc_frag_netmem() argument
984 unsigned int max_size = PAGE_SIZE << pool->p.order; in page_pool_alloc_frag_netmem()
985 netmem_ref netmem = pool->frag_page; in page_pool_alloc_frag_netmem()
991 *offset = pool->frag_offset; in page_pool_alloc_frag_netmem()
994 netmem = page_pool_drain_frag(pool, netmem); in page_pool_alloc_frag_netmem()
996 recycle_stat_inc(pool, cached); in page_pool_alloc_frag_netmem()
997 alloc_stat_inc(pool, fast); in page_pool_alloc_frag_netmem()
1003 netmem = page_pool_alloc_netmems(pool, gfp); in page_pool_alloc_frag_netmem()
1005 pool->frag_page = 0; in page_pool_alloc_frag_netmem()
1009 pool->frag_page = netmem; in page_pool_alloc_frag_netmem()
1012 pool->frag_users = 1; in page_pool_alloc_frag_netmem()
1014 pool->frag_offset = size; in page_pool_alloc_frag_netmem()
1019 pool->frag_users++; in page_pool_alloc_frag_netmem()
1020 pool->frag_offset = *offset + size; in page_pool_alloc_frag_netmem()
1025 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, in page_pool_alloc_frag() argument
1028 return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size, in page_pool_alloc_frag()
1033 static void page_pool_empty_ring(struct page_pool *pool) in page_pool_empty_ring() argument
1038 while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) { in page_pool_empty_ring()
1044 page_pool_return_page(pool, netmem); in page_pool_empty_ring()
1048 static void __page_pool_destroy(struct page_pool *pool) in __page_pool_destroy() argument
1050 if (pool->disconnect) in __page_pool_destroy()
1051 pool->disconnect(pool); in __page_pool_destroy()
1053 page_pool_unlist(pool); in __page_pool_destroy()
1054 page_pool_uninit(pool); in __page_pool_destroy()
1056 if (pool->mp_ops) { in __page_pool_destroy()
1057 pool->mp_ops->destroy(pool); in __page_pool_destroy()
1061 kfree(pool); in __page_pool_destroy()
1064 static void page_pool_empty_alloc_cache_once(struct page_pool *pool) in page_pool_empty_alloc_cache_once() argument
1068 if (pool->destroy_cnt) in page_pool_empty_alloc_cache_once()
1075 while (pool->alloc.count) { in page_pool_empty_alloc_cache_once()
1076 netmem = pool->alloc.cache[--pool->alloc.count]; in page_pool_empty_alloc_cache_once()
1077 page_pool_return_page(pool, netmem); in page_pool_empty_alloc_cache_once()
1081 static void page_pool_scrub(struct page_pool *pool) in page_pool_scrub() argument
1083 page_pool_empty_alloc_cache_once(pool); in page_pool_scrub()
1084 pool->destroy_cnt++; in page_pool_scrub()
1089 page_pool_empty_ring(pool); in page_pool_scrub()
1092 static int page_pool_release(struct page_pool *pool) in page_pool_release() argument
1096 page_pool_scrub(pool); in page_pool_release()
1097 inflight = page_pool_inflight(pool, true); in page_pool_release()
1099 __page_pool_destroy(pool); in page_pool_release()
1107 struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw); in page_pool_release_retry() local
1111 inflight = page_pool_release(pool); in page_pool_release_retry()
1122 netdev = READ_ONCE(pool->slow.netdev); in page_pool_release_retry()
1123 if (time_after_eq(jiffies, pool->defer_warn) && in page_pool_release_retry()
1125 int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ; in page_pool_release_retry()
1127 pr_warn("%s() stalled pool shutdown: id %u, %d inflight %d sec\n", in page_pool_release_retry()
1128 __func__, pool->user.id, inflight, sec); in page_pool_release_retry()
1129 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_release_retry()
1133 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_release_retry()
1136 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), in page_pool_use_xdp_mem() argument
1139 refcount_inc(&pool->user_cnt); in page_pool_use_xdp_mem()
1140 pool->disconnect = disconnect; in page_pool_use_xdp_mem()
1141 pool->xdp_mem_id = mem->id; in page_pool_use_xdp_mem()
1144 void page_pool_disable_direct_recycling(struct page_pool *pool) in page_pool_disable_direct_recycling() argument
1146 /* Disable direct recycling based on pool->cpuid. in page_pool_disable_direct_recycling()
1149 WRITE_ONCE(pool->cpuid, -1); in page_pool_disable_direct_recycling()
1151 if (!pool->p.napi) in page_pool_disable_direct_recycling()
1154 napi_assert_will_not_race(pool->p.napi); in page_pool_disable_direct_recycling()
1157 WRITE_ONCE(pool->p.napi, NULL); in page_pool_disable_direct_recycling()
1162 void page_pool_destroy(struct page_pool *pool) in page_pool_destroy() argument
1164 if (!pool) in page_pool_destroy()
1167 if (!page_pool_put(pool)) in page_pool_destroy()
1170 page_pool_disable_direct_recycling(pool); in page_pool_destroy()
1171 page_pool_free_frag(pool); in page_pool_destroy()
1173 if (!page_pool_release(pool)) in page_pool_destroy()
1176 page_pool_detached(pool); in page_pool_destroy()
1177 pool->defer_start = jiffies; in page_pool_destroy()
1178 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_destroy()
1180 INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry); in page_pool_destroy()
1181 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_destroy()
1186 void page_pool_update_nid(struct page_pool *pool, int new_nid) in page_pool_update_nid() argument
1190 trace_page_pool_update_nid(pool, new_nid); in page_pool_update_nid()
1191 pool->p.nid = new_nid; in page_pool_update_nid()
1193 /* Flush pool alloc cache, as refill will check NUMA node */ in page_pool_update_nid()
1194 while (pool->alloc.count) { in page_pool_update_nid()
1195 netmem = pool->alloc.cache[--pool->alloc.count]; in page_pool_update_nid()
1196 page_pool_return_page(pool, netmem); in page_pool_update_nid()
1206 /* Associate a niov with a page pool. Should follow with a matching
1209 void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov) in net_mp_niov_set_page_pool() argument
1213 page_pool_set_pp_info(pool, netmem); in net_mp_niov_set_page_pool()
1215 pool->pages_state_hold_cnt++; in net_mp_niov_set_page_pool()
1216 trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt); in net_mp_niov_set_page_pool()
1219 /* Disassociate a niov from a page pool. Should only be used in the