Lines Matching +full:dma +full:- +full:safe +full:- +full:map
1 /* SPDX-License-Identifier: GPL-2.0
14 #include <linux/dma-direction.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/page-flags.h>
29 memcpy(&pool->p, params, sizeof(pool->p)); in page_pool_init()
32 if (pool->p.flags & ~(PP_FLAG_ALL)) in page_pool_init()
33 return -EINVAL; in page_pool_init()
35 if (pool->p.pool_size) in page_pool_init()
36 ring_qsize = pool->p.pool_size; in page_pool_init()
40 return -E2BIG; in page_pool_init()
42 /* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL. in page_pool_init()
43 * DMA_BIDIRECTIONAL is for allowing page used for DMA sending, in page_pool_init()
44 * which is the XDP_TX use-case. in page_pool_init()
46 if (pool->p.flags & PP_FLAG_DMA_MAP) { in page_pool_init()
47 if ((pool->p.dma_dir != DMA_FROM_DEVICE) && in page_pool_init()
48 (pool->p.dma_dir != DMA_BIDIRECTIONAL)) in page_pool_init()
49 return -EINVAL; in page_pool_init()
52 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) { in page_pool_init()
53 /* In order to request DMA-sync-for-device the page in page_pool_init()
56 if (!(pool->p.flags & PP_FLAG_DMA_MAP)) in page_pool_init()
57 return -EINVAL; in page_pool_init()
59 if (!pool->p.max_len) in page_pool_init()
60 return -EINVAL; in page_pool_init()
62 /* pool->p.offset has to be set according to the address in page_pool_init()
63 * offset used by the DMA engine to start copying rx data in page_pool_init()
67 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) in page_pool_init()
68 return -ENOMEM; in page_pool_init()
70 atomic_set(&pool->pages_state_release_cnt, 0); in page_pool_init()
73 refcount_set(&pool->user_cnt, 1); in page_pool_init()
75 if (pool->p.flags & PP_FLAG_DMA_MAP) in page_pool_init()
76 get_device(pool->p.dev); in page_pool_init()
86 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid); in page_pool_create()
88 return ERR_PTR(-ENOMEM); in page_pool_create()
106 struct ptr_ring *r = &pool->ring; in page_pool_refill_alloc_cache()
115 * assumes CPU refilling driver RX-ring will also run RX-NAPI. in page_pool_refill_alloc_cache()
118 pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid; in page_pool_refill_alloc_cache()
120 /* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */ in page_pool_refill_alloc_cache()
124 /* Slower-path: Get pages from locked ring queue */ in page_pool_refill_alloc_cache()
125 spin_lock(&r->consumer_lock); in page_pool_refill_alloc_cache()
134 pool->alloc.cache[pool->alloc.count++] = page; in page_pool_refill_alloc_cache()
137 * (1) release 1 page to page-allocator and in page_pool_refill_alloc_cache()
145 } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); in page_pool_refill_alloc_cache()
148 if (likely(pool->alloc.count > 0)) in page_pool_refill_alloc_cache()
149 page = pool->alloc.cache[--pool->alloc.count]; in page_pool_refill_alloc_cache()
151 spin_unlock(&r->consumer_lock); in page_pool_refill_alloc_cache()
160 /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */ in __page_pool_get_cached()
161 if (likely(pool->alloc.count)) { in __page_pool_get_cached()
162 /* Fast-path */ in __page_pool_get_cached()
163 page = pool->alloc.cache[--pool->alloc.count]; in __page_pool_get_cached()
175 dma_sync_size = min(dma_sync_size, pool->p.max_len); in page_pool_dma_sync_for_device()
176 dma_sync_single_range_for_device(pool->p.dev, page->dma_addr, in page_pool_dma_sync_for_device()
177 pool->p.offset, dma_sync_size, in page_pool_dma_sync_for_device()
178 pool->p.dma_dir); in page_pool_dma_sync_for_device()
188 dma_addr_t dma; in __page_pool_alloc_pages_slow() local
191 * prep_new_page() can handle order-0 with __GFP_COMP. in __page_pool_alloc_pages_slow()
193 if (pool->p.order) in __page_pool_alloc_pages_slow()
198 * Current slow-path essentially falls back to single page in __page_pool_alloc_pages_slow()
205 page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); in __page_pool_alloc_pages_slow()
207 page = alloc_pages(gfp, pool->p.order); in __page_pool_alloc_pages_slow()
212 if (!(pool->p.flags & PP_FLAG_DMA_MAP)) in __page_pool_alloc_pages_slow()
215 /* Setup DMA mapping: use 'struct page' area for storing DMA-addr in __page_pool_alloc_pages_slow()
217 * into page private data (i.e 32bit cpu with 64bit DMA caps) in __page_pool_alloc_pages_slow()
220 dma = dma_map_page_attrs(pool->p.dev, page, 0, in __page_pool_alloc_pages_slow()
221 (PAGE_SIZE << pool->p.order), in __page_pool_alloc_pages_slow()
222 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC); in __page_pool_alloc_pages_slow()
223 if (dma_mapping_error(pool->p.dev, dma)) { in __page_pool_alloc_pages_slow()
227 page->dma_addr = dma; in __page_pool_alloc_pages_slow()
229 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) in __page_pool_alloc_pages_slow()
230 page_pool_dma_sync_for_device(pool, page, pool->p.max_len); in __page_pool_alloc_pages_slow()
233 /* Track how many pages are held 'in-flight' */ in __page_pool_alloc_pages_slow()
234 pool->pages_state_hold_cnt++; in __page_pool_alloc_pages_slow()
236 trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt); in __page_pool_alloc_pages_slow()
249 /* Fast-path: Get a page from cache */ in page_pool_alloc_pages()
254 /* Slow-path: cache empty, do real allocation */ in page_pool_alloc_pages()
263 #define _distance(a, b) (s32)((a) - (b))
267 u32 release_cnt = atomic_read(&pool->pages_state_release_cnt); in page_pool_inflight()
268 u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt); in page_pool_inflight()
274 WARN(inflight < 0, "Negative(%d) inflight packet-pages", inflight); in page_pool_inflight()
282 * page-allocator via put_page).
286 dma_addr_t dma; in page_pool_release_page() local
289 if (!(pool->p.flags & PP_FLAG_DMA_MAP)) in page_pool_release_page()
291 * map them in page_pool_release_page()
295 dma = page->dma_addr; in page_pool_release_page()
298 dma_unmap_page_attrs(pool->p.dev, dma, in page_pool_release_page()
299 PAGE_SIZE << pool->p.order, pool->p.dma_dir, in page_pool_release_page()
301 page->dma_addr = 0; in page_pool_release_page()
304 * it is not safe to reference pool afterwards. in page_pool_release_page()
306 count = atomic_inc_return(&pool->pages_state_release_cnt); in page_pool_release_page()
317 /* An optimization would be to call __free_pages(page, pool->p.order) in page_pool_return_page()
318 * knowing page is not part of page-cache (thus avoiding a in page_pool_return_page()
328 ret = ptr_ring_produce(&pool->ring, page); in page_pool_recycle_in_ring()
330 ret = ptr_ring_produce_bh(&pool->ring, page); in page_pool_recycle_in_ring()
336 * alloc side cache. E.g. during RX-NAPI processing for XDP_DROP use-case.
338 * Caller must provide appropriate safe context.
343 if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) in page_pool_recycle_in_cache()
347 pool->alloc.cache[pool->alloc.count++] = page; in page_pool_recycle_in_cache()
360 * if PP_FLAG_DMA_SYNC_DEV is set, we'll try to sync the DMA area for
361 * the configured size min(dma_sync_size, pool->max_len).
369 * one-frame-per-page, but have fallbacks that act like the in page_pool_put_page()
378 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) in page_pool_put_page()
392 /* Fallback/non-XDP mode: API user have elevated refcnt. in page_pool_put_page()
397 * switching between XDP/non-XDP. in page_pool_put_page()
399 * In-case page_pool maintains the DMA mapping, API user must in page_pool_put_page()
401 * case, the DMA is unmapped/released, as driver is likely in page_pool_put_page()
416 while ((page = ptr_ring_consume_bh(&pool->ring))) { in page_pool_empty_ring()
428 if (pool->disconnect) in page_pool_free()
429 pool->disconnect(pool); in page_pool_free()
431 ptr_ring_cleanup(&pool->ring, NULL); in page_pool_free()
433 if (pool->p.flags & PP_FLAG_DMA_MAP) in page_pool_free()
434 put_device(pool->p.dev); in page_pool_free()
443 if (pool->destroy_cnt) in page_pool_empty_alloc_cache_once()
447 * no-longer in use, and page_pool_alloc_pages() cannot be in page_pool_empty_alloc_cache_once()
450 while (pool->alloc.count) { in page_pool_empty_alloc_cache_once()
451 page = pool->alloc.cache[--pool->alloc.count]; in page_pool_empty_alloc_cache_once()
459 pool->destroy_cnt++; in page_pool_scrub()
462 * be in-flight. in page_pool_scrub()
490 if (time_after_eq(jiffies, pool->defer_warn)) { in page_pool_release_retry()
491 int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ; in page_pool_release_retry()
495 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_release_retry()
499 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_release_retry()
504 refcount_inc(&pool->user_cnt); in page_pool_use_xdp_mem()
505 pool->disconnect = disconnect; in page_pool_use_xdp_mem()
519 pool->defer_start = jiffies; in page_pool_destroy()
520 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_destroy()
522 INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry); in page_pool_destroy()
523 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_destroy()
527 /* Caller must provide appropriate safe context, e.g. NAPI. */
533 pool->p.nid = new_nid; in page_pool_update_nid()
536 while (pool->alloc.count) { in page_pool_update_nid()
537 page = pool->alloc.cache[--pool->alloc.count]; in page_pool_update_nid()