1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef _NET_PAGE_POOL_TYPES_H
4 #define _NET_PAGE_POOL_TYPES_H
5
6 #include <linux/dma-direction.h>
7 #include <linux/ptr_ring.h>
8 #include <linux/types.h>
9
10 #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA
11 * map/unmap
12 */
13 #define PP_FLAG_DMA_SYNC_DEV BIT(1) /* If set all pages that the driver gets
14 * from page_pool will be
15 * DMA-synced-for-device according to
16 * the length provided by the device
17 * driver.
18 * Please note DMA-sync-for-CPU is still
19 * device driver responsibility
20 */
21 #define PP_FLAG_ALL (PP_FLAG_DMA_MAP |\
22 PP_FLAG_DMA_SYNC_DEV)
23
24 /*
25 * Fast allocation side cache array/stack
26 *
27 * The cache size and refill watermark is related to the network
28 * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX
29 * ring is usually refilled and the max consumed elements will be 64,
30 * thus a natural max size of objects needed in the cache.
31 *
32 * Keeping room for more objects, is due to XDP_DROP use-case. As
33 * XDP_DROP allows the opportunity to recycle objects directly into
34 * this array, as it shares the same softirq/NAPI protection. If
35 * cache is already full (or partly full) then the XDP_DROP recycles
36 * would have to take a slower code path.
37 */
38 #define PP_ALLOC_CACHE_SIZE 128
39 #define PP_ALLOC_CACHE_REFILL 64
40 struct pp_alloc_cache {
41 u32 count;
42 struct page *cache[PP_ALLOC_CACHE_SIZE];
43 };
44
45 /**
46 * struct page_pool_params - page pool parameters
47 * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV
48 * @order: 2^order pages on allocation
49 * @pool_size: size of the ptr_ring
50 * @nid: NUMA node id to allocate from pages from
51 * @dev: device, for DMA pre-mapping purposes
52 * @netdev: netdev this pool will serve (leave as NULL if none or multiple)
53 * @napi: NAPI which is the sole consumer of pages, otherwise NULL
54 * @dma_dir: DMA mapping direction
55 * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV
56 * @offset: DMA sync address offset for PP_FLAG_DMA_SYNC_DEV
57 */
58 struct page_pool_params {
59 struct_group_tagged(page_pool_params_fast, fast,
60 unsigned int flags;
61 unsigned int order;
62 unsigned int pool_size;
63 int nid;
64 struct device *dev;
65 struct napi_struct *napi;
66 enum dma_data_direction dma_dir;
67 unsigned int max_len;
68 unsigned int offset;
69 );
70 struct_group_tagged(page_pool_params_slow, slow,
71 struct net_device *netdev;
72 /* private: used by test code only */
73 void (*init_callback)(struct page *page, void *arg);
74 void *init_arg;
75 );
76 };
77
78 #ifdef CONFIG_PAGE_POOL_STATS
79 /**
80 * struct page_pool_alloc_stats - allocation statistics
81 * @fast: successful fast path allocations
82 * @slow: slow path order-0 allocations
83 * @slow_high_order: slow path high order allocations
84 * @empty: ptr ring is empty, so a slow path allocation was forced
85 * @refill: an allocation which triggered a refill of the cache
86 * @waive: pages obtained from the ptr ring that cannot be added to
87 * the cache due to a NUMA mismatch
88 */
89 struct page_pool_alloc_stats {
90 u64 fast;
91 u64 slow;
92 u64 slow_high_order;
93 u64 empty;
94 u64 refill;
95 u64 waive;
96 };
97
98 /**
99 * struct page_pool_recycle_stats - recycling (freeing) statistics
100 * @cached: recycling placed page in the page pool cache
101 * @cache_full: page pool cache was full
102 * @ring: page placed into the ptr ring
103 * @ring_full: page released from page pool because the ptr ring was full
104 * @released_refcnt: page released (and not recycled) because refcnt > 1
105 */
106 struct page_pool_recycle_stats {
107 u64 cached;
108 u64 cache_full;
109 u64 ring;
110 u64 ring_full;
111 u64 released_refcnt;
112 };
113
114 /**
115 * struct page_pool_stats - combined page pool use statistics
116 * @alloc_stats: see struct page_pool_alloc_stats
117 * @recycle_stats: see struct page_pool_recycle_stats
118 *
119 * Wrapper struct for combining page pool stats with different storage
120 * requirements.
121 */
122 struct page_pool_stats {
123 struct page_pool_alloc_stats alloc_stats;
124 struct page_pool_recycle_stats recycle_stats;
125 };
126 #endif
127
128 struct page_pool {
129 struct page_pool_params_fast p;
130
131 bool has_init_callback;
132
133 long frag_users;
134 struct page *frag_page;
135 unsigned int frag_offset;
136 u32 pages_state_hold_cnt;
137
138 struct delayed_work release_dw;
139 void (*disconnect)(void *pool);
140 unsigned long defer_start;
141 unsigned long defer_warn;
142
143 #ifdef CONFIG_PAGE_POOL_STATS
144 /* these stats are incremented while in softirq context */
145 struct page_pool_alloc_stats alloc_stats;
146 #endif
147 u32 xdp_mem_id;
148
149 /*
150 * Data structure for allocation side
151 *
152 * Drivers allocation side usually already perform some kind
153 * of resource protection. Piggyback on this protection, and
154 * require driver to protect allocation side.
155 *
156 * For NIC drivers this means, allocate a page_pool per
157 * RX-queue. As the RX-queue is already protected by
158 * Softirq/BH scheduling and napi_schedule. NAPI schedule
159 * guarantee that a single napi_struct will only be scheduled
160 * on a single CPU (see napi_schedule).
161 */
162 struct pp_alloc_cache alloc ____cacheline_aligned_in_smp;
163
164 /* Data structure for storing recycled pages.
165 *
166 * Returning/freeing pages is more complicated synchronization
167 * wise, because free's can happen on remote CPUs, with no
168 * association with allocation resource.
169 *
170 * Use ptr_ring, as it separates consumer and producer
171 * efficiently, it a way that doesn't bounce cache-lines.
172 *
173 * TODO: Implement bulk return pages into this structure.
174 */
175 struct ptr_ring ring;
176
177 #ifdef CONFIG_PAGE_POOL_STATS
178 /* recycle stats are per-cpu to avoid locking */
179 struct page_pool_recycle_stats __percpu *recycle_stats;
180 #endif
181 atomic_t pages_state_release_cnt;
182
183 /* A page_pool is strictly tied to a single RX-queue being
184 * protected by NAPI, due to above pp_alloc_cache. This
185 * refcnt serves purpose is to simplify drivers error handling.
186 */
187 refcount_t user_cnt;
188
189 u64 destroy_cnt;
190
191 /* Slow/Control-path information follows */
192 struct page_pool_params_slow slow;
193 /* User-facing fields, protected by page_pools_lock */
194 struct {
195 struct hlist_node list;
196 u64 detach_time;
197 u32 napi_id;
198 u32 id;
199 } user;
200 };
201
202 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
203 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
204 unsigned int size, gfp_t gfp);
205 struct page_pool *page_pool_create(const struct page_pool_params *params);
206
207 struct xdp_mem_info;
208
209 #ifdef CONFIG_PAGE_POOL
210 void page_pool_unlink_napi(struct page_pool *pool);
211 void page_pool_destroy(struct page_pool *pool);
212 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
213 struct xdp_mem_info *mem);
214 void page_pool_put_page_bulk(struct page_pool *pool, void **data,
215 int count);
216 #else
page_pool_unlink_napi(struct page_pool * pool)217 static inline void page_pool_unlink_napi(struct page_pool *pool)
218 {
219 }
220
page_pool_destroy(struct page_pool * pool)221 static inline void page_pool_destroy(struct page_pool *pool)
222 {
223 }
224
page_pool_use_xdp_mem(struct page_pool * pool,void (* disconnect)(void *),struct xdp_mem_info * mem)225 static inline void page_pool_use_xdp_mem(struct page_pool *pool,
226 void (*disconnect)(void *),
227 struct xdp_mem_info *mem)
228 {
229 }
230
page_pool_put_page_bulk(struct page_pool * pool,void ** data,int count)231 static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
232 int count)
233 {
234 }
235 #endif
236
237 void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
238 unsigned int dma_sync_size,
239 bool allow_direct);
240
is_page_pool_compiled_in(void)241 static inline bool is_page_pool_compiled_in(void)
242 {
243 #ifdef CONFIG_PAGE_POOL
244 return true;
245 #else
246 return false;
247 #endif
248 }
249
250 /* Caller must provide appropriate safe context, e.g. NAPI. */
251 void page_pool_update_nid(struct page_pool *pool, int new_nid);
252
253 #endif /* _NET_PAGE_POOL_H */
254