1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Interface for implementing AF_XDP zero-copy support in drivers.
3 * Copyright(c) 2020 Intel Corporation.
4 */
5
6 #ifndef _LINUX_XDP_SOCK_DRV_H
7 #define _LINUX_XDP_SOCK_DRV_H
8
9 #include <net/xdp_sock.h>
10 #include <net/xsk_buff_pool.h>
11
12 #define XDP_UMEM_MIN_CHUNK_SHIFT 11
13 #define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT)
14
15 struct xsk_cb_desc {
16 void *src;
17 u8 off;
18 u8 bytes;
19 };
20
21 #ifdef CONFIG_XDP_SOCKETS
22
23 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
24 bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
25 u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
26 void xsk_tx_release(struct xsk_buff_pool *pool);
27 struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
28 u16 queue_id);
29 void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool);
30 void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool);
31 void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
32 void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
33 bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
34
xsk_pool_get_headroom(struct xsk_buff_pool * pool)35 static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
36 {
37 return XDP_PACKET_HEADROOM + pool->headroom;
38 }
39
xsk_pool_get_chunk_size(struct xsk_buff_pool * pool)40 static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
41 {
42 return pool->chunk_size;
43 }
44
xsk_pool_get_rx_frame_size(struct xsk_buff_pool * pool)45 static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
46 {
47 return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool);
48 }
49
xsk_pool_set_rxq_info(struct xsk_buff_pool * pool,struct xdp_rxq_info * rxq)50 static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
51 struct xdp_rxq_info *rxq)
52 {
53 xp_set_rxq_info(pool, rxq);
54 }
55
xsk_pool_fill_cb(struct xsk_buff_pool * pool,struct xsk_cb_desc * desc)56 static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool,
57 struct xsk_cb_desc *desc)
58 {
59 xp_fill_cb(pool, desc);
60 }
61
xsk_pool_dma_unmap(struct xsk_buff_pool * pool,unsigned long attrs)62 static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
63 unsigned long attrs)
64 {
65 xp_dma_unmap(pool, attrs);
66 }
67
xsk_pool_dma_map(struct xsk_buff_pool * pool,struct device * dev,unsigned long attrs)68 static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
69 struct device *dev, unsigned long attrs)
70 {
71 struct xdp_umem *umem = pool->umem;
72
73 return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs);
74 }
75
xsk_buff_xdp_get_dma(struct xdp_buff * xdp)76 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
77 {
78 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
79
80 return xp_get_dma(xskb);
81 }
82
xsk_buff_xdp_get_frame_dma(struct xdp_buff * xdp)83 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
84 {
85 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
86
87 return xp_get_frame_dma(xskb);
88 }
89
xsk_buff_alloc(struct xsk_buff_pool * pool)90 static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
91 {
92 return xp_alloc(pool);
93 }
94
xsk_is_eop_desc(const struct xdp_desc * desc)95 static inline bool xsk_is_eop_desc(const struct xdp_desc *desc)
96 {
97 return !xp_mb_desc(desc);
98 }
99
100 /* Returns as many entries as possible up to max. 0 <= N <= max. */
xsk_buff_alloc_batch(struct xsk_buff_pool * pool,struct xdp_buff ** xdp,u32 max)101 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
102 {
103 return xp_alloc_batch(pool, xdp, max);
104 }
105
xsk_buff_can_alloc(struct xsk_buff_pool * pool,u32 count)106 static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
107 {
108 return xp_can_alloc(pool, count);
109 }
110
xsk_buff_free(struct xdp_buff * xdp)111 static inline void xsk_buff_free(struct xdp_buff *xdp)
112 {
113 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
114 struct list_head *xskb_list = &xskb->pool->xskb_list;
115 struct xdp_buff_xsk *pos, *tmp;
116
117 if (likely(!xdp_buff_has_frags(xdp)))
118 goto out;
119
120 list_for_each_entry_safe(pos, tmp, xskb_list, list_node) {
121 list_del(&pos->list_node);
122 xp_free(pos);
123 }
124
125 xdp_get_shared_info_from_buff(xdp)->nr_frags = 0;
126 out:
127 xp_free(xskb);
128 }
129
xsk_buff_add_frag(struct xdp_buff * head,struct xdp_buff * xdp)130 static inline bool xsk_buff_add_frag(struct xdp_buff *head,
131 struct xdp_buff *xdp)
132 {
133 const void *data = xdp->data;
134 struct xdp_buff_xsk *frag;
135
136 if (!__xdp_buff_add_frag(head, virt_to_netmem(data),
137 offset_in_page(data), xdp->data_end - data,
138 xdp->frame_sz, false))
139 return false;
140
141 frag = container_of(xdp, struct xdp_buff_xsk, xdp);
142 list_add_tail(&frag->list_node, &frag->pool->xskb_list);
143
144 return true;
145 }
146
xsk_buff_get_frag(const struct xdp_buff * first)147 static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
148 {
149 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
150 struct xdp_buff *ret = NULL;
151 struct xdp_buff_xsk *frag;
152
153 frag = list_first_entry_or_null(&xskb->pool->xskb_list,
154 struct xdp_buff_xsk, list_node);
155 if (frag) {
156 list_del(&frag->list_node);
157 ret = &frag->xdp;
158 }
159
160 return ret;
161 }
162
xsk_buff_del_tail(struct xdp_buff * tail)163 static inline void xsk_buff_del_tail(struct xdp_buff *tail)
164 {
165 struct xdp_buff_xsk *xskb = container_of(tail, struct xdp_buff_xsk, xdp);
166
167 list_del(&xskb->list_node);
168 }
169
xsk_buff_get_tail(struct xdp_buff * first)170 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
171 {
172 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
173 struct xdp_buff_xsk *frag;
174
175 frag = list_last_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk,
176 list_node);
177 return &frag->xdp;
178 }
179
xsk_buff_set_size(struct xdp_buff * xdp,u32 size)180 static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
181 {
182 xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
183 xdp->data_meta = xdp->data;
184 xdp->data_end = xdp->data + size;
185 xdp->flags = 0;
186 }
187
xsk_buff_raw_get_dma(struct xsk_buff_pool * pool,u64 addr)188 static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
189 u64 addr)
190 {
191 return xp_raw_get_dma(pool, addr);
192 }
193
xsk_buff_raw_get_data(struct xsk_buff_pool * pool,u64 addr)194 static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
195 {
196 return xp_raw_get_data(pool, addr);
197 }
198
199 /**
200 * xsk_buff_raw_get_ctx - get &xdp_desc context
201 * @pool: XSk buff pool desc address belongs to
202 * @addr: desc address (from userspace)
203 *
204 * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for
205 * details.
206 *
207 * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata
208 * pointer, if it is present and valid (initialized to %NULL otherwise).
209 */
210 static inline struct xdp_desc_ctx
xsk_buff_raw_get_ctx(const struct xsk_buff_pool * pool,u64 addr)211 xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
212 {
213 return xp_raw_get_ctx(pool, addr);
214 }
215
216 #define XDP_TXMD_FLAGS_VALID ( \
217 XDP_TXMD_FLAGS_TIMESTAMP | \
218 XDP_TXMD_FLAGS_CHECKSUM | \
219 XDP_TXMD_FLAGS_LAUNCH_TIME | \
220 0)
221
222 static inline bool
xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata * meta)223 xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta)
224 {
225 return !(meta->flags & ~XDP_TXMD_FLAGS_VALID);
226 }
227
228 static inline struct xsk_tx_metadata *
__xsk_buff_get_metadata(const struct xsk_buff_pool * pool,void * data)229 __xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data)
230 {
231 struct xsk_tx_metadata *meta;
232
233 if (!pool->tx_metadata_len)
234 return NULL;
235
236 meta = data - pool->tx_metadata_len;
237 if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
238 return NULL; /* no way to signal the error to the user */
239
240 return meta;
241 }
242
243 static inline struct xsk_tx_metadata *
xsk_buff_get_metadata(struct xsk_buff_pool * pool,u64 addr)244 xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
245 {
246 return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr));
247 }
248
xsk_buff_dma_sync_for_cpu(struct xdp_buff * xdp)249 static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
250 {
251 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
252
253 xp_dma_sync_for_cpu(xskb);
254 }
255
xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool * pool,dma_addr_t dma,size_t size)256 static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
257 dma_addr_t dma,
258 size_t size)
259 {
260 xp_dma_sync_for_device(pool, dma, size);
261 }
262
263 #else
264
xsk_tx_completed(struct xsk_buff_pool * pool,u32 nb_entries)265 static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
266 {
267 }
268
xsk_tx_peek_desc(struct xsk_buff_pool * pool,struct xdp_desc * desc)269 static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
270 struct xdp_desc *desc)
271 {
272 return false;
273 }
274
xsk_tx_peek_release_desc_batch(struct xsk_buff_pool * pool,u32 max)275 static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
276 {
277 return 0;
278 }
279
xsk_tx_release(struct xsk_buff_pool * pool)280 static inline void xsk_tx_release(struct xsk_buff_pool *pool)
281 {
282 }
283
284 static inline struct xsk_buff_pool *
xsk_get_pool_from_qid(struct net_device * dev,u16 queue_id)285 xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id)
286 {
287 return NULL;
288 }
289
xsk_set_rx_need_wakeup(struct xsk_buff_pool * pool)290 static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
291 {
292 }
293
xsk_set_tx_need_wakeup(struct xsk_buff_pool * pool)294 static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
295 {
296 }
297
xsk_clear_rx_need_wakeup(struct xsk_buff_pool * pool)298 static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
299 {
300 }
301
xsk_clear_tx_need_wakeup(struct xsk_buff_pool * pool)302 static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
303 {
304 }
305
xsk_uses_need_wakeup(struct xsk_buff_pool * pool)306 static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
307 {
308 return false;
309 }
310
xsk_pool_get_headroom(struct xsk_buff_pool * pool)311 static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
312 {
313 return 0;
314 }
315
xsk_pool_get_chunk_size(struct xsk_buff_pool * pool)316 static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
317 {
318 return 0;
319 }
320
xsk_pool_get_rx_frame_size(struct xsk_buff_pool * pool)321 static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
322 {
323 return 0;
324 }
325
xsk_pool_set_rxq_info(struct xsk_buff_pool * pool,struct xdp_rxq_info * rxq)326 static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
327 struct xdp_rxq_info *rxq)
328 {
329 }
330
xsk_pool_fill_cb(struct xsk_buff_pool * pool,struct xsk_cb_desc * desc)331 static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool,
332 struct xsk_cb_desc *desc)
333 {
334 }
335
xsk_pool_dma_unmap(struct xsk_buff_pool * pool,unsigned long attrs)336 static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
337 unsigned long attrs)
338 {
339 }
340
xsk_pool_dma_map(struct xsk_buff_pool * pool,struct device * dev,unsigned long attrs)341 static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
342 struct device *dev, unsigned long attrs)
343 {
344 return 0;
345 }
346
xsk_buff_xdp_get_dma(struct xdp_buff * xdp)347 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
348 {
349 return 0;
350 }
351
xsk_buff_xdp_get_frame_dma(struct xdp_buff * xdp)352 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
353 {
354 return 0;
355 }
356
xsk_buff_alloc(struct xsk_buff_pool * pool)357 static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
358 {
359 return NULL;
360 }
361
xsk_is_eop_desc(const struct xdp_desc * desc)362 static inline bool xsk_is_eop_desc(const struct xdp_desc *desc)
363 {
364 return false;
365 }
366
xsk_buff_alloc_batch(struct xsk_buff_pool * pool,struct xdp_buff ** xdp,u32 max)367 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
368 {
369 return 0;
370 }
371
xsk_buff_can_alloc(struct xsk_buff_pool * pool,u32 count)372 static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
373 {
374 return false;
375 }
376
xsk_buff_free(struct xdp_buff * xdp)377 static inline void xsk_buff_free(struct xdp_buff *xdp)
378 {
379 }
380
xsk_buff_add_frag(struct xdp_buff * head,struct xdp_buff * xdp)381 static inline bool xsk_buff_add_frag(struct xdp_buff *head,
382 struct xdp_buff *xdp)
383 {
384 return false;
385 }
386
xsk_buff_get_frag(const struct xdp_buff * first)387 static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
388 {
389 return NULL;
390 }
391
xsk_buff_del_tail(struct xdp_buff * tail)392 static inline void xsk_buff_del_tail(struct xdp_buff *tail)
393 {
394 }
395
xsk_buff_get_tail(struct xdp_buff * first)396 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
397 {
398 return NULL;
399 }
400
xsk_buff_set_size(struct xdp_buff * xdp,u32 size)401 static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
402 {
403 }
404
xsk_buff_raw_get_dma(struct xsk_buff_pool * pool,u64 addr)405 static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
406 u64 addr)
407 {
408 return 0;
409 }
410
xsk_buff_raw_get_data(struct xsk_buff_pool * pool,u64 addr)411 static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
412 {
413 return NULL;
414 }
415
416 static inline struct xdp_desc_ctx
xsk_buff_raw_get_ctx(const struct xsk_buff_pool * pool,u64 addr)417 xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
418 {
419 return (struct xdp_desc_ctx){ };
420 }
421
xsk_buff_valid_tx_metadata(struct xsk_tx_metadata * meta)422 static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta)
423 {
424 return false;
425 }
426
427 static inline struct xsk_tx_metadata *
__xsk_buff_get_metadata(const struct xsk_buff_pool * pool,void * data)428 __xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data)
429 {
430 return NULL;
431 }
432
433 static inline struct xsk_tx_metadata *
xsk_buff_get_metadata(struct xsk_buff_pool * pool,u64 addr)434 xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
435 {
436 return NULL;
437 }
438
xsk_buff_dma_sync_for_cpu(struct xdp_buff * xdp)439 static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
440 {
441 }
442
xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool * pool,dma_addr_t dma,size_t size)443 static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
444 dma_addr_t dma,
445 size_t size)
446 {
447 }
448
449 #endif /* CONFIG_XDP_SOCKETS */
450
451 #endif /* _LINUX_XDP_SOCK_DRV_H */
452