1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Interface for implementing AF_XDP zero-copy support in drivers.
3  * Copyright(c) 2020 Intel Corporation.
4  */
5 
6 #ifndef _LINUX_XDP_SOCK_DRV_H
7 #define _LINUX_XDP_SOCK_DRV_H
8 
9 #include <net/xdp_sock.h>
10 #include <net/xsk_buff_pool.h>
11 
12 #define XDP_UMEM_MIN_CHUNK_SHIFT 11
13 #define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT)
14 
15 struct xsk_cb_desc {
16 	void *src;
17 	u8 off;
18 	u8 bytes;
19 };
20 
21 #ifdef CONFIG_XDP_SOCKETS
22 
23 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
24 bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
25 u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
26 void xsk_tx_release(struct xsk_buff_pool *pool);
27 struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
28 					    u16 queue_id);
29 void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool);
30 void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool);
31 void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
32 void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
33 bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
34 
xsk_pool_get_headroom(struct xsk_buff_pool * pool)35 static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
36 {
37 	return XDP_PACKET_HEADROOM + pool->headroom;
38 }
39 
xsk_pool_get_chunk_size(struct xsk_buff_pool * pool)40 static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
41 {
42 	return pool->chunk_size;
43 }
44 
xsk_pool_get_rx_frame_size(struct xsk_buff_pool * pool)45 static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
46 {
47 	return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool);
48 }
49 
xsk_pool_set_rxq_info(struct xsk_buff_pool * pool,struct xdp_rxq_info * rxq)50 static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
51 					 struct xdp_rxq_info *rxq)
52 {
53 	xp_set_rxq_info(pool, rxq);
54 }
55 
xsk_pool_fill_cb(struct xsk_buff_pool * pool,struct xsk_cb_desc * desc)56 static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool,
57 				    struct xsk_cb_desc *desc)
58 {
59 	xp_fill_cb(pool, desc);
60 }
61 
xsk_pool_dma_unmap(struct xsk_buff_pool * pool,unsigned long attrs)62 static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
63 				      unsigned long attrs)
64 {
65 	xp_dma_unmap(pool, attrs);
66 }
67 
xsk_pool_dma_map(struct xsk_buff_pool * pool,struct device * dev,unsigned long attrs)68 static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
69 				   struct device *dev, unsigned long attrs)
70 {
71 	struct xdp_umem *umem = pool->umem;
72 
73 	return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs);
74 }
75 
xsk_buff_xdp_get_dma(struct xdp_buff * xdp)76 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
77 {
78 	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
79 
80 	return xp_get_dma(xskb);
81 }
82 
xsk_buff_xdp_get_frame_dma(struct xdp_buff * xdp)83 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
84 {
85 	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
86 
87 	return xp_get_frame_dma(xskb);
88 }
89 
xsk_buff_alloc(struct xsk_buff_pool * pool)90 static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
91 {
92 	return xp_alloc(pool);
93 }
94 
xsk_is_eop_desc(const struct xdp_desc * desc)95 static inline bool xsk_is_eop_desc(const struct xdp_desc *desc)
96 {
97 	return !xp_mb_desc(desc);
98 }
99 
100 /* Returns as many entries as possible up to max. 0 <= N <= max. */
xsk_buff_alloc_batch(struct xsk_buff_pool * pool,struct xdp_buff ** xdp,u32 max)101 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
102 {
103 	return xp_alloc_batch(pool, xdp, max);
104 }
105 
xsk_buff_can_alloc(struct xsk_buff_pool * pool,u32 count)106 static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
107 {
108 	return xp_can_alloc(pool, count);
109 }
110 
xsk_buff_free(struct xdp_buff * xdp)111 static inline void xsk_buff_free(struct xdp_buff *xdp)
112 {
113 	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
114 	struct list_head *xskb_list = &xskb->pool->xskb_list;
115 	struct xdp_buff_xsk *pos, *tmp;
116 
117 	if (likely(!xdp_buff_has_frags(xdp)))
118 		goto out;
119 
120 	list_for_each_entry_safe(pos, tmp, xskb_list, list_node) {
121 		list_del(&pos->list_node);
122 		xp_free(pos);
123 	}
124 
125 	xdp_get_shared_info_from_buff(xdp)->nr_frags = 0;
126 out:
127 	xp_free(xskb);
128 }
129 
xsk_buff_add_frag(struct xdp_buff * head,struct xdp_buff * xdp)130 static inline bool xsk_buff_add_frag(struct xdp_buff *head,
131 				     struct xdp_buff *xdp)
132 {
133 	const void *data = xdp->data;
134 	struct xdp_buff_xsk *frag;
135 
136 	if (!__xdp_buff_add_frag(head, virt_to_netmem(data),
137 				 offset_in_page(data), xdp->data_end - data,
138 				 xdp->frame_sz, false))
139 		return false;
140 
141 	frag = container_of(xdp, struct xdp_buff_xsk, xdp);
142 	list_add_tail(&frag->list_node, &frag->pool->xskb_list);
143 
144 	return true;
145 }
146 
xsk_buff_get_frag(const struct xdp_buff * first)147 static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
148 {
149 	struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
150 	struct xdp_buff *ret = NULL;
151 	struct xdp_buff_xsk *frag;
152 
153 	frag = list_first_entry_or_null(&xskb->pool->xskb_list,
154 					struct xdp_buff_xsk, list_node);
155 	if (frag) {
156 		list_del(&frag->list_node);
157 		ret = &frag->xdp;
158 	}
159 
160 	return ret;
161 }
162 
xsk_buff_del_tail(struct xdp_buff * tail)163 static inline void xsk_buff_del_tail(struct xdp_buff *tail)
164 {
165 	struct xdp_buff_xsk *xskb = container_of(tail, struct xdp_buff_xsk, xdp);
166 
167 	list_del(&xskb->list_node);
168 }
169 
xsk_buff_get_tail(struct xdp_buff * first)170 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
171 {
172 	struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
173 	struct xdp_buff_xsk *frag;
174 
175 	frag = list_last_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk,
176 			       list_node);
177 	return &frag->xdp;
178 }
179 
xsk_buff_set_size(struct xdp_buff * xdp,u32 size)180 static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
181 {
182 	xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
183 	xdp->data_meta = xdp->data;
184 	xdp->data_end = xdp->data + size;
185 	xdp->flags = 0;
186 }
187 
xsk_buff_raw_get_dma(struct xsk_buff_pool * pool,u64 addr)188 static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
189 					      u64 addr)
190 {
191 	return xp_raw_get_dma(pool, addr);
192 }
193 
xsk_buff_raw_get_data(struct xsk_buff_pool * pool,u64 addr)194 static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
195 {
196 	return xp_raw_get_data(pool, addr);
197 }
198 
199 /**
200  * xsk_buff_raw_get_ctx - get &xdp_desc context
201  * @pool: XSk buff pool desc address belongs to
202  * @addr: desc address (from userspace)
203  *
204  * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for
205  * details.
206  *
207  * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata
208  * pointer, if it is present and valid (initialized to %NULL otherwise).
209  */
210 static inline struct xdp_desc_ctx
xsk_buff_raw_get_ctx(const struct xsk_buff_pool * pool,u64 addr)211 xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
212 {
213 	return xp_raw_get_ctx(pool, addr);
214 }
215 
216 #define XDP_TXMD_FLAGS_VALID ( \
217 		XDP_TXMD_FLAGS_TIMESTAMP | \
218 		XDP_TXMD_FLAGS_CHECKSUM | \
219 		XDP_TXMD_FLAGS_LAUNCH_TIME | \
220 	0)
221 
222 static inline bool
xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata * meta)223 xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta)
224 {
225 	return !(meta->flags & ~XDP_TXMD_FLAGS_VALID);
226 }
227 
228 static inline struct xsk_tx_metadata *
__xsk_buff_get_metadata(const struct xsk_buff_pool * pool,void * data)229 __xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data)
230 {
231 	struct xsk_tx_metadata *meta;
232 
233 	if (!pool->tx_metadata_len)
234 		return NULL;
235 
236 	meta = data - pool->tx_metadata_len;
237 	if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
238 		return NULL; /* no way to signal the error to the user */
239 
240 	return meta;
241 }
242 
243 static inline struct xsk_tx_metadata *
xsk_buff_get_metadata(struct xsk_buff_pool * pool,u64 addr)244 xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
245 {
246 	return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr));
247 }
248 
xsk_buff_dma_sync_for_cpu(struct xdp_buff * xdp)249 static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
250 {
251 	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
252 
253 	xp_dma_sync_for_cpu(xskb);
254 }
255 
xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool * pool,dma_addr_t dma,size_t size)256 static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
257 						    dma_addr_t dma,
258 						    size_t size)
259 {
260 	xp_dma_sync_for_device(pool, dma, size);
261 }
262 
263 #else
264 
xsk_tx_completed(struct xsk_buff_pool * pool,u32 nb_entries)265 static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
266 {
267 }
268 
xsk_tx_peek_desc(struct xsk_buff_pool * pool,struct xdp_desc * desc)269 static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
270 				    struct xdp_desc *desc)
271 {
272 	return false;
273 }
274 
xsk_tx_peek_release_desc_batch(struct xsk_buff_pool * pool,u32 max)275 static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
276 {
277 	return 0;
278 }
279 
xsk_tx_release(struct xsk_buff_pool * pool)280 static inline void xsk_tx_release(struct xsk_buff_pool *pool)
281 {
282 }
283 
284 static inline struct xsk_buff_pool *
xsk_get_pool_from_qid(struct net_device * dev,u16 queue_id)285 xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id)
286 {
287 	return NULL;
288 }
289 
xsk_set_rx_need_wakeup(struct xsk_buff_pool * pool)290 static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
291 {
292 }
293 
xsk_set_tx_need_wakeup(struct xsk_buff_pool * pool)294 static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
295 {
296 }
297 
xsk_clear_rx_need_wakeup(struct xsk_buff_pool * pool)298 static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
299 {
300 }
301 
xsk_clear_tx_need_wakeup(struct xsk_buff_pool * pool)302 static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
303 {
304 }
305 
xsk_uses_need_wakeup(struct xsk_buff_pool * pool)306 static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
307 {
308 	return false;
309 }
310 
xsk_pool_get_headroom(struct xsk_buff_pool * pool)311 static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
312 {
313 	return 0;
314 }
315 
xsk_pool_get_chunk_size(struct xsk_buff_pool * pool)316 static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
317 {
318 	return 0;
319 }
320 
xsk_pool_get_rx_frame_size(struct xsk_buff_pool * pool)321 static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
322 {
323 	return 0;
324 }
325 
xsk_pool_set_rxq_info(struct xsk_buff_pool * pool,struct xdp_rxq_info * rxq)326 static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
327 					 struct xdp_rxq_info *rxq)
328 {
329 }
330 
xsk_pool_fill_cb(struct xsk_buff_pool * pool,struct xsk_cb_desc * desc)331 static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool,
332 				    struct xsk_cb_desc *desc)
333 {
334 }
335 
xsk_pool_dma_unmap(struct xsk_buff_pool * pool,unsigned long attrs)336 static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
337 				      unsigned long attrs)
338 {
339 }
340 
xsk_pool_dma_map(struct xsk_buff_pool * pool,struct device * dev,unsigned long attrs)341 static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
342 				   struct device *dev, unsigned long attrs)
343 {
344 	return 0;
345 }
346 
xsk_buff_xdp_get_dma(struct xdp_buff * xdp)347 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
348 {
349 	return 0;
350 }
351 
xsk_buff_xdp_get_frame_dma(struct xdp_buff * xdp)352 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
353 {
354 	return 0;
355 }
356 
xsk_buff_alloc(struct xsk_buff_pool * pool)357 static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
358 {
359 	return NULL;
360 }
361 
xsk_is_eop_desc(const struct xdp_desc * desc)362 static inline bool xsk_is_eop_desc(const struct xdp_desc *desc)
363 {
364 	return false;
365 }
366 
xsk_buff_alloc_batch(struct xsk_buff_pool * pool,struct xdp_buff ** xdp,u32 max)367 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
368 {
369 	return 0;
370 }
371 
xsk_buff_can_alloc(struct xsk_buff_pool * pool,u32 count)372 static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
373 {
374 	return false;
375 }
376 
xsk_buff_free(struct xdp_buff * xdp)377 static inline void xsk_buff_free(struct xdp_buff *xdp)
378 {
379 }
380 
xsk_buff_add_frag(struct xdp_buff * head,struct xdp_buff * xdp)381 static inline bool xsk_buff_add_frag(struct xdp_buff *head,
382 				     struct xdp_buff *xdp)
383 {
384 	return false;
385 }
386 
xsk_buff_get_frag(const struct xdp_buff * first)387 static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
388 {
389 	return NULL;
390 }
391 
xsk_buff_del_tail(struct xdp_buff * tail)392 static inline void xsk_buff_del_tail(struct xdp_buff *tail)
393 {
394 }
395 
xsk_buff_get_tail(struct xdp_buff * first)396 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
397 {
398 	return NULL;
399 }
400 
xsk_buff_set_size(struct xdp_buff * xdp,u32 size)401 static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
402 {
403 }
404 
xsk_buff_raw_get_dma(struct xsk_buff_pool * pool,u64 addr)405 static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
406 					      u64 addr)
407 {
408 	return 0;
409 }
410 
xsk_buff_raw_get_data(struct xsk_buff_pool * pool,u64 addr)411 static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
412 {
413 	return NULL;
414 }
415 
416 static inline struct xdp_desc_ctx
xsk_buff_raw_get_ctx(const struct xsk_buff_pool * pool,u64 addr)417 xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)
418 {
419 	return (struct xdp_desc_ctx){ };
420 }
421 
xsk_buff_valid_tx_metadata(struct xsk_tx_metadata * meta)422 static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta)
423 {
424 	return false;
425 }
426 
427 static inline struct xsk_tx_metadata *
__xsk_buff_get_metadata(const struct xsk_buff_pool * pool,void * data)428 __xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data)
429 {
430 	return NULL;
431 }
432 
433 static inline struct xsk_tx_metadata *
xsk_buff_get_metadata(struct xsk_buff_pool * pool,u64 addr)434 xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
435 {
436 	return NULL;
437 }
438 
xsk_buff_dma_sync_for_cpu(struct xdp_buff * xdp)439 static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
440 {
441 }
442 
xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool * pool,dma_addr_t dma,size_t size)443 static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
444 						    dma_addr_t dma,
445 						    size_t size)
446 {
447 }
448 
449 #endif /* CONFIG_XDP_SOCKETS */
450 
451 #endif /* _LINUX_XDP_SOCK_DRV_H */
452