net/libeth/xsk.h

7 #include <net/libeth/xdp.h>
19  * @bq: XDP Tx bulk to queue the head frag to
20  * @xdp: XSk buffer with the head to queue
25 					    struct libeth_xdp_buff *xdp)
28 		.xsk	= xdp,
29 		__libeth_xdp_tx_len(xdp->base.data_end - xdp->data,
33 	if (likely(!xdp_buff_has_frags(&xdp->base)))
43  * @bq: XDP Tx bulk to queue the frag to
57  * @bq: XDP Tx bulk to queue the frame to
58  * @xdp: XSk buffer to queue
65 			 struct libeth_xdp_buff *xdp,
73 		libeth_xsk_buff_free_slow(xdp);
77 	if (!libeth_xsk_tx_queue_head(bq, xdp))
80 	for (const struct libeth_xdp_buff *head = xdp; ; ) {
81 		xdp = container_of(xsk_buff_get_frag(&head->base),
82 				   typeof(*xdp), base);
83 		if (!xdp)
92 		libeth_xsk_tx_queue_frag(bq, xdp);
103  * @frm: XDP Tx frame from the bulk
108  * Return: XDP Tx descriptor with the synced DMA and other info to pass to
115 	struct libeth_xdp_buff *xdp = frm.xsk;
117 		.addr	= xsk_buff_xdp_get_dma(&xdp->base),
125 	sqe->xsk = xdp;
134 				 xdp_get_shared_info_from_buff(&xdp->base));
187  * Return: XDP Tx descriptor with the DMA, metadata request bits, and other
220  * Return: XDP Tx descriptor with the DMA and other info to pass to
241  * the quickest helper to build a libeth XDP Tx descriptor.
243  * Return: XDP Tx descriptor with the synced DMA, metadata request bits,
278  * Note that unlike other XDP Tx ops, the queue must be locked and cleaned
317  * libeth_xsk_tx_init_bulk - initialize XDP Tx bulk for an XSk Rx NAPI poll
319  * @prog: RCU pointer to the XDP program (never %NULL)
324  * Should be called on an onstack XDP Tx bulk before the XSk NAPI polling loop.
326  * Never checks if @prog is %NULL or @num == 0 as XDP must always be enabled
334 						 struct libeth_xdp_buff *xdp);
339  * @xdp: XSk buffer to process
352 			struct libeth_xdp_buff *xdp, u32 len)
355 		libeth_xsk_buff_free_slow(xdp);
359 	xsk_buff_set_size(&xdp->base, len);
360 	xsk_buff_dma_sync_for_cpu(&xdp->base);
363 		return libeth_xsk_buff_add_frag(head, xdp);
365 	prefetch(xdp->data);
367 	return xdp;
371 				 const struct libeth_xdp_buff *xdp);
373 u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp,
378  * __libeth_xsk_run_prog - run XDP program on XSk buffer
379  * @xdp: XSk buffer to run the prog on
382  * Internal inline abstraction to run XDP program on XSk Rx path. Handles
385  * Reports an XDP prog exception on errors.
390 __libeth_xsk_run_prog(struct libeth_xdp_buff *xdp,
396 	act = bpf_prog_run_xdp(bq->prog, &xdp->base);
399 		return __libeth_xsk_run_prog_slow(xdp, bq, act, ret);
401 	ret = xdp_do_redirect(bq->dev, &xdp->base, bq->prog);
409  * libeth_xsk_run_prog - run XDP program on XSk path and handle all verdicts
410  * @xdp: XSk buffer to process
411  * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
414  * Run the attached XDP program and handle all possible verdicts.
419 #define libeth_xsk_run_prog(xdp, bq, fl)				     \
420 	__libeth_xdp_run_flush(xdp, bq, __libeth_xsk_run_prog,		     \
424  * __libeth_xsk_run_pass - helper to run XDP program and handle the result
425  * @xdp: XSk buffer to process
426  * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
431  * @run: driver wrapper to run XDP program
441 __libeth_xsk_run_pass(struct libeth_xdp_buff *xdp,
444 		      void (*prep)(struct libeth_xdp_buff *xdp,
446 		      u32 (*run)(struct libeth_xdp_buff *xdp,
449 				       const struct libeth_xdp_buff *xdp,
455 	rs->bytes += xdp->base.data_end - xdp->data;
458 	if (unlikely(xdp_buff_has_frags(&xdp->base)))
459 		libeth_xsk_buff_stats_frags(rs, xdp);
462 		prep(xdp, md);
464 	act = run(xdp, bq);
471 	skb = xdp_build_skb_from_zc(&xdp->base);
473 		libeth_xsk_buff_free_slow(xdp);
477 	if (unlikely(!populate(skb, xdp, rs))) {
488  * libeth_xsk_run_pass - helper to run XDP program and handle the result
489  * @xdp: XSk buffer to process
490  * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
494  * @run: driver wrapper to run XDP program
498  * means just writing the pointer to the HW descriptor as @xdp->desc.
500 #define libeth_xsk_run_pass(xdp, bq, napi, rs, desc, run, populate)	     \
501 	__libeth_xsk_run_pass(xdp, bq, napi, rs, desc, libeth_xdp_prep_desc, \
511  * the XDP maps.
536  *		if (!driver_xsk_run(xdp, &bq, napi, &rs, desc))
552  * LIBETH_XSK_DEFINE_RUN_PROG - define a driver XDP program run function
562  * @run: driver callback to run XDP program (above)
571  * @run: name of the XDP prog run function to define
625  * @xdp: buffer to get the DMA addr for
627 #define libeth_xsk_buff_xdp_get_dma(xdp)				     \
628 	xsk_buff_xdp_get_dma(&(xdp)->base)