6 #include <net/xdp.h>
355  * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
360  * XDP requires more memory, than rx_buf provides.
438  * @xdp: SW ring of xdp_buff that will hold the buffers
449 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,  in ice_fill_rx_descs()  argument
456 	buffs = xsk_buff_alloc_batch(pool, xdp, count);  in ice_fill_rx_descs()
458 		dma = xsk_buff_xdp_get_dma(*xdp);  in ice_fill_rx_descs()
465 		ice_xdp_meta_set_desc(*xdp, rx_desc);  in ice_fill_rx_descs()
468 		xdp++;  in ice_fill_rx_descs()
491 	struct xdp_buff **xdp;  in __ice_alloc_rx_bufs_zc()  local
494 	xdp = ice_xdp_buf(rx_ring, ntu);  in __ice_alloc_rx_bufs_zc()
497 		nb_buffs_extra = ice_fill_rx_descs(rx_ring->xsk_pool, xdp,  in __ice_alloc_rx_bufs_zc()
505 		xdp = ice_xdp_buf(rx_ring, 0);  in __ice_alloc_rx_bufs_zc()
511 	nb_buffs = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, rx_desc, count);  in __ice_alloc_rx_bufs_zc()
551  * @xdp: Pointer to XDP buffer
558 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)  in ice_construct_skb_zc()  argument
560 	unsigned int totalsize = xdp->data_end - xdp->data_meta;  in ice_construct_skb_zc()
561 	unsigned int metasize = xdp->data - xdp->data_meta;  in ice_construct_skb_zc()
566 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_construct_skb_zc()
567 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_construct_skb_zc()
570 	net_prefetch(xdp->data_meta);  in ice_construct_skb_zc()
577 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,  in ice_construct_skb_zc()
585 	if (likely(!xdp_buff_has_frags(xdp)))  in ice_construct_skb_zc()
608 	xsk_buff_free(xdp);  in ice_construct_skb_zc()
614  * @xdp_ring: XDP Tx ring
651 			xsk_buff_free(tx_buf->xdp);  in ice_clean_xdp_irq_zc()
674  * @xdp: XDP buffer to xmit
675  * @xdp_ring: XDP ring to produce descriptor onto
682  * was not enough space on XDP ring
684 static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,  in ice_xmit_xdp_tx_zc()  argument
688 	u32 size = xdp->data_end - xdp->data;  in ice_xmit_xdp_tx_zc()
704 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_xmit_xdp_tx_zc()
705 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_xmit_xdp_tx_zc()
713 	head = xdp;  in ice_xmit_xdp_tx_zc()
718 		dma = xsk_buff_xdp_get_dma(xdp);  in ice_xmit_xdp_tx_zc()
721 		tx_buf->xdp = xdp;  in ice_xmit_xdp_tx_zc()
737 		xdp = xsk_buff_get_frag(head);  in ice_xmit_xdp_tx_zc()
756  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
758  * @xdp: xdp_buff used as input to the XDP program
759  * @xdp_prog: XDP program to run
765 ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,  in ice_run_xdp_zc()  argument
771 	act = bpf_prog_run_xdp(xdp_prog, xdp);  in ice_run_xdp_zc()
774 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in ice_run_xdp_zc()
788 		result = ice_xmit_xdp_tx_zc(xdp, xdp_ring);  in ice_run_xdp_zc()
810 		 struct xdp_buff *xdp, const unsigned int size)  in ice_add_xsk_frag()  argument
829 				   virt_to_page(xdp->data_hard_start),  in ice_add_xsk_frag()
832 	xsk_buff_add_frag(xdp);  in ice_add_xsk_frag()
858 	/* ZC patch is enabled only when XDP program is set,  in ice_clean_rx_irq_zc()
870 		struct xdp_buff *xdp;  in ice_clean_rx_irq_zc()  local
890 		xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_clean_rx_irq_zc()
895 		xsk_buff_set_size(xdp, size);  in ice_clean_rx_irq_zc()
896 		xsk_buff_dma_sync_for_cpu(xdp, xsk_pool);  in ice_clean_rx_irq_zc()
899 			first = xdp;  in ice_clean_rx_irq_zc()
900 		} else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) {  in ice_clean_rx_irq_zc()
981  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
1004  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1034  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1054  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1165 		struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_xsk_clean_rx_ring()  local
1167 		xsk_buff_free(xdp);  in ice_xsk_clean_rx_ring()
1175  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
1188 			xsk_buff_free(tx_buf->xdp);  in ice_xsk_clean_xdp_ring()