7 #include <net/xdp.h>
341  * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
346  * XDP requires more memory, than rx_buf provides.
427  * @xdp: SW ring of xdp_buff that will hold the buffers
438 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,  in ice_fill_rx_descs()  argument
445 	buffs = xsk_buff_alloc_batch(pool, xdp, count);  in ice_fill_rx_descs()
447 		dma = xsk_buff_xdp_get_dma(*xdp);  in ice_fill_rx_descs()
454 		ice_xdp_meta_set_desc(*xdp, rx_desc);  in ice_fill_rx_descs()
457 		xdp++;  in ice_fill_rx_descs()
482 	struct xdp_buff **xdp;  in __ice_alloc_rx_bufs_zc()  local
485 	xdp = ice_xdp_buf(rx_ring, ntu);  in __ice_alloc_rx_bufs_zc()
488 		nb_buffs_extra = ice_fill_rx_descs(xsk_pool, xdp, rx_desc,  in __ice_alloc_rx_bufs_zc()
495 		xdp = ice_xdp_buf(rx_ring, 0);  in __ice_alloc_rx_bufs_zc()
501 	nb_buffs = ice_fill_rx_descs(xsk_pool, xdp, rx_desc, count);  in __ice_alloc_rx_bufs_zc()
543  * @xdp: Pointer to XDP buffer
550 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)  in ice_construct_skb_zc()  argument
552 	unsigned int totalsize = xdp->data_end - xdp->data_meta;  in ice_construct_skb_zc()
553 	unsigned int metasize = xdp->data - xdp->data_meta;  in ice_construct_skb_zc()
558 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_construct_skb_zc()
559 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_construct_skb_zc()
562 	net_prefetch(xdp->data_meta);  in ice_construct_skb_zc()
568 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,  in ice_construct_skb_zc()
576 	if (likely(!xdp_buff_has_frags(xdp)))  in ice_construct_skb_zc()
599 	xsk_buff_free(xdp);  in ice_construct_skb_zc()
605  * @xdp_ring: XDP Tx ring
644 			xsk_buff_free(tx_buf->xdp);  in ice_clean_xdp_irq_zc()
667  * @xdp: XDP buffer to xmit
668  * @xdp_ring: XDP ring to produce descriptor onto
676  * was not enough space on XDP ring
678 static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,  in ice_xmit_xdp_tx_zc()  argument
683 	u32 size = xdp->data_end - xdp->data;  in ice_xmit_xdp_tx_zc()
699 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_xmit_xdp_tx_zc()
700 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_xmit_xdp_tx_zc()
708 	head = xdp;  in ice_xmit_xdp_tx_zc()
713 		dma = xsk_buff_xdp_get_dma(xdp);  in ice_xmit_xdp_tx_zc()
716 		tx_buf->xdp = xdp;  in ice_xmit_xdp_tx_zc()
732 		xdp = xsk_buff_get_frag(head);  in ice_xmit_xdp_tx_zc()
751  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
753  * @xdp: xdp_buff used as input to the XDP program
754  * @xdp_prog: XDP program to run
761 ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,  in ice_run_xdp_zc()  argument
768 	act = bpf_prog_run_xdp(xdp_prog, xdp);  in ice_run_xdp_zc()
771 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in ice_run_xdp_zc()
785 		result = ice_xmit_xdp_tx_zc(xdp, xdp_ring, xsk_pool);  in ice_run_xdp_zc()
828 	/* ZC patch is enabled only when XDP program is set,  in ice_clean_rx_irq_zc()
840 		struct xdp_buff *xdp;  in ice_clean_rx_irq_zc()  local
860 		xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_clean_rx_irq_zc()
865 		xsk_buff_set_size(xdp, size);  in ice_clean_rx_irq_zc()
866 		xsk_buff_dma_sync_for_cpu(xdp);  in ice_clean_rx_irq_zc()
869 			first = xdp;  in ice_clean_rx_irq_zc()
870 		} else if (likely(size) && !xsk_buff_add_frag(first, xdp)) {  in ice_clean_rx_irq_zc()
954  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
979  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1013  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1036  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1153 		struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_xsk_clean_rx_ring()  local
1155 		xsk_buff_free(xdp);  in ice_xsk_clean_rx_ring()
1163  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
1176 			xsk_buff_free(tx_buf->xdp);  in ice_xsk_clean_xdp_ring()