2 /* net/core/xdp.c
21 #include <net/xdp.h>
23 #include <trace/events/xdp.h>
372  * xdp_reg_page_pool - register &page_pool as a memory provider for XDP
375  * Can be used to register pools manually without connecting to any XDP RxQ
376  * info, so that the XDP layer will be aware of them. Then, they can be
409  * @xdp_rxq: XDP RxQ info to attach the pool to
427 /* XDP RX runs under NAPI protection, and in different delivery error
434 		  bool napi_direct, struct xdp_buff *xdp)  in __xdp_return()  argument
455 		xsk_buff_free(xdp);  in __xdp_return()
459 		WARN(1, "Incorrect XDP memory type (%d) usage", mem_type);  in __xdp_return()
498 /* XDP bulk APIs introduce a defer/flush mechanism to return
539  * xdp_return_frag -- free one XDP frag or decrement its refcount
541  * @xdp: &xdp_buff to release the frag for
543 void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp)  in xdp_return_frag()  argument
545 	__xdp_return(netmem, xdp->rxq->mem.type, true, NULL);  in xdp_return_frag()
549 void xdp_return_buff(struct xdp_buff *xdp)  in xdp_return_buff()  argument
553 	if (likely(!xdp_buff_has_frags(xdp)))  in xdp_return_buff()
556 	sinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_return_buff()
559 			     xdp->rxq->mem.type, true, xdp);  in xdp_return_buff()
562 	__xdp_return(virt_to_netmem(xdp->data), xdp->rxq->mem.type, true, xdp);  in xdp_return_buff()
576 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)  in xdp_convert_zc_to_xdp_frame()  argument
584 	metasize = xdp_data_meta_unsupported(xdp) ? 0 :  in xdp_convert_zc_to_xdp_frame()
585 		   xdp->data - xdp->data_meta;  in xdp_convert_zc_to_xdp_frame()
586 	totsize = xdp->data_end - xdp->data + metasize;  in xdp_convert_zc_to_xdp_frame()
600 	data_to_copy = metasize ? xdp->data_meta : xdp->data;  in xdp_convert_zc_to_xdp_frame()
610 	xsk_buff_free(xdp);  in xdp_convert_zc_to_xdp_frame()
624  * @xdp: &xdp_buff to convert to an skb
633 struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp)  in xdp_build_skb_from_buff()  argument
635 	const struct xdp_rxq_info *rxq = xdp->rxq;  in xdp_build_skb_from_buff()
641 	if (unlikely(xdp_buff_has_frags(xdp))) {  in xdp_build_skb_from_buff()
642 		sinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_build_skb_from_buff()
646 	skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);  in xdp_build_skb_from_buff()
650 	skb_reserve(skb, xdp->data - xdp->data_hard_start);  in xdp_build_skb_from_buff()
651 	__skb_put(skb, xdp->data_end - xdp->data);  in xdp_build_skb_from_buff()
653 	metalen = xdp->data - xdp->data_meta;  in xdp_build_skb_from_buff()
665 		tsize = sinfo->xdp_frags_truesize ? : nr_frags * xdp->frame_sz;  in xdp_build_skb_from_buff()
667 					  tsize, xdp_buff_get_skb_flags(xdp));  in xdp_build_skb_from_buff()
679  * @xdp: XSk &xdp_buff from which the frags will be copied
688 					    const struct xdp_buff *xdp,  in xdp_copy_frags_from_zc()  argument
696 	xinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_copy_frags_from_zc()
728  * @xdp: source XSk buff
734  * If new skb was built successfully, @xdp is returned to XSk pool's freelist.
739 struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp)  in xdp_build_skb_from_zc()  argument
741 	const struct xdp_rxq_info *rxq = xdp->rxq;  in xdp_build_skb_from_zc()
742 	u32 len = xdp->data_end - xdp->data_meta;  in xdp_build_skb_from_zc()
743 	u32 truesize = xdp->frame_sz;  in xdp_build_skb_from_zc()
765 	skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);  in xdp_build_skb_from_zc()
767 	memcpy(__skb_put(skb, len), xdp->data_meta, LARGEST_ALIGN(len));  in xdp_build_skb_from_zc()
769 	metalen = xdp->data - xdp->data_meta;  in xdp_build_skb_from_zc()
777 	if (unlikely(xdp_buff_has_frags(xdp)) &&  in xdp_build_skb_from_zc()
778 	    unlikely(!xdp_copy_frags_from_zc(skb, xdp, pp))) {  in xdp_build_skb_from_zc()
784 	xsk_buff_free(xdp);  in xdp_build_skb_from_zc()
803 	/* xdp frags frame */  in __xdp_build_skb_from_frame()
894  * bpf_xdp_metadata_rx_timestamp - Read XDP frame RX timestamp.
895  * @ctx: XDP context pointer.
909  * bpf_xdp_metadata_rx_hash - Read XDP frame RX hash.
910  * @ctx: XDP context pointer.
932  * bpf_xdp_metadata_rx_vlan_tag - Get XDP packet outermost VLAN tag
933  * @ctx: XDP context pointer.