464 void xdp_return_frame(struct xdp_frame *xdpf)  in xdp_return_frame()  argument
468 	if (likely(!xdp_frame_has_frags(xdpf)))  in xdp_return_frame()
471 	sinfo = xdp_get_shared_info_from_frame(xdpf);  in xdp_return_frame()
473 		__xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,  in xdp_return_frame()
477 	__xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, false, NULL);  in xdp_return_frame()
481 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)  in xdp_return_frame_rx_napi()  argument
485 	if (likely(!xdp_frame_has_frags(xdpf)))  in xdp_return_frame_rx_napi()
488 	sinfo = xdp_get_shared_info_from_frame(xdpf);  in xdp_return_frame_rx_napi()
490 		__xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type,  in xdp_return_frame_rx_napi()
494 	__xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, true, NULL);  in xdp_return_frame_rx_napi()
510 void xdp_return_frame_bulk(struct xdp_frame *xdpf,  in xdp_return_frame_bulk()  argument
513 	if (xdpf->mem_type != MEM_TYPE_PAGE_POOL) {  in xdp_return_frame_bulk()
514 		xdp_return_frame(xdpf);  in xdp_return_frame_bulk()
521 	if (unlikely(xdp_frame_has_frags(xdpf))) {  in xdp_return_frame_bulk()
525 		sinfo = xdp_get_shared_info_from_frame(xdpf);  in xdp_return_frame_bulk()
534 	bq->q[bq->count++] = virt_to_netmem(xdpf->data);  in xdp_return_frame_bulk()
580 	struct xdp_frame *xdpf;  in xdp_convert_zc_to_xdp_frame()  local
588 	if (sizeof(*xdpf) + totsize > PAGE_SIZE)  in xdp_convert_zc_to_xdp_frame()
596 	xdpf = addr;  in xdp_convert_zc_to_xdp_frame()
597 	memset(xdpf, 0, sizeof(*xdpf));  in xdp_convert_zc_to_xdp_frame()
599 	addr += sizeof(*xdpf);  in xdp_convert_zc_to_xdp_frame()
603 	xdpf->data = addr + metasize;  in xdp_convert_zc_to_xdp_frame()
604 	xdpf->len = totsize - metasize;  in xdp_convert_zc_to_xdp_frame()
605 	xdpf->headroom = 0;  in xdp_convert_zc_to_xdp_frame()
606 	xdpf->metasize = metasize;  in xdp_convert_zc_to_xdp_frame()
607 	xdpf->frame_sz = PAGE_SIZE;  in xdp_convert_zc_to_xdp_frame()
608 	xdpf->mem_type = MEM_TYPE_PAGE_ORDER0;  in xdp_convert_zc_to_xdp_frame()
611 	return xdpf;  in xdp_convert_zc_to_xdp_frame()
794 struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,  in __xdp_build_skb_from_frame()  argument
798 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);  in __xdp_build_skb_from_frame()
804 	if (unlikely(xdp_frame_has_frags(xdpf)))  in __xdp_build_skb_from_frame()
808 	headroom = sizeof(*xdpf) + xdpf->headroom;  in __xdp_build_skb_from_frame()
813 	frame_size = xdpf->frame_sz;  in __xdp_build_skb_from_frame()
815 	hard_start = xdpf->data - headroom;  in __xdp_build_skb_from_frame()
821 	__skb_put(skb, xdpf->len);  in __xdp_build_skb_from_frame()
822 	if (xdpf->metasize)  in __xdp_build_skb_from_frame()
823 		skb_metadata_set(skb, xdpf->metasize);  in __xdp_build_skb_from_frame()
825 	if (unlikely(xdp_frame_has_frags(xdpf)))  in __xdp_build_skb_from_frame()
827 					  nr_frags * xdpf->frame_sz,  in __xdp_build_skb_from_frame()
828 					  xdp_frame_get_skb_flags(xdpf));  in __xdp_build_skb_from_frame()
839 	if (xdpf->mem_type == MEM_TYPE_PAGE_POOL)  in __xdp_build_skb_from_frame()
843 	xdp_scrub_frame(xdpf);  in __xdp_build_skb_from_frame()
849 struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,  in xdp_build_skb_from_frame()  argument
860 	return __xdp_build_skb_from_frame(xdpf, skb, dev);  in xdp_build_skb_from_frame()
864 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf)  in xdpf_clone()  argument
871 	headroom = xdpf->headroom + sizeof(*xdpf);  in xdpf_clone()
872 	totalsize = headroom + xdpf->len;  in xdpf_clone()
881 	memcpy(addr, xdpf, totalsize);  in xdpf_clone()