Lines Matching full:xdp
2 /* include/net/xdp.h
17 * DOC: XDP RX-queue information
19 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
24 * reference to this xdp_rxq_info structure. This provides the XDP
34 * The struct is not directly tied to the XDP prog. A new XDP prog
45 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
51 /* XDP flags for ndo_xdp_xmit */
75 XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
76 XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
79 /* frags have unreadable mem, this can't be true for real XDP packets,
80 * but drivers may use XDP helpers to construct Rx pkt state even when
81 * XDP program is not attached.
109 static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp) in xdp_buff_has_frags() argument
111 return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); in xdp_buff_has_frags()
114 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp) in xdp_buff_set_frags_flag() argument
116 xdp->flags |= XDP_FLAGS_HAS_FRAGS; in xdp_buff_set_frags_flag()
119 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp) in xdp_buff_clear_frags_flag() argument
121 xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; in xdp_buff_clear_frags_flag()
124 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_set_frag_pfmemalloc() argument
126 xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; in xdp_buff_set_frag_pfmemalloc()
129 static __always_inline void xdp_buff_set_frag_unreadable(struct xdp_buff *xdp) in xdp_buff_set_frag_unreadable() argument
131 xdp->flags |= XDP_FLAGS_FRAGS_UNREADABLE; in xdp_buff_set_frag_unreadable()
134 static __always_inline u32 xdp_buff_get_skb_flags(const struct xdp_buff *xdp) in xdp_buff_get_skb_flags() argument
136 return xdp->flags; in xdp_buff_get_skb_flags()
139 static __always_inline void xdp_buff_clear_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_clear_frag_pfmemalloc() argument
141 xdp->flags &= ~XDP_FLAGS_FRAGS_PF_MEMALLOC; in xdp_buff_clear_frag_pfmemalloc()
145 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) in xdp_init_buff() argument
147 xdp->rxq = rxq; in xdp_init_buff()
155 xdp->frame_sz_flags_init = frame_sz; in xdp_init_buff()
157 xdp->frame_sz = frame_sz; in xdp_init_buff()
158 xdp->flags = 0; in xdp_init_buff()
163 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, in xdp_prepare_buff() argument
168 xdp->data_hard_start = hard_start; in xdp_prepare_buff()
169 xdp->data = data; in xdp_prepare_buff()
170 xdp->data_end = data + data_len; in xdp_prepare_buff()
171 xdp->data_meta = meta_valid ? data : data + 1; in xdp_prepare_buff()
176 * This macro reserves tailroom in the XDP buffer by limiting the
177 * XDP/BPF data access to data_hard_end. Notice same area (and size)
180 #define xdp_data_hard_end(xdp) \ argument
181 ((xdp)->data_hard_start + (xdp)->frame_sz - \
185 xdp_get_shared_info_from_buff(const struct xdp_buff *xdp) in xdp_get_shared_info_from_buff() argument
187 return (struct skb_shared_info *)xdp_data_hard_end(xdp); in xdp_get_shared_info_from_buff()
191 xdp_get_buff_len(const struct xdp_buff *xdp) in xdp_get_buff_len() argument
193 unsigned int len = xdp->data_end - xdp->data; in xdp_get_buff_len()
196 if (likely(!xdp_buff_has_frags(xdp))) in xdp_get_buff_len()
199 sinfo = xdp_get_shared_info_from_buff(xdp); in xdp_get_buff_len()
205 void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp);
209 * @xdp: XDP buffer to attach the frag to
216 * Attach frag to the XDP buffer. If it currently has no frags attached,
226 static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem, in __xdp_buff_add_frag() argument
230 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); in __xdp_buff_add_frag()
234 if (!xdp_buff_has_frags(xdp)) { in __xdp_buff_add_frag()
235 xdp_buff_set_frags_flag(xdp); in __xdp_buff_add_frag()
251 xdp_return_frag(netmem, xdp); in __xdp_buff_add_frag()
269 * @xdp: XDP buffer to attach the frag to
280 static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem, in xdp_buff_add_frag() argument
283 if (!__xdp_buff_add_frag(xdp, netmem, offset, size, truesize, true)) in xdp_buff_add_frag()
287 xdp_buff_set_frag_pfmemalloc(xdp); in xdp_buff_add_frag()
289 xdp_buff_set_frag_unreadable(xdp); in xdp_buff_add_frag()
377 struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp);
378 struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp);
379 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
389 struct xdp_buff *xdp) in xdp_convert_frame_to_buff() argument
391 xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_convert_frame_to_buff()
392 xdp->data = frame->data; in xdp_convert_frame_to_buff()
393 xdp->data_end = frame->data + frame->len; in xdp_convert_frame_to_buff()
394 xdp->data_meta = frame->data - frame->metasize; in xdp_convert_frame_to_buff()
395 xdp->frame_sz = frame->frame_sz; in xdp_convert_frame_to_buff()
396 xdp->flags = frame->flags; in xdp_convert_frame_to_buff()
400 int xdp_update_frame_from_buff(const struct xdp_buff *xdp, in xdp_update_frame_from_buff() argument
406 headroom = xdp->data - xdp->data_hard_start; in xdp_update_frame_from_buff()
407 metasize = xdp->data - xdp->data_meta; in xdp_update_frame_from_buff()
413 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { in xdp_update_frame_from_buff()
418 xdp_frame->data = xdp->data; in xdp_update_frame_from_buff()
419 xdp_frame->len = xdp->data_end - xdp->data; in xdp_update_frame_from_buff()
422 xdp_frame->frame_sz = xdp->frame_sz; in xdp_update_frame_from_buff()
423 xdp_frame->flags = xdp->flags; in xdp_update_frame_from_buff()
430 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) in xdp_convert_buff_to_frame() argument
434 if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) in xdp_convert_buff_to_frame()
435 return xdp_convert_zc_to_xdp_frame(xdp); in xdp_convert_buff_to_frame()
438 xdp_frame = xdp->data_hard_start; in xdp_convert_buff_to_frame()
439 if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) in xdp_convert_buff_to_frame()
443 xdp_frame->mem_type = xdp->rxq->mem.type; in xdp_convert_buff_to_frame()
449 bool napi_direct, struct xdp_buff *xdp);
452 void xdp_return_buff(struct xdp_buff *xdp);
507 * @xdp_rxq: XDP RxQ info to attach the memory info to
522 * @xdp_rxq: XDP RxQ info to detach the memory info from
533 /* Drivers not supporting XDP metadata can use this helper, which
537 xdp_set_data_meta_invalid(struct xdp_buff *xdp) in xdp_set_data_meta_invalid() argument
539 xdp->data_meta = xdp->data + 1; in xdp_set_data_meta_invalid()
543 xdp_data_meta_unsupported(const struct xdp_buff *xdp) in xdp_data_meta_unsupported() argument
545 return unlikely(xdp->data_meta > xdp->data); in xdp_data_meta_unsupported()
569 /* Define the relationship between xdp-rx-metadata kfunc and
690 struct xdp_buff *xdp) in bpf_prog_run_xdp() argument
692 /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus in bpf_prog_run_xdp()
696 u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp)); in bpf_prog_run_xdp()
699 if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev)) in bpf_prog_run_xdp()
700 act = xdp_master_redirect(xdp); in bpf_prog_run_xdp()