Lines Matching +full:xdp +full:- +full:rx +full:- +full:metadata
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* include/net/xdp.h
15 * DOC: XDP RX-queue information
17 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
18 * level RX-ring queues. It is information that is specific to how
19 * the driver has configured a given RX-ring queue.
22 * reference to this xdp_rxq_info structure. This provides the XDP
23 * data-path read-access to RX-info for both kernel and bpf-side
27 * context. Contents are read-mostly and must not be updated during
32 * The struct is not directly tied to the XDP prog. A new XDP prog
34 * RX-ring. If the RX-ring does change significantly, the NIC driver
35 * naturally needs to stop the RX-ring before purging and reallocating
38 * also mandatory during RX-ring setup.
42 MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
43 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
49 /* XDP flags for ndo_xdp_xmit */
67 } ____cacheline_aligned; /* perf critical, avoid false-sharing */
74 XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
75 XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
91 static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp) in xdp_buff_has_frags() argument
93 return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); in xdp_buff_has_frags()
96 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp) in xdp_buff_set_frags_flag() argument
98 xdp->flags |= XDP_FLAGS_HAS_FRAGS; in xdp_buff_set_frags_flag()
101 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp) in xdp_buff_clear_frags_flag() argument
103 xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; in xdp_buff_clear_frags_flag()
106 static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_is_frag_pfmemalloc() argument
108 return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); in xdp_buff_is_frag_pfmemalloc()
111 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_set_frag_pfmemalloc() argument
113 xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; in xdp_buff_set_frag_pfmemalloc()
117 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) in xdp_init_buff() argument
119 xdp->frame_sz = frame_sz; in xdp_init_buff()
120 xdp->rxq = rxq; in xdp_init_buff()
121 xdp->flags = 0; in xdp_init_buff()
125 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, in xdp_prepare_buff() argument
130 xdp->data_hard_start = hard_start; in xdp_prepare_buff()
131 xdp->data = data; in xdp_prepare_buff()
132 xdp->data_end = data + data_len; in xdp_prepare_buff()
133 xdp->data_meta = meta_valid ? data : data + 1; in xdp_prepare_buff()
136 /* Reserve memory area at end-of data area.
138 * This macro reserves tailroom in the XDP buffer by limiting the
139 * XDP/BPF data access to data_hard_end. Notice same area (and size)
142 #define xdp_data_hard_end(xdp) \ argument
143 ((xdp)->data_hard_start + (xdp)->frame_sz - \
147 xdp_get_shared_info_from_buff(struct xdp_buff *xdp) in xdp_get_shared_info_from_buff() argument
149 return (struct skb_shared_info *)xdp_data_hard_end(xdp); in xdp_get_shared_info_from_buff()
152 static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp) in xdp_get_buff_len() argument
154 unsigned int len = xdp->data_end - xdp->data; in xdp_get_buff_len()
157 if (likely(!xdp_buff_has_frags(xdp))) in xdp_get_buff_len()
160 sinfo = xdp_get_shared_info_from_buff(xdp); in xdp_get_buff_len()
161 len += sinfo->xdp_frags_size; in xdp_get_buff_len()
170 u32 metasize; /* uses lower 8-bits */
182 return !!(frame->flags & XDP_FLAGS_HAS_FRAGS); in xdp_frame_has_frags()
187 return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); in xdp_frame_is_frag_pfmemalloc()
199 /* bq->count will be zero'ed when bq->xa gets updated */ in xdp_frame_bulk_init()
200 bq->xa = NULL; in xdp_frame_bulk_init()
206 void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_get_shared_info_from_frame()
208 return (struct skb_shared_info *)(data_hard_start + frame->frame_sz - in xdp_get_shared_info_from_frame()
221 frame->data = NULL; in xdp_scrub_frame()
222 frame->dev_rx = NULL; in xdp_scrub_frame()
230 skb_shinfo(skb)->nr_frags = nr_frags; in xdp_update_skb_shared_info()
232 skb->len += size; in xdp_update_skb_shared_info()
233 skb->data_len += size; in xdp_update_skb_shared_info()
234 skb->truesize += truesize; in xdp_update_skb_shared_info()
235 skb->pfmemalloc |= pfmemalloc; in xdp_update_skb_shared_info()
238 /* Avoids inlining WARN macro in fast-path */
242 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
252 void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) in xdp_convert_frame_to_buff() argument
254 xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_convert_frame_to_buff()
255 xdp->data = frame->data; in xdp_convert_frame_to_buff()
256 xdp->data_end = frame->data + frame->len; in xdp_convert_frame_to_buff()
257 xdp->data_meta = frame->data - frame->metasize; in xdp_convert_frame_to_buff()
258 xdp->frame_sz = frame->frame_sz; in xdp_convert_frame_to_buff()
259 xdp->flags = frame->flags; in xdp_convert_frame_to_buff()
263 int xdp_update_frame_from_buff(struct xdp_buff *xdp, in xdp_update_frame_from_buff() argument
269 headroom = xdp->data - xdp->data_hard_start; in xdp_update_frame_from_buff()
270 metasize = xdp->data - xdp->data_meta; in xdp_update_frame_from_buff()
272 if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) in xdp_update_frame_from_buff()
273 return -ENOSPC; in xdp_update_frame_from_buff()
276 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { in xdp_update_frame_from_buff()
278 return -ENOSPC; in xdp_update_frame_from_buff()
281 xdp_frame->data = xdp->data; in xdp_update_frame_from_buff()
282 xdp_frame->len = xdp->data_end - xdp->data; in xdp_update_frame_from_buff()
283 xdp_frame->headroom = headroom - sizeof(*xdp_frame); in xdp_update_frame_from_buff()
284 xdp_frame->metasize = metasize; in xdp_update_frame_from_buff()
285 xdp_frame->frame_sz = xdp->frame_sz; in xdp_update_frame_from_buff()
286 xdp_frame->flags = xdp->flags; in xdp_update_frame_from_buff()
293 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) in xdp_convert_buff_to_frame() argument
297 if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) in xdp_convert_buff_to_frame()
298 return xdp_convert_zc_to_xdp_frame(xdp); in xdp_convert_buff_to_frame()
301 xdp_frame = xdp->data_hard_start; in xdp_convert_buff_to_frame()
302 if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) in xdp_convert_buff_to_frame()
306 xdp_frame->mem = xdp->rxq->mem; in xdp_convert_buff_to_frame()
312 struct xdp_buff *xdp);
315 void xdp_return_buff(struct xdp_buff *xdp);
323 unsigned int len = xdpf->len; in xdp_get_frame_len()
329 len += sinfo->xdp_frags_size; in xdp_get_frame_len()
355 /* Drivers not supporting XDP metadata can use this helper, which
356 * rejects any room expansion for metadata as a result.
359 xdp_set_data_meta_invalid(struct xdp_buff *xdp) in xdp_set_data_meta_invalid() argument
361 xdp->data_meta = xdp->data + 1; in xdp_set_data_meta_invalid()
365 xdp_data_meta_unsupported(const struct xdp_buff *xdp) in xdp_data_meta_unsupported() argument
367 return unlikely(xdp->data_meta > xdp->data); in xdp_data_meta_unsupported()
391 /* Define the relationship between xdp-rx-metadata kfunc and
393 * - xdp_rx_metadata enum
394 * - netdev netlink enum (Documentation/netlink/specs/netdev.yaml)
395 * - kfunc name
396 * - xdp_metadata_ops field
508 struct xdp_buff *xdp) in bpf_prog_run_xdp() argument
510 /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus in bpf_prog_run_xdp()
514 u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp)); in bpf_prog_run_xdp()
517 if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev)) in bpf_prog_run_xdp()
518 act = xdp_master_redirect(xdp); in bpf_prog_run_xdp()