Home
last modified time | relevance | path

Searched full:xdp (Results 1 – 25 of 473) sorted by relevance

12345678910>>...19

/linux/include/net/
H A Dxdp.h2 /* include/net/xdp.h
17 * DOC: XDP RX-queue information
19 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
24 * reference to this xdp_rxq_info structure. This provides the XDP
34 * The struct is not directly tied to the XDP prog. A new XDP prog
45 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
51 /* XDP flags for ndo_xdp_xmit */
75 XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
76 XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
79 /* frags have unreadable mem, this can't be true for real XDP packets,
[all …]
H A Dxdp_sock_drv.h67 * matching the XDP core's xdp_data_hard_end() layout. in xsk_pool_get_rx_frame_size()
106 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_dma() argument
108 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma()
113 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_frame_dma() argument
115 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma()
131 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) in xsk_buff_alloc_batch() argument
133 return xp_alloc_batch(pool, xdp, max); in xsk_buff_alloc_batch()
141 static inline void xsk_buff_free(struct xdp_buff *xdp) in xsk_buff_free() argument
143 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free()
147 if (likely(!xdp_buff_has_frags(xdp))) in xsk_buff_free()
[all …]
/linux/include/net/libeth/
H A Dxsk.h7 #include <net/libeth/xdp.h>
19 * @bq: XDP Tx bulk to queue the head frag to
20 * @xdp: XSk buffer with the head to queue
25 struct libeth_xdp_buff *xdp) in libeth_xsk_tx_queue_head() argument
28 .xsk = xdp, in libeth_xsk_tx_queue_head()
29 __libeth_xdp_tx_len(xdp->base.data_end - xdp->data, in libeth_xsk_tx_queue_head()
33 if (likely(!xdp_buff_has_frags(&xdp->base))) in libeth_xsk_tx_queue_head()
43 * @bq: XDP Tx bulk to queue the frag to
57 * @bq: XDP Tx bulk to queue the frame to
58 * @xdp: XSk buffer to queue
[all …]
H A Dxdp.h43 * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
58 offsetof(struct xdp_buff_xsk, xdp.data));
114 * Return: number of XDP Tx queues the device needs to use.
317 * struct libeth_xdp_tx_frame - represents one XDP Tx element
324 * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
368 * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
369 * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
372 * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
376 * All XDP Tx operations except XSk xmit queue each frame to the bulk first
434 * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
[all …]
/linux/tools/testing/selftests/bpf/progs/
H A Dxdp_redirect_map.c16 SEC("xdp")
17 int xdp_redirect_map_0(struct xdp_md *xdp) in xdp_redirect_map_0() argument
22 SEC("xdp")
23 int xdp_redirect_map_1(struct xdp_md *xdp) in xdp_redirect_map_1() argument
28 SEC("xdp")
29 int xdp_redirect_map_2(struct xdp_md *xdp) in xdp_redirect_map_2() argument
41 static int xdp_count(struct xdp_md *xdp, __u32 key) in xdp_count() argument
43 void *data_end = (void *)(long)xdp->data_end; in xdp_count()
44 void *data = (void *)(long)xdp->data; in xdp_count()
61 SEC("xdp")
[all …]
H A Dxdp_features.c65 xdp_process_echo_packet(struct xdp_md *xdp, bool dut) in xdp_process_echo_packet() argument
67 void *data_end = (void *)(long)xdp->data_end; in xdp_process_echo_packet()
68 void *data = (void *)(long)xdp->data; in xdp_process_echo_packet()
135 xdp_update_stats(struct xdp_md *xdp, bool tx, bool dut) in xdp_update_stats() argument
139 if (xdp_process_echo_packet(xdp, tx)) in xdp_update_stats()
155 SEC("xdp")
156 int xdp_tester_check_tx(struct xdp_md *xdp) in xdp_tester_check_tx() argument
158 xdp_update_stats(xdp, true, false); in xdp_tester_check_tx()
163 SEC("xdp")
164 int xdp_tester_check_rx(struct xdp_md *xdp) in xdp_tester_check_rx() argument
[all …]
H A Dverifier_xdp_direct_packet_access.c8 SEC("xdp")
9 __description("XDP pkt read, pkt_end mangling, bad access 1")
29 SEC("xdp")
30 __description("XDP pkt read, pkt_end mangling, bad access 2")
50 SEC("xdp")
51 __description("XDP pkt read, pkt_data' > pkt_end, corner case, good access")
70 SEC("xdp")
71 __description("XDP pkt read, pkt_data' > pkt_end, bad access 1")
91 SEC("xdp")
92 __description("XDP pkt read, pkt_data' > pkt_end, bad access 2")
[all …]
H A Dxsk_xdp_progs.c22 SEC("xdp.frags") int xsk_def_prog(struct xdp_md *xdp) in xsk_def_prog() argument
27 SEC("xdp.frags") int xsk_xdp_drop(struct xdp_md *xdp) in xsk_xdp_drop() argument
38 SEC("xdp.frags") int xsk_xdp_populate_metadata(struct xdp_md *xdp) in xsk_xdp_populate_metadata() argument
45 err = bpf_xdp_adjust_meta(xdp, -(int)sizeof(struct xdp_info)); in xsk_xdp_populate_metadata()
49 data = (void *)(long)xdp->data; in xsk_xdp_populate_metadata()
50 data_meta = (void *)(long)xdp->data_meta; in xsk_xdp_populate_metadata()
61 SEC("xdp") int xsk_xdp_shared_umem(struct xdp_md *xdp) in xsk_xdp_shared_umem() argument
63 void *data = (void *)(long)xdp->data; in xsk_xdp_shared_umem()
64 void *data_end = (void *)(long)xdp->data_end; in xsk_xdp_shared_umem()
78 SEC("xdp.frags") int xsk_xdp_adjust_tail(struct xdp_md *xdp) in xsk_xdp_adjust_tail() argument
[all …]
H A Dtest_xdp_do_redirect.c11 * @MARK_IN: frame is being processed by the input XDP prog.
28 SEC("xdp")
29 int xdp_redirect(struct xdp_md *xdp) in xdp_redirect() argument
31 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_redirect()
32 void *data_end = (void *)(long)xdp->data_end; in xdp_redirect()
33 void *data = (void *)(long)xdp->data; in xdp_redirect()
41 if (xdp->ingress_ifindex != (__u32)ifindex_in) in xdp_redirect()
55 if (bpf_xdp_adjust_meta(xdp, sizeof(__u64))) in xdp_redirect()
85 SEC("xdp")
86 int xdp_count_pkts(struct xdp_md *xdp) in xdp_count_pkts() argument
[all …]
H A Dtest_xdp.c79 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
81 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
82 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
182 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
H A Dtest_xdp_loop.c75 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
77 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
78 void *data = (void *)(long)xdp->data; in handle_ipv4()
108 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
111 data = (void *)(long)xdp->data; in handle_ipv4()
112 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
148 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
150 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
151 void *data = (void *)(long)xdp->data; in handle_ipv6()
178 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
/linux/Documentation/bpf/
H A Dredirect.rst25 :doc: xdp redirect
29 those that do, not all of them support non-linear frames. Non-linear xdp
41 The following bpftrace command can be used to capture and count all XDP tracepoints:
45 sudo bpftrace -e 'tracepoint:xdp:* { @cnt[probe] = count(); }'
49 @cnt[tracepoint:xdp:mem_connect]: 18
50 @cnt[tracepoint:xdp:mem_disconnect]: 18
51 @cnt[tracepoint:xdp:xdp_exception]: 19605
52 @cnt[tracepoint:xdp:xdp_devmap_xmit]: 1393604
53 @cnt[tracepoint:xdp:xdp_redirect]: 22292200
56 The various xdp tracepoints can be found in ``source/include/trace/events/xdp.h``
[all …]
/linux/drivers/net/ethernet/broadcom/bnxt/
H A Dbnxt_xdp.c29 struct xdp_buff *xdp) in bnxt_xmit_bd() argument
39 if (xdp && xdp_buff_has_frags(xdp)) { in bnxt_xmit_bd()
40 sinfo = xdp_get_shared_info_from_buff(xdp); in bnxt_xmit_bd()
48 if (xdp) in bnxt_xmit_bd()
49 tx_buf->page = virt_to_head_page(xdp->data); in bnxt_xmit_bd()
97 struct xdp_buff *xdp) in __bnxt_xmit_xdp() argument
101 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); in __bnxt_xmit_xdp()
184 struct xdp_buff *xdp) in bnxt_xdp_buff_init() argument
199 xdp_init_buff(xdp, buflen, &rxr->xdp_rxq); in bnxt_xdp_buff_init()
200 xdp_prepare_buff(xdp, data_ptr - offset, offset, len, true); in bnxt_xdp_buff_init()
[all …]
/linux/net/core/
H A Dxdp.c2 /* net/core/xdp.c
21 #include <net/xdp.h>
23 #include <trace/events/xdp.h>
372 * xdp_reg_page_pool - register &page_pool as a memory provider for XDP
375 * Can be used to register pools manually without connecting to any XDP RxQ
376 * info, so that the XDP layer will be aware of them. Then, they can be
409 * @xdp_rxq: XDP RxQ info to attach the pool to
427 /* XDP RX runs under NAPI protection, and in different delivery error
434 bool napi_direct, struct xdp_buff *xdp) in __xdp_return() argument
455 xsk_buff_free(xdp); in __xdp_return()
[all …]
/linux/drivers/net/ethernet/intel/libeth/
H A Dxdp.c8 #include <net/libeth/xdp.h>
29 netdev_warn(dev, "XDPSQ sharing enabled, possible XDP Tx slowdown\n"); in __libeth_xdpsq_get()
114 * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames
115 * @bq: XDP Tx frame bulk
120 * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust
205 * @src: source XDP buffer
208 * Use the fields from the passed XDP buffer to initialize the stash on the
228 LIBETH_XDP_ONSTACK_BUFF(xdp); in __libeth_xdp_return_stash()
230 libeth_xdp_load_stash(xdp, stash); in __libeth_xdp_return_stash()
231 libeth_xdp_return_buff_slow(xdp); in __libeth_xdp_return_stash()
[all …]
H A Dxsk.c31 * @xdp: buffer to free
36 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp) in libeth_xsk_buff_free_slow() argument
38 xsk_buff_free(&xdp->base); in libeth_xsk_buff_free_slow()
45 * @xdp: frag buffer
54 struct libeth_xdp_buff *xdp) in libeth_xsk_buff_add_frag() argument
56 if (!xsk_buff_add_frag(&head->base, &xdp->base)) in libeth_xsk_buff_add_frag()
62 libeth_xsk_buff_free_slow(xdp); in libeth_xsk_buff_add_frag()
72 * @xdp: buffer to account
78 const struct libeth_xdp_buff *xdp) in libeth_xsk_buff_stats_frags() argument
80 libeth_xdp_buff_stats_frags(rs, xdp); in libeth_xsk_buff_stats_frags()
[all …]
/linux/Documentation/networking/
H A Dxdp-rx-metadata.rst4 XDP RX Metadata
7 This document describes how an eXpress Data Path (XDP) program can access
14 XDP has access to a set of kfuncs to manipulate the metadata in an XDP frame.
16 implement these kfuncs. The set of kfuncs is declared in ``include/net/xdp.h``
22 .. kernel-doc:: net/core/xdp.c
25 .. kernel-doc:: net/core/xdp.c
28 .. kernel-doc:: net/core/xdp.c
31 An XDP program can use these kfuncs to read the metadata into stack
33 consumers, an XDP program can store it into the metadata area carried
42 Within an XDP frame, the metadata layout (accessed via ``xdp_buff``) is
[all …]
/linux/drivers/net/vmxnet3/
H A Dvmxnet3_xdp.c48 NL_SET_ERR_MSG_FMT_MOD(extack, "MTU %u too large for XDP", in vmxnet3_xdp_set()
54 NL_SET_ERR_MSG_MOD(extack, "LRO is not supported with XDP"); in vmxnet3_xdp_set()
86 "failed to re-create rx queues for XDP."); in vmxnet3_xdp_set()
92 "failed to activate device for XDP."); in vmxnet3_xdp_set()
100 /* This is the main xdp call used by kernel to set/unset eBPF program. */
152 } else { /* XDP buffer from page pool */ in vmxnet3_xdp_xmit_frame()
261 vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct xdp_buff *xdp, in vmxnet3_run_xdp() argument
270 act = bpf_prog_run_xdp(prog, xdp); in vmxnet3_run_xdp()
271 page = virt_to_page(xdp->data_hard_start); in vmxnet3_run_xdp()
277 err = xdp_do_redirect(rq->adapter->netdev, xdp, prog); in vmxnet3_run_xdp()
[all …]
/linux/drivers/net/ethernet/intel/igb/
H A Digb_xsk.c6 #include <net/xdp.h>
176 static u16 igb_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in igb_fill_rx_descs() argument
187 buffs = xsk_buff_alloc_batch(pool, xdp, count); in igb_fill_rx_descs()
189 dma = xsk_buff_xdp_get_dma(*xdp); in igb_fill_rx_descs()
194 xdp++; in igb_fill_rx_descs()
207 struct xdp_buff **xdp; in igb_alloc_rx_buffers_zc() local
210 xdp = &rx_ring->rx_buffer_info_zc[ntu]; in igb_alloc_rx_buffers_zc()
213 nb_buffs_extra = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, in igb_alloc_rx_buffers_zc()
220 xdp = rx_ring->rx_buffer_info_zc; in igb_alloc_rx_buffers_zc()
225 nb_buffs = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, count); in igb_alloc_rx_buffers_zc()
[all …]
/linux/drivers/net/ethernet/intel/ice/
H A Dice_xsk.c6 #include <net/libeth/xdp.h>
8 #include <net/xdp.h>
249 * @xdp: SW ring of xdp_buff that will hold the buffers
260 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in ice_fill_rx_descs() argument
267 buffs = xsk_buff_alloc_batch(pool, xdp, count); in ice_fill_rx_descs()
269 dma = xsk_buff_xdp_get_dma(*xdp); in ice_fill_rx_descs()
274 xdp++; in ice_fill_rx_descs()
299 struct xdp_buff **xdp; in __ice_alloc_rx_bufs_zc() local
302 xdp = ice_xdp_buf(rx_ring, ntu); in __ice_alloc_rx_bufs_zc()
305 nb_buffs_extra = ice_fill_rx_descs(xsk_pool, xdp, rx_desc, in __ice_alloc_rx_bufs_zc()
[all …]
/linux/drivers/net/ethernet/intel/ixgbe/
H A Dixgbe_xsk.c6 #include <net/xdp.h>
99 struct xdp_buff *xdp) in ixgbe_run_xdp_zc() argument
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
125 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
168 bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); in ixgbe_alloc_rx_buffers_zc()
169 if (!bi->xdp) { in ixgbe_alloc_rx_buffers_zc()
174 dma = xsk_buff_xdp_get_dma(bi->xdp); in ixgbe_alloc_rx_buffers_zc()
214 const struct xdp_buff *xdp) in ixgbe_construct_skb_zc() argument
216 unsigned int totalsize = xdp->data_end - xdp->data_meta; in ixgbe_construct_skb_zc()
[all …]
/linux/tools/testing/selftests/net/
H A Dveth.sh249 ip -n $NS_DST link set dev veth$DST xdp object ${BPF_FILE} section xdp
250 chk_gro_flag "gro vs xdp while down - gro flag off" $DST off
253 ip -n $NS_DST link set dev veth$DST xdp off
254 chk_gro_flag " - after xdp off" $DST off
257 ip -n $NS_SRC link set dev veth$SRC xdp object ${BPF_FILE} section xdp
258 chk_gro_flag " - after peer xdp" $DST off
263 ip -n $NS_DST link set dev veth$DST xdp object ${BPF_FILE} section xdp
265 chk_gro_flag "gro vs xdp while down - gro flag on" $DST on
268 ip -n $NS_DST link set dev veth$DST xdp off
269 chk_gro_flag " - after xdp off" $DST on
[all …]
/linux/drivers/net/hyperv/
H A Dnetvsc_bpf.c17 #include <net/xdp.h>
25 struct xdp_buff *xdp) in netvsc_run_xdp() argument
35 xdp->data_hard_start = NULL; in netvsc_run_xdp()
56 xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); in netvsc_run_xdp()
57 xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); in netvsc_run_xdp()
59 memcpy(xdp->data, data, len); in netvsc_run_xdp()
61 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
73 if (!xdp_do_redirect(ndev, xdp, prog)) { in netvsc_run_xdp()
107 xdp->data_hard_start = NULL; in netvsc_run_xdp()
138 netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n", in netvsc_xdp_set()
[all …]
/linux/tools/testing/selftests/drivers/net/
H A Dxdp.py5 This file contains tests to verify native XDP support in network drivers.
7 directory, with each test focusing on a specific aspect of XDP functionality.
23 """Enum for XDP configuration options."""
31 """Enum for XDP actions."""
40 """Enum for XDP statistics."""
53 xdp_sec: str = "xdp" # XDP section name (e.g., "xdp" or "xdp.frags")
101 Loads an XDP program onto a network interface.
108 dict: A dictionary containing the XDP program ID, name, and associated map IDs.
123 prog_info["id"] = xdp_info["xdp"]["prog"]["id"]
124 prog_info["name"] = xdp_info["xdp"]["prog"]["name"]
[all …]
/linux/samples/bpf/
H A Dxdp_tx_iptunnel_kern.c77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]

12345678910>>...19